Commit 2bffd810157a8e7c0b2500a13921dd8c45694b8a | zig

CMakeLists.txt-3

 @@ -209,9 +209,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/array_list.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/ascii.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/atomic.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/atomic/Atomic.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/atomic/queue.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/BitStack.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"

lib/std/Thread.zig+7-8

...	@@ -8,7 +8,6 @@ const math = std.math;	...	@@ -8,7 +8,6 @@ const math = std.math;
8	const os = std.os;	8	const os = std.os;
9	const assert = std.debug.assert;	9	const assert = std.debug.assert;
10	const target = builtin.target;	10	const target = builtin.target;
11	const Atomic = std.atomic.Atomic;
12		11
13	pub const Futex = @import("Thread/Futex.zig");	12	pub const Futex = @import("Thread/Futex.zig");
14	pub const ResetEvent = @import("Thread/ResetEvent.zig");	13	pub const ResetEvent = @import("Thread/ResetEvent.zig");
...	@@ -388,7 +387,7 @@ pub fn yield() YieldError!void {	...	@@ -388,7 +387,7 @@ pub fn yield() YieldError!void {
388	}	387	}
389		388
390	/// State to synchronize detachment of spawner thread to spawned thread	389	/// State to synchronize detachment of spawner thread to spawned thread
391	const Completion = Atomic(enum(u8) {	390	const Completion = std.atomic.Value(enum(u8) {
392	running,	391	running,
393	detached,	392	detached,
394	completed,	393	completed,
...	@@ -746,7 +745,7 @@ const WasiThreadImpl = struct {	...	@@ -746,7 +745,7 @@ const WasiThreadImpl = struct {
746		745
747	const WasiThread = struct {	746	const WasiThread = struct {
748	/// Thread ID	747	/// Thread ID
749	tid: Atomic(i32) = Atomic(i32).init(0),	748	tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(0),
750	/// Contains all memory which was allocated to bootstrap this thread, including:	749	/// Contains all memory which was allocated to bootstrap this thread, including:
751	/// - Guard page	750	/// - Guard page
752	/// - Stack	751	/// - Stack
...	@@ -784,7 +783,7 @@ const WasiThreadImpl = struct {	...	@@ -784,7 +783,7 @@ const WasiThreadImpl = struct {
784	original_stack_pointer: [*]u8,	783	original_stack_pointer: [*]u8,
785	};	784	};
786		785
787	const State = Atomic(enum(u8) { running, completed, detached });	786	const State = std.atomic.Value(enum(u8) { running, completed, detached });
788		787
789	fn getCurrentId() Id {	788	fn getCurrentId() Id {
790	return tls_thread_id;	789	return tls_thread_id;
...	@@ -1048,7 +1047,7 @@ const LinuxThreadImpl = struct {	...	@@ -1048,7 +1047,7 @@ const LinuxThreadImpl = struct {
1048		1047
1049	const ThreadCompletion = struct {	1048	const ThreadCompletion = struct {
1050	completion: Completion = Completion.init(.running),	1049	completion: Completion = Completion.init(.running),
1051	child_tid: Atomic(i32) = Atomic(i32).init(1),	1050	child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1),
1052	parent_tid: i32 = undefined,	1051	parent_tid: i32 = undefined,
1053	mapped: []align(std.mem.page_size) u8,	1052	mapped: []align(std.mem.page_size) u8,
1054		1053
...	@@ -1304,7 +1303,7 @@ const LinuxThreadImpl = struct {	...	@@ -1304,7 +1303,7 @@ const LinuxThreadImpl = struct {
1304	@intFromPtr(instance),	1303	@intFromPtr(instance),
1305	&instance.thread.parent_tid,	1304	&instance.thread.parent_tid,
1306	tls_ptr,	1305	tls_ptr,
1307	&instance.thread.child_tid.value,	1306	&instance.thread.child_tid.raw,
1308	))) {	1307	))) {
1309	.SUCCESS => return Impl{ .thread = &instance.thread },	1308	.SUCCESS => return Impl{ .thread = &instance.thread },
1310	.AGAIN => return error.ThreadQuotaExceeded,	1309	.AGAIN => return error.ThreadQuotaExceeded,
...	@@ -1346,7 +1345,7 @@ const LinuxThreadImpl = struct {	...	@@ -1346,7 +1345,7 @@ const LinuxThreadImpl = struct {
1346	}	1345	}
1347		1346
1348	switch (linux.getErrno(linux.futex_wait(	1347	switch (linux.getErrno(linux.futex_wait(
1349	&self.thread.child_tid.value,	1348	&self.thread.child_tid.raw,
1350	linux.FUTEX.WAIT,	1349	linux.FUTEX.WAIT,
1351	tid,	1350	tid,
1352	null,	1351	null,
...	@@ -1387,7 +1386,7 @@ test "setName, getName" {	...	@@ -1387,7 +1386,7 @@ test "setName, getName" {
1387	test_done_event: ResetEvent = .{},	1386	test_done_event: ResetEvent = .{},
1388	thread_done_event: ResetEvent = .{},	1387	thread_done_event: ResetEvent = .{},
1389		1388
1390	done: std.atomic.Atomic(bool) = std.atomic.Atomic(bool).init(false),	1389	done: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),
1391	thread: Thread = undefined,	1390	thread: Thread = undefined,
1392		1391
1393	pub fn run(ctx: *@This()) !void {	1392	pub fn run(ctx: *@This()) !void {

lib/std/Thread/Condition.zig+6-7

lib/std/Thread/Futex.zig+37-37

lib/std/Thread/Mutex.zig+9-18

 @@ -26,7 +26,6 @@ const Mutex = @This();
 const os = std.os;
 const assert = std.debug.assert;
 const testing = std.testing;
-const Atomic = std.atomic.Atomic;
 const Thread = std.Thread;
 const Futex = Thread.Futex;
 @@ -67,7 +66,7 @@ else
     FutexImpl;
 const DebugImpl = struct {
-    locking_thread: Atomic(Thread.Id) = Atomic(Thread.Id).init(0), // 0 means it's not locked.
+    locking_thread: std.atomic.Value(Thread.Id) = std.atomic.Value(Thread.Id).init(0), // 0 means it's not locked.
     impl: ReleaseImpl = .{},
     inline fn tryLock(self: *@This()) bool {
 @@ -151,37 +150,29 @@ const DarwinImpl = struct {
 };
 const FutexImpl = struct {
-    state: Atomic(u32) = Atomic(u32).init(unlocked),
+    state: std.atomic.Value(u32) = std.atomic.Value(u32).init(unlocked),
-    const unlocked = 0b00;
+    const unlocked: u32 = 0b00;
-    const locked = 0b01;
+    const locked: u32 = 0b01;
-    const contended = 0b11; // must contain the `locked` bit for x86 optimization below
+    const contended: u32 = 0b11; // must contain the `locked` bit for x86 optimization below
-    fn tryLock(self: *@This()) bool {
-        // Lock with compareAndSwap instead of tryCompareAndSwap to avoid reporting spurious CAS failure.
-        return self.lockFast("compareAndSwap");
-    }
     fn lock(self: *@This()) void {
-        // Lock with tryCompareAndSwap instead of compareAndSwap due to being more inline-able on LL/SC archs like ARM.
+        if (!self.tryLock())
-        if (!self.lockFast("tryCompareAndSwap")) {
             self.lockSlow();
-        }
     }
-    inline fn lockFast(self: *@This(), comptime cas_fn_name: []const u8) bool {
+    fn tryLock(self: *@This()) bool {
         // On x86, use `lock bts` instead of `lock cmpxchg` as:
         // - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
         // - `lock bts` is smaller instruction-wise which makes it better for inlining
         if (comptime builtin.target.cpu.arch.isX86()) {
-            const locked_bit = @ctz(@as(u32, locked));
+            const locked_bit = @ctz(locked);
             return self.state.bitSet(locked_bit, .Acquire) == 0;
         }
         // Acquire barrier ensures grabbing the lock happens before the critical section
         // and that the previous lock holder's critical section happens before we grab the lock.
-        const casFn = @field(@TypeOf(self.state), cas_fn_name);
+        return self.state.cmpxchgWeak(unlocked, locked, .Acquire, .Monotonic) == null;
-        return casFn(&self.state, unlocked, locked, .Acquire, .Monotonic) == null;
     }
     fn lockSlow(self: *@This()) void {

lib/std/Thread/ResetEvent.zig+3-4

...	@@ -9,7 +9,6 @@ const ResetEvent = @This();	...	@@ -9,7 +9,6 @@ const ResetEvent = @This();
9	const os = std.os;	9	const os = std.os;
10	const assert = std.debug.assert;	10	const assert = std.debug.assert;
11	const testing = std.testing;	11	const testing = std.testing;
12	const Atomic = std.atomic.Atomic;
13	const Futex = std.Thread.Futex;	12	const Futex = std.Thread.Futex;
14		13
15	impl: Impl = .{},	14	impl: Impl = .{},
...	@@ -89,7 +88,7 @@ const SingleThreadedImpl = struct {	...	@@ -89,7 +88,7 @@ const SingleThreadedImpl = struct {
89	};	88	};
90		89
91	const FutexImpl = struct {	90	const FutexImpl = struct {
92	state: Atomic(u32) = Atomic(u32).init(unset),	91	state: std.atomic.Value(u32) = std.atomic.Value(u32).init(unset),
93		92
94	const unset = 0;	93	const unset = 0;
95	const waiting = 1;	94	const waiting = 1;
...	@@ -115,7 +114,7 @@ const FutexImpl = struct {	...	@@ -115,7 +114,7 @@ const FutexImpl = struct {
115	// We avoid using any strict barriers until the end when we know the ResetEvent is set.	114	// We avoid using any strict barriers until the end when we know the ResetEvent is set.
116	var state = self.state.load(.Monotonic);	115	var state = self.state.load(.Monotonic);
117	if (state == unset) {	116	if (state == unset) {
118	state = self.state.compareAndSwap(state, waiting, .Monotonic, .Monotonic) orelse waiting;	117	state = self.state.cmpxchgStrong(state, waiting, .Monotonic, .Monotonic) orelse waiting;
119	}	118	}
120		119
121	// Wait until the ResetEvent is set since the state is waiting.	120	// Wait until the ResetEvent is set since the state is waiting.
...	@@ -252,7 +251,7 @@ test "ResetEvent - broadcast" {	...	@@ -252,7 +251,7 @@ test "ResetEvent - broadcast" {
252	const num_threads = 10;	251	const num_threads = 10;
253	const Barrier = struct {	252	const Barrier = struct {
254	event: ResetEvent = .{},	253	event: ResetEvent = .{},
255	counter: Atomic(usize) = Atomic(usize).init(num_threads),	254	counter: std.atomic.Value(usize) = std.atomic.Value(usize).init(num_threads),
256		255
257	fn wait(self: *@This()) void {	256	fn wait(self: *@This()) void {
258	if (self.counter.fetchSub(1, .AcqRel) == 1) {	257	if (self.counter.fetchSub(1, .AcqRel) == 1) {

lib/std/Thread/RwLock.zig+1-1

...	@@ -307,7 +307,7 @@ test "RwLock - concurrent access" {	...	@@ -307,7 +307,7 @@ test "RwLock - concurrent access" {
307		307
308	rwl: RwLock = .{},	308	rwl: RwLock = .{},
309	writes: usize = 0,	309	writes: usize = 0,
310	reads: std.atomic.Atomic(usize) = std.atomic.Atomic(usize).init(0),	310	reads: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
311		311
312	term1: usize = 0,	312	term1: usize = 0,
313	term2: usize = 0,	313	term2: usize = 0,

lib/std/Thread/WaitGroup.zig+1-2

 @@ -1,12 +1,11 @@
 const std = @import("std");
-const Atomic = std.atomic.Atomic;
 const assert = std.debug.assert;
 const WaitGroup = @This();
 const is_waiting: usize = 1 << 0;
 const one_pending: usize = 1 << 1;
-state: Atomic(usize) = Atomic(usize).init(0),
+state: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
 event: std.Thread.ResetEvent = .{},
 pub fn start(self: *WaitGroup) void {

lib/std/atomic.zig+369-31

...	@@ -1,40 +1,374 @@	...	@@ -1,40 +1,374 @@
1	const std = @import("std.zig");	1	/// This is a thin wrapper around a primitive value to prevent accidental data races.
2	const builtin = @import("builtin");	2	pub fn Value(comptime T: type) type {
		3	return extern struct {
		4	/// Care must be taken to avoid data races when interacting with this field directly.
		5	raw: T,
		6
		7	const Self = @This();
		8
		9	pub fn init(value: T) Self {
		10	return .{ .raw = value };
		11	}
		12
		13	/// Perform an atomic fence which uses the atomic value as a hint for
		14	/// the modification order. Use this when you want to imply a fence on
		15	/// an atomic variable without necessarily performing a memory access.
		16	pub inline fn fence(self: *Self, comptime order: AtomicOrder) void {
		17	// LLVM's ThreadSanitizer doesn't support the normal fences so we specialize for it.
		18	if (builtin.sanitize_thread) {
		19	const tsan = struct {
		20	extern "c" fn __tsan_acquire(addr: *anyopaque) void;
		21	extern "c" fn __tsan_release(addr: *anyopaque) void;
		22	};
		23
		24	const addr: *anyopaque = self;
		25	return switch (order) {
		26	.Unordered, .Monotonic => @compileError(@tagName(order) ++ " only applies to atomic loads and stores"),
		27	.Acquire => tsan.__tsan_acquire(addr),
		28	.Release => tsan.__tsan_release(addr),
		29	.AcqRel, .SeqCst => {
		30	tsan.__tsan_acquire(addr);
		31	tsan.__tsan_release(addr);
		32	},
		33	};
		34	}
		35
		36	return @fence(order);
		37	}
		38
		39	pub inline fn load(self: *const Self, comptime order: AtomicOrder) T {
		40	return @atomicLoad(T, &self.raw, order);
		41	}
		42
		43	pub inline fn store(self: *Self, value: T, comptime order: AtomicOrder) void {
		44	@atomicStore(T, &self.raw, value, order);
		45	}
		46
		47	pub inline fn swap(self: *Self, operand: T, comptime order: AtomicOrder) T {
		48	return @atomicRmw(T, &self.raw, .Xchg, operand, order);
		49	}
		50
		51	pub inline fn cmpxchgWeak(
		52	self: *Self,
		53	expected_value: T,
		54	new_value: T,
		55	comptime success_order: AtomicOrder,
		56	comptime fail_order: AtomicOrder,
		57	) ?T {
		58	return @cmpxchgWeak(T, &self.raw, expected_value, new_value, success_order, fail_order);
		59	}
		60
		61	pub inline fn cmpxchgStrong(
		62	self: *Self,
		63	expected_value: T,
		64	new_value: T,
		65	comptime success_order: AtomicOrder,
		66	comptime fail_order: AtomicOrder,
		67	) ?T {
		68	return @cmpxchgStrong(T, &self.raw, expected_value, new_value, success_order, fail_order);
		69	}
		70
		71	pub inline fn fetchAdd(self: *Self, operand: T, comptime order: AtomicOrder) T {
		72	return @atomicRmw(T, &self.raw, .Add, operand, order);
		73	}
		74
		75	pub inline fn fetchSub(self: *Self, operand: T, comptime order: AtomicOrder) T {
		76	return @atomicRmw(T, &self.raw, .Sub, operand, order);
		77	}
		78
		79	pub inline fn fetchMin(self: *Self, operand: T, comptime order: AtomicOrder) T {
		80	return @atomicRmw(T, &self.raw, .Min, operand, order);
		81	}
		82
		83	pub inline fn fetchMax(self: *Self, operand: T, comptime order: AtomicOrder) T {
		84	return @atomicRmw(T, &self.raw, .Max, operand, order);
		85	}
		86
		87	pub inline fn fetchAnd(self: *Self, operand: T, comptime order: AtomicOrder) T {
		88	return @atomicRmw(T, &self.raw, .And, operand, order);
		89	}
		90
		91	pub inline fn fetchNand(self: *Self, operand: T, comptime order: AtomicOrder) T {
		92	return @atomicRmw(T, &self.raw, .Nand, operand, order);
		93	}
		94
		95	pub inline fn fetchXor(self: *Self, operand: T, comptime order: AtomicOrder) T {
		96	return @atomicRmw(T, &self.raw, .Xor, operand, order);
		97	}
3		98
4	pub const Ordering = std.builtin.AtomicOrder;	99	pub inline fn fetchOr(self: *Self, operand: T, comptime order: AtomicOrder) T {
		100	return @atomicRmw(T, &self.raw, .Or, operand, order);
		101	}
5		102
6	pub const Stack = @import("atomic/stack.zig").Stack;	103	pub inline fn rmw(
7	pub const Queue = @import("atomic/queue.zig").Queue;	104	self: *Self,
8	pub const Atomic = @import("atomic/Atomic.zig").Atomic;	105	comptime op: std.builtin.AtomicRmwOp,
		106	operand: T,
		107	comptime order: AtomicOrder,
		108	) T {
		109	return @atomicRmw(T, &self.raw, op, operand, order);
		110	}
9		111
10	test {	112	const Bit = std.math.Log2Int(T);
11	_ = @import("atomic/stack.zig");	113
12	_ = @import("atomic/queue.zig");	114	/// Marked `inline` so that if `bit` is comptime-known, the instruction
13	_ = @import("atomic/Atomic.zig");	115	/// can be lowered to a more efficient machine code instruction if
		116	/// possible.
		117	pub inline fn bitSet(self: *Self, bit: Bit, comptime order: AtomicOrder) u1 {
		118	const mask = @as(T, 1) << bit;
		119	const value = self.fetchOr(mask, order);
		120	return @intFromBool(value & mask != 0);
		121	}
		122
		123	/// Marked `inline` so that if `bit` is comptime-known, the instruction
		124	/// can be lowered to a more efficient machine code instruction if
		125	/// possible.
		126	pub inline fn bitReset(self: *Self, bit: Bit, comptime order: AtomicOrder) u1 {
		127	const mask = @as(T, 1) << bit;
		128	const value = self.fetchAnd(~mask, order);
		129	return @intFromBool(value & mask != 0);
		130	}
		131
		132	/// Marked `inline` so that if `bit` is comptime-known, the instruction
		133	/// can be lowered to a more efficient machine code instruction if
		134	/// possible.
		135	pub inline fn bitToggle(self: *Self, bit: Bit, comptime order: AtomicOrder) u1 {
		136	const mask = @as(T, 1) << bit;
		137	const value = self.fetchXor(mask, order);
		138	return @intFromBool(value & mask != 0);
		139	}
		140	};
14	}	141	}
15		142
16	pub inline fn fence(comptime ordering: Ordering) void {	143	test Value {
17	switch (ordering) {	144	const RefCount = struct {
18	.Acquire, .Release, .AcqRel, .SeqCst => {	145	count: Value(usize),
19	@fence(ordering);	146	dropFn: const fn (RefCount) void,
20	},	147
21	else => {	148	const RefCount = @This();
22	@compileLog(ordering, " only applies to a given memory location");	149
23	},	150	fn ref(rc: *RefCount) void {
		151	// No ordering necessary; just updating a counter.
		152	_ = rc.count.fetchAdd(1, .Monotonic);
		153	}
		154
		155	fn unref(rc: *RefCount) void {
		156	// Release ensures code before unref() happens-before the
		157	// count is decremented as dropFn could be called by then.
		158	if (rc.count.fetchSub(1, .Release) == 1) {
		159	// Acquire ensures count decrement and code before
		160	// previous unrefs()s happens-before we call dropFn
		161	// below.
		162	// Another alternative is to use .AcqRel on the
		163	// fetchSub count decrement but it's extra barrier in
		164	// possibly hot path.
		165	rc.count.fence(.Acquire);
		166	(rc.dropFn)(rc);
		167	}
		168	}
		169
		170	fn noop(rc: *RefCount) void {
		171	_ = rc;
		172	}
		173	};
		174
		175	var ref_count: RefCount = .{
		176	.count = Value(usize).init(0),
		177	.dropFn = RefCount.noop,
		178	};
		179	ref_count.ref();
		180	ref_count.unref();
		181	}
		182
		183	test "Value.swap" {
		184	var x = Value(usize).init(5);
		185	try testing.expectEqual(@as(usize, 5), x.swap(10, .SeqCst));
		186	try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
		187
		188	const E = enum(usize) { a, b, c };
		189	var y = Value(E).init(.c);
		190	try testing.expectEqual(E.c, y.swap(.a, .SeqCst));
		191	try testing.expectEqual(E.a, y.load(.SeqCst));
		192
		193	var z = Value(f32).init(5.0);
		194	try testing.expectEqual(@as(f32, 5.0), z.swap(10.0, .SeqCst));
		195	try testing.expectEqual(@as(f32, 10.0), z.load(.SeqCst));
		196
		197	var a = Value(bool).init(false);
		198	try testing.expectEqual(false, a.swap(true, .SeqCst));
		199	try testing.expectEqual(true, a.load(.SeqCst));
		200
		201	var b = Value(?*u8).init(null);
		202	try testing.expectEqual(@as(?u8, null), b.swap(@as(?u8, @ptrFromInt(@alignOf(u8))), .SeqCst));
		203	try testing.expectEqual(@as(?*u8, @ptrFromInt(@alignOf(u8))), b.load(.SeqCst));
		204	}
		205
		206	test "Value.store" {
		207	var x = Value(usize).init(5);
		208	x.store(10, .SeqCst);
		209	try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
		210	}
		211
		212	test "Value.cmpxchgWeak" {
		213	var x = Value(usize).init(0);
		214
		215	try testing.expectEqual(@as(?usize, 0), x.cmpxchgWeak(1, 0, .SeqCst, .SeqCst));
		216	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		217
		218	while (x.cmpxchgWeak(0, 1, .SeqCst, .SeqCst)) \|_\| {}
		219	try testing.expectEqual(@as(usize, 1), x.load(.SeqCst));
		220
		221	while (x.cmpxchgWeak(1, 0, .SeqCst, .SeqCst)) \|_\| {}
		222	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		223	}
		224
		225	test "Value.cmpxchgStrong" {
		226	var x = Value(usize).init(0);
		227	try testing.expectEqual(@as(?usize, 0), x.cmpxchgStrong(1, 0, .SeqCst, .SeqCst));
		228	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		229	try testing.expectEqual(@as(?usize, null), x.cmpxchgStrong(0, 1, .SeqCst, .SeqCst));
		230	try testing.expectEqual(@as(usize, 1), x.load(.SeqCst));
		231	try testing.expectEqual(@as(?usize, null), x.cmpxchgStrong(1, 0, .SeqCst, .SeqCst));
		232	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		233	}
		234
		235	test "Value.fetchAdd" {
		236	var x = Value(usize).init(5);
		237	try testing.expectEqual(@as(usize, 5), x.fetchAdd(5, .SeqCst));
		238	try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
		239	try testing.expectEqual(@as(usize, 10), x.fetchAdd(std.math.maxInt(usize), .SeqCst));
		240	try testing.expectEqual(@as(usize, 9), x.load(.SeqCst));
		241	}
		242
		243	test "Value.fetchSub" {
		244	var x = Value(usize).init(5);
		245	try testing.expectEqual(@as(usize, 5), x.fetchSub(5, .SeqCst));
		246	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		247	try testing.expectEqual(@as(usize, 0), x.fetchSub(1, .SeqCst));
		248	try testing.expectEqual(@as(usize, std.math.maxInt(usize)), x.load(.SeqCst));
		249	}
		250
		251	test "Value.fetchMin" {
		252	var x = Value(usize).init(5);
		253	try testing.expectEqual(@as(usize, 5), x.fetchMin(0, .SeqCst));
		254	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		255	try testing.expectEqual(@as(usize, 0), x.fetchMin(10, .SeqCst));
		256	try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
		257	}
		258
		259	test "Value.fetchMax" {
		260	var x = Value(usize).init(5);
		261	try testing.expectEqual(@as(usize, 5), x.fetchMax(10, .SeqCst));
		262	try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
		263	try testing.expectEqual(@as(usize, 10), x.fetchMax(5, .SeqCst));
		264	try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
		265	}
		266
		267	test "Value.fetchAnd" {
		268	var x = Value(usize).init(0b11);
		269	try testing.expectEqual(@as(usize, 0b11), x.fetchAnd(0b10, .SeqCst));
		270	try testing.expectEqual(@as(usize, 0b10), x.load(.SeqCst));
		271	try testing.expectEqual(@as(usize, 0b10), x.fetchAnd(0b00, .SeqCst));
		272	try testing.expectEqual(@as(usize, 0b00), x.load(.SeqCst));
		273	}
		274
		275	test "Value.fetchNand" {
		276	var x = Value(usize).init(0b11);
		277	try testing.expectEqual(@as(usize, 0b11), x.fetchNand(0b10, .SeqCst));
		278	try testing.expectEqual(~@as(usize, 0b10), x.load(.SeqCst));
		279	try testing.expectEqual(~@as(usize, 0b10), x.fetchNand(0b00, .SeqCst));
		280	try testing.expectEqual(~@as(usize, 0b00), x.load(.SeqCst));
		281	}
		282
		283	test "Value.fetchOr" {
		284	var x = Value(usize).init(0b11);
		285	try testing.expectEqual(@as(usize, 0b11), x.fetchOr(0b100, .SeqCst));
		286	try testing.expectEqual(@as(usize, 0b111), x.load(.SeqCst));
		287	try testing.expectEqual(@as(usize, 0b111), x.fetchOr(0b010, .SeqCst));
		288	try testing.expectEqual(@as(usize, 0b111), x.load(.SeqCst));
		289	}
		290
		291	test "Value.fetchXor" {
		292	var x = Value(usize).init(0b11);
		293	try testing.expectEqual(@as(usize, 0b11), x.fetchXor(0b10, .SeqCst));
		294	try testing.expectEqual(@as(usize, 0b01), x.load(.SeqCst));
		295	try testing.expectEqual(@as(usize, 0b01), x.fetchXor(0b01, .SeqCst));
		296	try testing.expectEqual(@as(usize, 0b00), x.load(.SeqCst));
		297	}
		298
		299	test "Value.bitSet" {
		300	var x = Value(usize).init(0);
		301
		302	for (0..@bitSizeOf(usize)) \|bit_index\| {
		303	const bit = @as(std.math.Log2Int(usize), @intCast(bit_index));
		304	const mask = @as(usize, 1) << bit;
		305
		306	// setting the bit should change the bit
		307	try testing.expect(x.load(.SeqCst) & mask == 0);
		308	try testing.expectEqual(@as(u1, 0), x.bitSet(bit, .SeqCst));
		309	try testing.expect(x.load(.SeqCst) & mask != 0);
		310
		311	// setting it again shouldn't change the bit
		312	try testing.expectEqual(@as(u1, 1), x.bitSet(bit, .SeqCst));
		313	try testing.expect(x.load(.SeqCst) & mask != 0);
		314
		315	// all the previous bits should have not changed (still be set)
		316	for (0..bit_index) \|prev_bit_index\| {
		317	const prev_bit = @as(std.math.Log2Int(usize), @intCast(prev_bit_index));
		318	const prev_mask = @as(usize, 1) << prev_bit;
		319	try testing.expect(x.load(.SeqCst) & prev_mask != 0);
		320	}
24	}	321	}
25	}	322	}
26		323
27	pub inline fn compilerFence(comptime ordering: Ordering) void {	324	test "Value.bitReset" {
28	switch (ordering) {	325	var x = Value(usize).init(0);
29	.Acquire, .Release, .AcqRel, .SeqCst => asm volatile ("" ::: "memory"),	326
30	else => @compileLog(ordering, " only applies to a given memory location"),	327	for (0..@bitSizeOf(usize)) \|bit_index\| {
		328	const bit = @as(std.math.Log2Int(usize), @intCast(bit_index));
		329	const mask = @as(usize, 1) << bit;
		330	x.raw \|= mask;
		331
		332	// unsetting the bit should change the bit
		333	try testing.expect(x.load(.SeqCst) & mask != 0);
		334	try testing.expectEqual(@as(u1, 1), x.bitReset(bit, .SeqCst));
		335	try testing.expect(x.load(.SeqCst) & mask == 0);
		336
		337	// unsetting it again shouldn't change the bit
		338	try testing.expectEqual(@as(u1, 0), x.bitReset(bit, .SeqCst));
		339	try testing.expect(x.load(.SeqCst) & mask == 0);
		340
		341	// all the previous bits should have not changed (still be reset)
		342	for (0..bit_index) \|prev_bit_index\| {
		343	const prev_bit = @as(std.math.Log2Int(usize), @intCast(prev_bit_index));
		344	const prev_mask = @as(usize, 1) << prev_bit;
		345	try testing.expect(x.load(.SeqCst) & prev_mask == 0);
		346	}
31	}	347	}
32	}	348	}
33		349
34	test "fence/compilerFence" {	350	test "Value.bitToggle" {
35	inline for (.{ .Acquire, .Release, .AcqRel, .SeqCst }) \|ordering\| {	351	var x = Value(usize).init(0);
36	compilerFence(ordering);	352
37	fence(ordering);	353	for (0..@bitSizeOf(usize)) \|bit_index\| {
		354	const bit = @as(std.math.Log2Int(usize), @intCast(bit_index));
		355	const mask = @as(usize, 1) << bit;
		356
		357	// toggling the bit should change the bit
		358	try testing.expect(x.load(.SeqCst) & mask == 0);
		359	try testing.expectEqual(@as(u1, 0), x.bitToggle(bit, .SeqCst));
		360	try testing.expect(x.load(.SeqCst) & mask != 0);
		361
		362	// toggling it again should change the bit
		363	try testing.expectEqual(@as(u1, 1), x.bitToggle(bit, .SeqCst));
		364	try testing.expect(x.load(.SeqCst) & mask == 0);
		365
		366	// all the previous bits should have not changed (still be toggled back)
		367	for (0..bit_index) \|prev_bit_index\| {
		368	const prev_bit = @as(std.math.Log2Int(usize), @intCast(prev_bit_index));
		369	const prev_mask = @as(usize, 1) << prev_bit;
		370	try testing.expect(x.load(.SeqCst) & prev_mask == 0);
		371	}
38	}	372	}
39	}	373	}
40		374
...	@@ -74,9 +408,8 @@ pub inline fn spinLoopHint() void {	...	@@ -74,9 +408,8 @@ pub inline fn spinLoopHint() void {
74	}	408	}
75	}	409	}
76		410
77	test "spinLoopHint" {	411	test spinLoopHint {
78	var i: usize = 10;	412	for (0..10) \|_\| {
79	while (i > 0) : (i -= 1) {
80	spinLoopHint();	413	spinLoopHint();
81	}	414	}
82	}	415	}
...	@@ -85,8 +418,8 @@ test "spinLoopHint" {	...	@@ -85,8 +418,8 @@ test "spinLoopHint" {
85	/// Add this much padding or align to this boundary to avoid atomically-updated	418	/// Add this much padding or align to this boundary to avoid atomically-updated
86	/// memory from forcing cache invalidations on near, but non-atomic, memory.	419	/// memory from forcing cache invalidations on near, but non-atomic, memory.
87	///	420	///
88	// https://en.wikipedia.org/wiki/False_sharing	421	/// https://en.wikipedia.org/wiki/False_sharing
89	// https://github.com/golang/go/search?q=CacheLinePadSize	422	/// https://github.com/golang/go/search?q=CacheLinePadSize
90	pub const cache_line = switch (builtin.cpu.arch) {	423	pub const cache_line = switch (builtin.cpu.arch) {
91	// x86_64: Starting from Intel's Sandy Bridge, the spatial prefetcher pulls in pairs of 64-byte cache lines at a time.	424	// x86_64: Starting from Intel's Sandy Bridge, the spatial prefetcher pulls in pairs of 64-byte cache lines at a time.
92	// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf	425	// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
...	@@ -118,3 +451,8 @@ pub const cache_line = switch (builtin.cpu.arch) {	...	@@ -118,3 +451,8 @@ pub const cache_line = switch (builtin.cpu.arch) {
118	// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7	451	// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7
119	else => 64,	452	else => 64,
120	};	453	};
		454
		455	const std = @import("std.zig");
		456	const builtin = @import("builtin");
		457	const AtomicOrder = std.builtin.AtomicOrder;
		458	const testing = std.testing;

lib/std/atomic/Atomic.zig deleted-619

...	@@ -1,619 +0,0 @@
1	const std = @import("../std.zig");
2	const builtin = @import("builtin");
3
4	const testing = std.testing;
5	const Ordering = std.atomic.Ordering;
6
7	pub fn Atomic(comptime T: type) type {
8	return extern struct {
9	value: T,
10
11	const Self = @This();
12
13	pub fn init(value: T) Self {
14	return .{ .value = value };
15	}
16
17	/// Perform an atomic fence which uses the atomic value as a hint for the modification order.
18	/// Use this when you want to imply a fence on an atomic variable without necessarily performing a memory access.
19	///
20	/// Example:
21	/// ```
22	/// const RefCount = struct {
23	/// count: Atomic(usize),
24	/// dropFn: const fn (RefCount) void,
25	///
26	/// fn ref(self: *RefCount) void {
27	/// _ = self.count.fetchAdd(1, .Monotonic); // no ordering necessary, just updating a counter
28	/// }
29	///
30	/// fn unref(self: *RefCount) void {
31	/// // Release ensures code before unref() happens-before the count is decremented as dropFn could be called by then.
32	/// if (self.count.fetchSub(1, .Release)) {
33	/// // Acquire ensures count decrement and code before previous unrefs()s happens-before we call dropFn below.
34	/// // NOTE: another alternative is to use .AcqRel on the fetchSub count decrement but it's extra barrier in possibly hot path.
35	/// self.count.fence(.Acquire);
36	/// (self.dropFn)(self);
37	/// }
38	/// }
39	/// };
40	/// ```
41	pub inline fn fence(self: *Self, comptime ordering: Ordering) void {
42	// LLVM's ThreadSanitizer doesn't support the normal fences so we specialize for it.
43	if (builtin.sanitize_thread) {
44	const tsan = struct {
45	extern "c" fn __tsan_acquire(addr: *anyopaque) void;
46	extern "c" fn __tsan_release(addr: *anyopaque) void;
47	};
48
49	const addr: *anyopaque = self;
50	return switch (ordering) {
51	.Unordered, .Monotonic => @compileError(@tagName(ordering) ++ " only applies to atomic loads and stores"),
52	.Acquire => tsan.__tsan_acquire(addr),
53	.Release => tsan.__tsan_release(addr),
54	.AcqRel, .SeqCst => {
55	tsan.__tsan_acquire(addr);
56	tsan.__tsan_release(addr);
57	},
58	};
59	}
60
61	return std.atomic.fence(ordering);
62	}
63
64	/// Non-atomically load from the atomic value without synchronization.
65	/// Care must be taken to avoid data-races when interacting with other atomic operations.
66	pub inline fn loadUnchecked(self: Self) T {
67	return self.value;
68	}
69
70	/// Non-atomically store to the atomic value without synchronization.
71	/// Care must be taken to avoid data-races when interacting with other atomic operations.
72	pub inline fn storeUnchecked(self: *Self, value: T) void {
73	self.value = value;
74	}
75
76	pub inline fn load(self: *const Self, comptime ordering: Ordering) T {
77	return switch (ordering) {
78	.AcqRel => @compileError(@tagName(ordering) ++ " implies " ++ @tagName(Ordering.Release) ++ " which is only allowed on atomic stores"),
79	.Release => @compileError(@tagName(ordering) ++ " is only allowed on atomic stores"),
80	else => @atomicLoad(T, &self.value, ordering),
81	};
82	}
83
84	pub inline fn store(self: *Self, value: T, comptime ordering: Ordering) void {
85	switch (ordering) {
86	.AcqRel => @compileError(@tagName(ordering) ++ " implies " ++ @tagName(Ordering.Acquire) ++ " which is only allowed on atomic loads"),
87	.Acquire => @compileError(@tagName(ordering) ++ " is only allowed on atomic loads"),
88	else => @atomicStore(T, &self.value, value, ordering),
89	}
90	}
91
92	pub inline fn swap(self: *Self, value: T, comptime ordering: Ordering) T {
93	return self.rmw(.Xchg, value, ordering);
94	}
95
96	pub inline fn compareAndSwap(
97	self: *Self,
98	compare: T,
99	exchange: T,
100	comptime success: Ordering,
101	comptime failure: Ordering,
102	) ?T {
103	return self.cmpxchg(true, compare, exchange, success, failure);
104	}
105
106	pub inline fn tryCompareAndSwap(
107	self: *Self,
108	compare: T,
109	exchange: T,
110	comptime success: Ordering,
111	comptime failure: Ordering,
112	) ?T {
113	return self.cmpxchg(false, compare, exchange, success, failure);
114	}
115
116	inline fn cmpxchg(
117	self: *Self,
118	comptime is_strong: bool,
119	compare: T,
120	exchange: T,
121	comptime success: Ordering,
122	comptime failure: Ordering,
123	) ?T {
124	if (success == .Unordered or failure == .Unordered) {
125	@compileError(@tagName(Ordering.Unordered) ++ " is only allowed on atomic loads and stores");
126	}
127
128	const success_is_stronger = switch (failure) {
129	.SeqCst => success == .SeqCst,
130	.AcqRel => @compileError(@tagName(failure) ++ " implies " ++ @tagName(Ordering.Release) ++ " which is only allowed on success"),
131	.Acquire => success == .SeqCst or success == .AcqRel or success == .Acquire,
132	.Release => @compileError(@tagName(failure) ++ " is only allowed on success"),
133	.Monotonic => true,
134	.Unordered => unreachable,
135	};
136
137	if (!success_is_stronger) {
138	@compileError(@tagName(success) ++ " must be stronger than " ++ @tagName(failure));
139	}
140
141	return switch (is_strong) {
142	true => @cmpxchgStrong(T, &self.value, compare, exchange, success, failure),
143	false => @cmpxchgWeak(T, &self.value, compare, exchange, success, failure),
144	};
145	}
146
147	inline fn rmw(
148	self: *Self,
149	comptime op: std.builtin.AtomicRmwOp,
150	value: T,
151	comptime ordering: Ordering,
152	) T {
153	return @atomicRmw(T, &self.value, op, value, ordering);
154	}
155
156	pub inline fn fetchAdd(self: *Self, value: T, comptime ordering: Ordering) T {
157	return self.rmw(.Add, value, ordering);
158	}
159
160	pub inline fn fetchSub(self: *Self, value: T, comptime ordering: Ordering) T {
161	return self.rmw(.Sub, value, ordering);
162	}
163
164	pub inline fn fetchMin(self: *Self, value: T, comptime ordering: Ordering) T {
165	return self.rmw(.Min, value, ordering);
166	}
167
168	pub inline fn fetchMax(self: *Self, value: T, comptime ordering: Ordering) T {
169	return self.rmw(.Max, value, ordering);
170	}
171
172	pub inline fn fetchAnd(self: *Self, value: T, comptime ordering: Ordering) T {
173	return self.rmw(.And, value, ordering);
174	}
175
176	pub inline fn fetchNand(self: *Self, value: T, comptime ordering: Ordering) T {
177	return self.rmw(.Nand, value, ordering);
178	}
179
180	pub inline fn fetchOr(self: *Self, value: T, comptime ordering: Ordering) T {
181	return self.rmw(.Or, value, ordering);
182	}
183
184	pub inline fn fetchXor(self: *Self, value: T, comptime ordering: Ordering) T {
185	return self.rmw(.Xor, value, ordering);
186	}
187
188	const Bit = std.math.Log2Int(T);
189	const BitRmwOp = enum {
190	Set,
191	Reset,
192	Toggle,
193	};
194
195	pub inline fn bitSet(self: *Self, bit: Bit, comptime ordering: Ordering) u1 {
196	return bitRmw(self, .Set, bit, ordering);
197	}
198
199	pub inline fn bitReset(self: *Self, bit: Bit, comptime ordering: Ordering) u1 {
200	return bitRmw(self, .Reset, bit, ordering);
201	}
202
203	pub inline fn bitToggle(self: *Self, bit: Bit, comptime ordering: Ordering) u1 {
204	return bitRmw(self, .Toggle, bit, ordering);
205	}
206
207	inline fn bitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: Ordering) u1 {
208	// x86 supports dedicated bitwise instructions
209	if (comptime builtin.target.cpu.arch.isX86() and @sizeOf(T) >= 2 and @sizeOf(T) <= 8) {
210	// TODO: this causes std lib test failures when enabled
211	if (false) {
212	return x86BitRmw(self, op, bit, ordering);
213	}
214	}
215
216	const mask = @as(T, 1) << bit;
217	const value = switch (op) {
218	.Set => self.fetchOr(mask, ordering),
219	.Reset => self.fetchAnd(~mask, ordering),
220	.Toggle => self.fetchXor(mask, ordering),
221	};
222
223	return @intFromBool(value & mask != 0);
224	}
225
226	inline fn x86BitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: Ordering) u1 {
227	const old_bit: u8 = switch (@sizeOf(T)) {
228	2 => switch (op) {
229	.Set => asm volatile ("lock btsw %[bit], %[ptr]"
230	// LLVM doesn't support u1 flag register return values
231	: [result] "={@ccc}" (-> u8),
232	: [ptr] "*m" (&self.value),
233	[bit] "X" (@as(T, bit)),
234	: "cc", "memory"
235	),
236	.Reset => asm volatile ("lock btrw %[bit], %[ptr]"
237	// LLVM doesn't support u1 flag register return values
238	: [result] "={@ccc}" (-> u8),
239	: [ptr] "*m" (&self.value),
240	[bit] "X" (@as(T, bit)),
241	: "cc", "memory"
242	),
243	.Toggle => asm volatile ("lock btcw %[bit], %[ptr]"
244	// LLVM doesn't support u1 flag register return values
245	: [result] "={@ccc}" (-> u8),
246	: [ptr] "*m" (&self.value),
247	[bit] "X" (@as(T, bit)),
248	: "cc", "memory"
249	),
250	},
251	4 => switch (op) {
252	.Set => asm volatile ("lock btsl %[bit], %[ptr]"
253	// LLVM doesn't support u1 flag register return values
254	: [result] "={@ccc}" (-> u8),
255	: [ptr] "*m" (&self.value),
256	[bit] "X" (@as(T, bit)),
257	: "cc", "memory"
258	),
259	.Reset => asm volatile ("lock btrl %[bit], %[ptr]"
260	// LLVM doesn't support u1 flag register return values
261	: [result] "={@ccc}" (-> u8),
262	: [ptr] "*m" (&self.value),
263	[bit] "X" (@as(T, bit)),
264	: "cc", "memory"
265	),
266	.Toggle => asm volatile ("lock btcl %[bit], %[ptr]"
267	// LLVM doesn't support u1 flag register return values
268	: [result] "={@ccc}" (-> u8),
269	: [ptr] "*m" (&self.value),
270	[bit] "X" (@as(T, bit)),
271	: "cc", "memory"
272	),
273	},
274	8 => switch (op) {
275	.Set => asm volatile ("lock btsq %[bit], %[ptr]"
276	// LLVM doesn't support u1 flag register return values
277	: [result] "={@ccc}" (-> u8),
278	: [ptr] "*m" (&self.value),
279	[bit] "X" (@as(T, bit)),
280	: "cc", "memory"
281	),
282	.Reset => asm volatile ("lock btrq %[bit], %[ptr]"
283	// LLVM doesn't support u1 flag register return values
284	: [result] "={@ccc}" (-> u8),
285	: [ptr] "*m" (&self.value),
286	[bit] "X" (@as(T, bit)),
287	: "cc", "memory"
288	),
289	.Toggle => asm volatile ("lock btcq %[bit], %[ptr]"
290	// LLVM doesn't support u1 flag register return values
291	: [result] "={@ccc}" (-> u8),
292	: [ptr] "*m" (&self.value),
293	[bit] "X" (@as(T, bit)),
294	: "cc", "memory"
295	),
296	},
297	else => @compileError("Invalid atomic type " ++ @typeName(T)),
298	};
299
300	// TODO: emit appropriate tsan fence if compiling with tsan
301	_ = ordering;
302
303	return @intCast(old_bit);
304	}
305	};
306	}
307
308	test "Atomic.fence" {
309	inline for (.{ .Acquire, .Release, .AcqRel, .SeqCst }) \|ordering\| {
310	var x = Atomic(usize).init(0);
311	x.fence(ordering);
312	}
313	}
314
315	fn atomicIntTypes() []const type {
316	comptime var bytes = 1;
317	comptime var types: []const type = &[_]type{};
318	inline while (bytes <= @sizeOf(usize)) : (bytes *= 2) {
319	types = types ++ &[_]type{std.meta.Int(.unsigned, bytes * 8)};
320	}
321	return types;
322	}
323
324	test "Atomic.loadUnchecked" {
325	inline for (atomicIntTypes()) \|Int\| {
326	var x = Atomic(Int).init(5);
327	try testing.expectEqual(x.loadUnchecked(), 5);
328	}
329	}
330
331	test "Atomic.storeUnchecked" {
332	inline for (atomicIntTypes()) \|Int\| {
333	_ = Int;
334	var x = Atomic(usize).init(5);
335	x.storeUnchecked(10);
336	try testing.expectEqual(x.loadUnchecked(), 10);
337	}
338	}
339
340	test "Atomic.load" {
341	inline for (atomicIntTypes()) \|Int\| {
342	inline for (.{ .Unordered, .Monotonic, .Acquire, .SeqCst }) \|ordering\| {
343	var x = Atomic(Int).init(5);
344	try testing.expectEqual(x.load(ordering), 5);
345	}
346	}
347	}
348
349	test "Atomic.store" {
350	inline for (atomicIntTypes()) \|Int\| {
351	inline for (.{ .Unordered, .Monotonic, .Release, .SeqCst }) \|ordering\| {
352	_ = Int;
353	var x = Atomic(usize).init(5);
354	x.store(10, ordering);
355	try testing.expectEqual(x.load(.SeqCst), 10);
356	}
357	}
358	}
359
360	const atomic_rmw_orderings = [_]Ordering{
361	.Monotonic,
362	.Acquire,
363	.Release,
364	.AcqRel,
365	.SeqCst,
366	};
367
368	test "Atomic.swap" {
369	inline for (atomic_rmw_orderings) \|ordering\| {
370	var x = Atomic(usize).init(5);
371	try testing.expectEqual(x.swap(10, ordering), 5);
372	try testing.expectEqual(x.load(.SeqCst), 10);
373
374	var y = Atomic(enum(usize) { a, b, c }).init(.c);
375	try testing.expectEqual(y.swap(.a, ordering), .c);
376	try testing.expectEqual(y.load(.SeqCst), .a);
377
378	var z = Atomic(f32).init(5.0);
379	try testing.expectEqual(z.swap(10.0, ordering), 5.0);
380	try testing.expectEqual(z.load(.SeqCst), 10.0);
381
382	var a = Atomic(bool).init(false);
383	try testing.expectEqual(a.swap(true, ordering), false);
384	try testing.expectEqual(a.load(.SeqCst), true);
385
386	var b = Atomic(?*u8).init(null);
387	try testing.expectEqual(b.swap(@as(?*u8, @ptrFromInt(@alignOf(u8))), ordering), null);
388	try testing.expectEqual(b.load(.SeqCst), @as(?*u8, @ptrFromInt(@alignOf(u8))));
389	}
390	}
391
392	const atomic_cmpxchg_orderings = [_][2]Ordering{
393	.{ .Monotonic, .Monotonic },
394	.{ .Acquire, .Monotonic },
395	.{ .Acquire, .Acquire },
396	.{ .Release, .Monotonic },
397	// Although accepted by LLVM, acquire failure implies AcqRel success
398	// .{ .Release, .Acquire },
399	.{ .AcqRel, .Monotonic },
400	.{ .AcqRel, .Acquire },
401	.{ .SeqCst, .Monotonic },
402	.{ .SeqCst, .Acquire },
403	.{ .SeqCst, .SeqCst },
404	};
405
406	test "Atomic.compareAndSwap" {
407	inline for (atomicIntTypes()) \|Int\| {
408	inline for (atomic_cmpxchg_orderings) \|ordering\| {
409	var x = Atomic(Int).init(0);
410	try testing.expectEqual(x.compareAndSwap(1, 0, ordering[0], ordering[1]), 0);
411	try testing.expectEqual(x.load(.SeqCst), 0);
412	try testing.expectEqual(x.compareAndSwap(0, 1, ordering[0], ordering[1]), null);
413	try testing.expectEqual(x.load(.SeqCst), 1);
414	try testing.expectEqual(x.compareAndSwap(1, 0, ordering[0], ordering[1]), null);
415	try testing.expectEqual(x.load(.SeqCst), 0);
416	}
417	}
418	}
419
420	test "Atomic.tryCompareAndSwap" {
421	inline for (atomicIntTypes()) \|Int\| {
422	inline for (atomic_cmpxchg_orderings) \|ordering\| {
423	var x = Atomic(Int).init(0);
424
425	try testing.expectEqual(x.tryCompareAndSwap(1, 0, ordering[0], ordering[1]), 0);
426	try testing.expectEqual(x.load(.SeqCst), 0);
427
428	while (x.tryCompareAndSwap(0, 1, ordering[0], ordering[1])) \|_\| {}
429	try testing.expectEqual(x.load(.SeqCst), 1);
430
431	while (x.tryCompareAndSwap(1, 0, ordering[0], ordering[1])) \|_\| {}
432	try testing.expectEqual(x.load(.SeqCst), 0);
433	}
434	}
435	}
436
437	test "Atomic.fetchAdd" {
438	inline for (atomicIntTypes()) \|Int\| {
439	inline for (atomic_rmw_orderings) \|ordering\| {
440	var x = Atomic(Int).init(5);
441	try testing.expectEqual(x.fetchAdd(5, ordering), 5);
442	try testing.expectEqual(x.load(.SeqCst), 10);
443	try testing.expectEqual(x.fetchAdd(std.math.maxInt(Int), ordering), 10);
444	try testing.expectEqual(x.load(.SeqCst), 9);
445	}
446	}
447	}
448
449	test "Atomic.fetchSub" {
450	inline for (atomicIntTypes()) \|Int\| {
451	inline for (atomic_rmw_orderings) \|ordering\| {
452	var x = Atomic(Int).init(5);
453	try testing.expectEqual(x.fetchSub(5, ordering), 5);
454	try testing.expectEqual(x.load(.SeqCst), 0);
455	try testing.expectEqual(x.fetchSub(1, ordering), 0);
456	try testing.expectEqual(x.load(.SeqCst), std.math.maxInt(Int));
457	}
458	}
459	}
460
461	test "Atomic.fetchMin" {
462	inline for (atomicIntTypes()) \|Int\| {
463	inline for (atomic_rmw_orderings) \|ordering\| {
464	var x = Atomic(Int).init(5);
465	try testing.expectEqual(x.fetchMin(0, ordering), 5);
466	try testing.expectEqual(x.load(.SeqCst), 0);
467	try testing.expectEqual(x.fetchMin(10, ordering), 0);
468	try testing.expectEqual(x.load(.SeqCst), 0);
469	}
470	}
471	}
472
473	test "Atomic.fetchMax" {
474	inline for (atomicIntTypes()) \|Int\| {
475	inline for (atomic_rmw_orderings) \|ordering\| {
476	var x = Atomic(Int).init(5);
477	try testing.expectEqual(x.fetchMax(10, ordering), 5);
478	try testing.expectEqual(x.load(.SeqCst), 10);
479	try testing.expectEqual(x.fetchMax(5, ordering), 10);
480	try testing.expectEqual(x.load(.SeqCst), 10);
481	}
482	}
483	}
484
485	test "Atomic.fetchAnd" {
486	inline for (atomicIntTypes()) \|Int\| {
487	inline for (atomic_rmw_orderings) \|ordering\| {
488	var x = Atomic(Int).init(0b11);
489	try testing.expectEqual(x.fetchAnd(0b10, ordering), 0b11);
490	try testing.expectEqual(x.load(.SeqCst), 0b10);
491	try testing.expectEqual(x.fetchAnd(0b00, ordering), 0b10);
492	try testing.expectEqual(x.load(.SeqCst), 0b00);
493	}
494	}
495	}
496
497	test "Atomic.fetchNand" {
498	inline for (atomicIntTypes()) \|Int\| {
499	inline for (atomic_rmw_orderings) \|ordering\| {
500	var x = Atomic(Int).init(0b11);
501	try testing.expectEqual(x.fetchNand(0b10, ordering), 0b11);
502	try testing.expectEqual(x.load(.SeqCst), ~@as(Int, 0b10));
503	try testing.expectEqual(x.fetchNand(0b00, ordering), ~@as(Int, 0b10));
504	try testing.expectEqual(x.load(.SeqCst), ~@as(Int, 0b00));
505	}
506	}
507	}
508
509	test "Atomic.fetchOr" {
510	inline for (atomicIntTypes()) \|Int\| {
511	inline for (atomic_rmw_orderings) \|ordering\| {
512	var x = Atomic(Int).init(0b11);
513	try testing.expectEqual(x.fetchOr(0b100, ordering), 0b11);
514	try testing.expectEqual(x.load(.SeqCst), 0b111);
515	try testing.expectEqual(x.fetchOr(0b010, ordering), 0b111);
516	try testing.expectEqual(x.load(.SeqCst), 0b111);
517	}
518	}
519	}
520
521	test "Atomic.fetchXor" {
522	inline for (atomicIntTypes()) \|Int\| {
523	inline for (atomic_rmw_orderings) \|ordering\| {
524	var x = Atomic(Int).init(0b11);
525	try testing.expectEqual(x.fetchXor(0b10, ordering), 0b11);
526	try testing.expectEqual(x.load(.SeqCst), 0b01);
527	try testing.expectEqual(x.fetchXor(0b01, ordering), 0b01);
528	try testing.expectEqual(x.load(.SeqCst), 0b00);
529	}
530	}
531	}
532
533	test "Atomic.bitSet" {
534	inline for (atomicIntTypes()) \|Int\| {
535	inline for (atomic_rmw_orderings) \|ordering\| {
536	var x = Atomic(Int).init(0);
537
538	for (0..@bitSizeOf(Int)) \|bit_index\| {
539	const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
540	const mask = @as(Int, 1) << bit;
541
542	// setting the bit should change the bit
543	try testing.expect(x.load(.SeqCst) & mask == 0);
544	try testing.expectEqual(x.bitSet(bit, ordering), 0);
545	try testing.expect(x.load(.SeqCst) & mask != 0);
546
547	// setting it again shouldn't change the bit
548	try testing.expectEqual(x.bitSet(bit, ordering), 1);
549	try testing.expect(x.load(.SeqCst) & mask != 0);
550
551	// all the previous bits should have not changed (still be set)
552	for (0..bit_index) \|prev_bit_index\| {
553	const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
554	const prev_mask = @as(Int, 1) << prev_bit;
555	try testing.expect(x.load(.SeqCst) & prev_mask != 0);
556	}
557	}
558	}
559	}
560	}
561
562	test "Atomic.bitReset" {
563	inline for (atomicIntTypes()) \|Int\| {
564	inline for (atomic_rmw_orderings) \|ordering\| {
565	var x = Atomic(Int).init(0);
566
567	for (0..@bitSizeOf(Int)) \|bit_index\| {
568	const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
569	const mask = @as(Int, 1) << bit;
570	x.storeUnchecked(x.loadUnchecked() \| mask);
571
572	// unsetting the bit should change the bit
573	try testing.expect(x.load(.SeqCst) & mask != 0);
574	try testing.expectEqual(x.bitReset(bit, ordering), 1);
575	try testing.expect(x.load(.SeqCst) & mask == 0);
576
577	// unsetting it again shouldn't change the bit
578	try testing.expectEqual(x.bitReset(bit, ordering), 0);
579	try testing.expect(x.load(.SeqCst) & mask == 0);
580
581	// all the previous bits should have not changed (still be reset)
582	for (0..bit_index) \|prev_bit_index\| {
583	const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
584	const prev_mask = @as(Int, 1) << prev_bit;
585	try testing.expect(x.load(.SeqCst) & prev_mask == 0);
586	}
587	}
588	}
589	}
590	}
591
592	test "Atomic.bitToggle" {
593	inline for (atomicIntTypes()) \|Int\| {
594	inline for (atomic_rmw_orderings) \|ordering\| {
595	var x = Atomic(Int).init(0);
596
597	for (0..@bitSizeOf(Int)) \|bit_index\| {
598	const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
599	const mask = @as(Int, 1) << bit;
600
601	// toggling the bit should change the bit
602	try testing.expect(x.load(.SeqCst) & mask == 0);
603	try testing.expectEqual(x.bitToggle(bit, ordering), 0);
604	try testing.expect(x.load(.SeqCst) & mask != 0);
605
606	// toggling it again should change the bit
607	try testing.expectEqual(x.bitToggle(bit, ordering), 1);
608	try testing.expect(x.load(.SeqCst) & mask == 0);
609
610	// all the previous bits should have not changed (still be toggled back)
611	for (0..bit_index) \|prev_bit_index\| {
612	const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
613	const prev_mask = @as(Int, 1) << prev_bit;
614	try testing.expect(x.load(.SeqCst) & prev_mask == 0);
615	}
616	}
617	}
618	}
619	}

lib/std/atomic/queue.zig deleted-413

...	@@ -1,413 +0,0 @@
1	const std = @import("../std.zig");
2	const builtin = @import("builtin");
3	const assert = std.debug.assert;
4	const expect = std.testing.expect;
5
6	/// Many producer, many consumer, non-allocating, thread-safe.
7	/// Uses a mutex to protect access.
8	/// The queue does not manage ownership and the user is responsible to
9	/// manage the storage of the nodes.
10	pub fn Queue(comptime T: type) type {
11	return struct {
12	head: ?*Node,
13	tail: ?*Node,
14	mutex: std.Thread.Mutex,
15
16	pub const Self = @This();
17	pub const Node = std.DoublyLinkedList(T).Node;
18
19	/// Initializes a new queue. The queue does not provide a `deinit()`
20	/// function, so the user must take care of cleaning up the queue elements.
21	pub fn init() Self {
22	return Self{
23	.head = null,
24	.tail = null,
25	.mutex = std.Thread.Mutex{},
26	};
27	}
28
29	/// Appends `node` to the queue.
30	/// The lifetime of `node` must be longer than the lifetime of the queue.
31	pub fn put(self: Self, node: Node) void {
32	node.next = null;
33
34	self.mutex.lock();
35	defer self.mutex.unlock();
36
37	node.prev = self.tail;
38	self.tail = node;
39	if (node.prev) \|prev_tail\| {
40	prev_tail.next = node;
41	} else {
42	assert(self.head == null);
43	self.head = node;
44	}
45	}
46
47	/// Gets a previously inserted node or returns `null` if there is none.
48	/// It is safe to `get()` a node from the queue while another thread tries
49	/// to `remove()` the same node at the same time.
50	pub fn get(self: Self) ?Node {
51	self.mutex.lock();
52	defer self.mutex.unlock();
53
54	const head = self.head orelse return null;
55	self.head = head.next;
56	if (head.next) \|new_head\| {
57	new_head.prev = null;
58	} else {
59	self.tail = null;
60	}
61	// This way, a get() and a remove() are thread-safe with each other.
62	head.prev = null;
63	head.next = null;
64	return head;
65	}
66
67	/// Prepends `node` to the front of the queue.
68	/// The lifetime of `node` must be longer than the lifetime of the queue.
69	pub fn unget(self: Self, node: Node) void {
70	node.prev = null;
71
72	self.mutex.lock();
73	defer self.mutex.unlock();
74
75	const opt_head = self.head;
76	self.head = node;
77	if (opt_head) \|old_head\| {
78	node.next = old_head;
79	} else {
80	assert(self.tail == null);
81	self.tail = node;
82	}
83	}
84
85	/// Removes a node from the queue, returns whether node was actually removed.
86	/// It is safe to `remove()` a node from the queue while another thread tries
87	/// to `get()` the same node at the same time.
88	pub fn remove(self: Self, node: Node) bool {
89	self.mutex.lock();
90	defer self.mutex.unlock();
91
92	if (node.prev == null and node.next == null and self.head != node) {
93	return false;
94	}
95
96	if (node.prev) \|prev\| {
97	prev.next = node.next;
98	} else {
99	self.head = node.next;
100	}
101	if (node.next) \|next\| {
102	next.prev = node.prev;
103	} else {
104	self.tail = node.prev;
105	}
106	node.prev = null;
107	node.next = null;
108	return true;
109	}
110
111	/// Returns `true` if the queue is currently empty.
112	/// Note that in a multi-consumer environment a return value of `false`
113	/// does not mean that `get` will yield a non-`null` value!
114	pub fn isEmpty(self: *Self) bool {
115	self.mutex.lock();
116	defer self.mutex.unlock();
117	return self.head == null;
118	}
119
120	/// Dumps the contents of the queue to `stderr`.
121	pub fn dump(self: *Self) void {
122	self.dumpToStream(std.io.getStdErr().writer()) catch return;
123	}
124
125	/// Dumps the contents of the queue to `stream`.
126	/// Up to 4 elements from the head are dumped and the tail of the queue is
127	/// dumped as well.
128	pub fn dumpToStream(self: *Self, stream: anytype) !void {
129	const S = struct {
130	fn dumpRecursive(
131	s: anytype,
132	optional_node: ?*Node,
133	indent: usize,
134	comptime depth: comptime_int,
135	) !void {
136	try s.writeByteNTimes(' ', indent);
137	if (optional_node) \|node\| {
138	try s.print("0x{x}={}\n", .{ @intFromPtr(node), node.data });
139	if (depth == 0) {
140	try s.print("(max depth)\n", .{});
141	return;
142	}
143	try dumpRecursive(s, node.next, indent + 1, depth - 1);
144	} else {
145	try s.print("(null)\n", .{});
146	}
147	}
148	};
149	self.mutex.lock();
150	defer self.mutex.unlock();
151
152	try stream.print("head: ", .{});
153	try S.dumpRecursive(stream, self.head, 0, 4);
154	try stream.print("tail: ", .{});
155	try S.dumpRecursive(stream, self.tail, 0, 4);
156	}
157	};
158	}
159
160	const Context = struct {
161	allocator: std.mem.Allocator,
162	queue: *Queue(i32),
163	put_sum: isize,
164	get_sum: isize,
165	get_count: usize,
166	puts_done: bool,
167	};
168
169	// TODO add lazy evaluated build options and then put puts_per_thread behind
170	// some option such as: "AggressiveMultithreadedFuzzTest". In the AppVeyor
171	// CI we would use a less aggressive setting since at 1 core, while we still
172	// want this test to pass, we need a smaller value since there is so much thrashing
173	// we would also use a less aggressive setting when running in valgrind
174	const puts_per_thread = 500;
175	const put_thread_count = 3;
176
177	test "std.atomic.Queue" {
178	const plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);
179	defer std.heap.page_allocator.free(plenty_of_memory);
180
181	var fixed_buffer_allocator = std.heap.FixedBufferAllocator.init(plenty_of_memory);
182	const a = fixed_buffer_allocator.threadSafeAllocator();
183
184	var queue = Queue(i32).init();
185	var context = Context{
186	.allocator = a,
187	.queue = &queue,
188	.put_sum = 0,
189	.get_sum = 0,
190	.puts_done = false,
191	.get_count = 0,
192	};
193
194	if (builtin.single_threaded) {
195	try expect(context.queue.isEmpty());
196	{
197	var i: usize = 0;
198	while (i < put_thread_count) : (i += 1) {
199	try expect(startPuts(&context) == 0);
200	}
201	}
202	try expect(!context.queue.isEmpty());
203	context.puts_done = true;
204	{
205	var i: usize = 0;
206	while (i < put_thread_count) : (i += 1) {
207	try expect(startGets(&context) == 0);
208	}
209	}
210	try expect(context.queue.isEmpty());
211	} else {
212	try expect(context.queue.isEmpty());
213
214	var putters: [put_thread_count]std.Thread = undefined;
215	for (&putters) \|*t\| {
216	t.* = try std.Thread.spawn(.{}, startPuts, .{&context});
217	}
218	var getters: [put_thread_count]std.Thread = undefined;
219	for (&getters) \|*t\| {
220	t.* = try std.Thread.spawn(.{}, startGets, .{&context});
221	}
222
223	for (putters) \|t\|
224	t.join();
225	@atomicStore(bool, &context.puts_done, true, .SeqCst);
226	for (getters) \|t\|
227	t.join();
228
229	try expect(context.queue.isEmpty());
230	}
231
232	if (context.put_sum != context.get_sum) {
233	std.debug.panic("failure\nput_sum:{} != get_sum:{}", .{ context.put_sum, context.get_sum });
234	}
235
236	if (context.get_count != puts_per_thread * put_thread_count) {
237	std.debug.panic("failure\nget_count:{} != puts_per_thread:{} * put_thread_count:{}", .{
238	context.get_count,
239	@as(u32, puts_per_thread),
240	@as(u32, put_thread_count),
241	});
242	}
243	}
244
245	fn startPuts(ctx: *Context) u8 {
246	var put_count: usize = puts_per_thread;
247	var prng = std.rand.DefaultPrng.init(0xdeadbeef);
248	const random = prng.random();
249	while (put_count != 0) : (put_count -= 1) {
250	std.time.sleep(1); // let the os scheduler be our fuzz
251	const x = @as(i32, @bitCast(random.int(u32)));
252	const node = ctx.allocator.create(Queue(i32).Node) catch unreachable;
253	node.* = .{
254	.prev = undefined,
255	.next = undefined,
256	.data = x,
257	};
258	ctx.queue.put(node);
259	_ = @atomicRmw(isize, &ctx.put_sum, .Add, x, .SeqCst);
260	}
261	return 0;
262	}
263
264	fn startGets(ctx: *Context) u8 {
265	while (true) {
266	const last = @atomicLoad(bool, &ctx.puts_done, .SeqCst);
267
268	while (ctx.queue.get()) \|node\| {
269	std.time.sleep(1); // let the os scheduler be our fuzz
270	_ = @atomicRmw(isize, &ctx.get_sum, .Add, node.data, .SeqCst);
271	_ = @atomicRmw(usize, &ctx.get_count, .Add, 1, .SeqCst);
272	}
273
274	if (last) return 0;
275	}
276	}
277
278	test "std.atomic.Queue single-threaded" {
279	var queue = Queue(i32).init();
280	try expect(queue.isEmpty());
281
282	var node_0 = Queue(i32).Node{
283	.data = 0,
284	.next = undefined,
285	.prev = undefined,
286	};
287	queue.put(&node_0);
288	try expect(!queue.isEmpty());
289
290	var node_1 = Queue(i32).Node{
291	.data = 1,
292	.next = undefined,
293	.prev = undefined,
294	};
295	queue.put(&node_1);
296	try expect(!queue.isEmpty());
297
298	try expect(queue.get().?.data == 0);
299	try expect(!queue.isEmpty());
300
301	var node_2 = Queue(i32).Node{
302	.data = 2,
303	.next = undefined,
304	.prev = undefined,
305	};
306	queue.put(&node_2);
307	try expect(!queue.isEmpty());
308
309	var node_3 = Queue(i32).Node{
310	.data = 3,
311	.next = undefined,
312	.prev = undefined,
313	};
314	queue.put(&node_3);
315	try expect(!queue.isEmpty());
316
317	try expect(queue.get().?.data == 1);
318	try expect(!queue.isEmpty());
319
320	try expect(queue.get().?.data == 2);
321	try expect(!queue.isEmpty());
322
323	var node_4 = Queue(i32).Node{
324	.data = 4,
325	.next = undefined,
326	.prev = undefined,
327	};
328	queue.put(&node_4);
329	try expect(!queue.isEmpty());
330
331	try expect(queue.get().?.data == 3);
332	node_3.next = null;
333	try expect(!queue.isEmpty());
334
335	queue.unget(&node_3);
336	try expect(queue.get().?.data == 3);
337	try expect(!queue.isEmpty());
338
339	try expect(queue.get().?.data == 4);
340	try expect(queue.isEmpty());
341
342	try expect(queue.get() == null);
343	try expect(queue.isEmpty());
344
345	// unget an empty queue
346	queue.unget(&node_4);
347	try expect(queue.tail == &node_4);
348	try expect(queue.head == &node_4);
349
350	try expect(queue.get().?.data == 4);
351
352	try expect(queue.get() == null);
353	try expect(queue.isEmpty());
354	}
355
356	test "std.atomic.Queue dump" {
357	const mem = std.mem;
358	var buffer: [1024]u8 = undefined;
359	var expected_buffer: [1024]u8 = undefined;
360	var fbs = std.io.fixedBufferStream(&buffer);
361
362	var queue = Queue(i32).init();
363
364	// Test empty stream
365	fbs.reset();
366	try queue.dumpToStream(fbs.writer());
367	try expect(mem.eql(u8, buffer[0..fbs.pos],
368	\\head: (null)
369	\\tail: (null)
370	\\
371	));
372
373	// Test a stream with one element
374	var node_0 = Queue(i32).Node{
375	.data = 1,
376	.next = undefined,
377	.prev = undefined,
378	};
379	queue.put(&node_0);
380
381	fbs.reset();
382	try queue.dumpToStream(fbs.writer());
383
384	var expected = try std.fmt.bufPrint(expected_buffer[0..],
385	\\head: 0x{x}=1
386	\\ (null)
387	\\tail: 0x{x}=1
388	\\ (null)
389	\\
390	, .{ @intFromPtr(queue.head), @intFromPtr(queue.tail) });
391	try expect(mem.eql(u8, buffer[0..fbs.pos], expected));
392
393	// Test a stream with two elements
394	var node_1 = Queue(i32).Node{
395	.data = 2,
396	.next = undefined,
397	.prev = undefined,
398	};
399	queue.put(&node_1);
400
401	fbs.reset();
402	try queue.dumpToStream(fbs.writer());
403
404	expected = try std.fmt.bufPrint(expected_buffer[0..],
405	\\head: 0x{x}=1
406	\\ 0x{x}=2
407	\\ (null)
408	\\tail: 0x{x}=2
409	\\ (null)
410	\\
411	, .{ @intFromPtr(queue.head), @intFromPtr(queue.head.?.next), @intFromPtr(queue.tail) });
412	try expect(mem.eql(u8, buffer[0..fbs.pos], expected));
413	}

lib/std/atomic/stack.zig deleted-178

...	@@ -1,178 +0,0 @@
1	const std = @import("../std.zig");
2	const builtin = @import("builtin");
3	const assert = std.debug.assert;
4	const expect = std.testing.expect;
5
6	/// Many reader, many writer, non-allocating, thread-safe.
7	/// Uses a spinlock to protect `push()` and `pop()`.
8	/// When building in single threaded mode, this is a simple linked list.
9	pub fn Stack(comptime T: type) type {
10	return struct {
11	root: ?*Node,
12	lock: @TypeOf(lock_init),
13
14	const lock_init = if (builtin.single_threaded) {} else false;
15
16	pub const Self = @This();
17
18	pub const Node = struct {
19	next: ?*Node,
20	data: T,
21	};
22
23	pub fn init() Self {
24	return Self{
25	.root = null,
26	.lock = lock_init,
27	};
28	}
29
30	/// push operation, but only if you are the first item in the stack. if you did not succeed in
31	/// being the first item in the stack, returns the other item that was there.
32	pub fn pushFirst(self: Self, node: Node) ?*Node {
33	node.next = null;
34	return @cmpxchgStrong(?*Node, &self.root, null, node, .SeqCst, .SeqCst);
35	}
36
37	pub fn push(self: Self, node: Node) void {
38	if (builtin.single_threaded) {
39	node.next = self.root;
40	self.root = node;
41	} else {
42	while (@atomicRmw(bool, &self.lock, .Xchg, true, .SeqCst)) {}
43	defer assert(@atomicRmw(bool, &self.lock, .Xchg, false, .SeqCst));
44
45	node.next = self.root;
46	self.root = node;
47	}
48	}
49
50	pub fn pop(self: Self) ?Node {
51	if (builtin.single_threaded) {
52	const root = self.root orelse return null;
53	self.root = root.next;
54	return root;
55	} else {
56	while (@atomicRmw(bool, &self.lock, .Xchg, true, .SeqCst)) {}
57	defer assert(@atomicRmw(bool, &self.lock, .Xchg, false, .SeqCst));
58
59	const root = self.root orelse return null;
60	self.root = root.next;
61	return root;
62	}
63	}
64
65	pub fn isEmpty(self: *Self) bool {
66	return @atomicLoad(?*Node, &self.root, .SeqCst) == null;
67	}
68	};
69	}
70
71	const Context = struct {
72	allocator: std.mem.Allocator,
73	stack: *Stack(i32),
74	put_sum: isize,
75	get_sum: isize,
76	get_count: usize,
77	puts_done: bool,
78	};
79	// TODO add lazy evaluated build options and then put puts_per_thread behind
80	// some option such as: "AggressiveMultithreadedFuzzTest". In the AppVeyor
81	// CI we would use a less aggressive setting since at 1 core, while we still
82	// want this test to pass, we need a smaller value since there is so much thrashing
83	// we would also use a less aggressive setting when running in valgrind
84	const puts_per_thread = 500;
85	const put_thread_count = 3;
86
87	test "std.atomic.stack" {
88	const plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);
89	defer std.heap.page_allocator.free(plenty_of_memory);
90
91	var fixed_buffer_allocator = std.heap.FixedBufferAllocator.init(plenty_of_memory);
92	const a = fixed_buffer_allocator.threadSafeAllocator();
93
94	var stack = Stack(i32).init();
95	var context = Context{
96	.allocator = a,
97	.stack = &stack,
98	.put_sum = 0,
99	.get_sum = 0,
100	.puts_done = false,
101	.get_count = 0,
102	};
103
104	if (builtin.single_threaded) {
105	{
106	var i: usize = 0;
107	while (i < put_thread_count) : (i += 1) {
108	try expect(startPuts(&context) == 0);
109	}
110	}
111	context.puts_done = true;
112	{
113	var i: usize = 0;
114	while (i < put_thread_count) : (i += 1) {
115	try expect(startGets(&context) == 0);
116	}
117	}
118	} else {
119	var putters: [put_thread_count]std.Thread = undefined;
120	for (&putters) \|*t\| {
121	t.* = try std.Thread.spawn(.{}, startPuts, .{&context});
122	}
123	var getters: [put_thread_count]std.Thread = undefined;
124	for (&getters) \|*t\| {
125	t.* = try std.Thread.spawn(.{}, startGets, .{&context});
126	}
127
128	for (putters) \|t\|
129	t.join();
130	@atomicStore(bool, &context.puts_done, true, .SeqCst);
131	for (getters) \|t\|
132	t.join();
133	}
134
135	if (context.put_sum != context.get_sum) {
136	std.debug.panic("failure\nput_sum:{} != get_sum:{}", .{ context.put_sum, context.get_sum });
137	}
138
139	if (context.get_count != puts_per_thread * put_thread_count) {
140	std.debug.panic("failure\nget_count:{} != puts_per_thread:{} * put_thread_count:{}", .{
141	context.get_count,
142	@as(u32, puts_per_thread),
143	@as(u32, put_thread_count),
144	});
145	}
146	}
147
148	fn startPuts(ctx: *Context) u8 {
149	var put_count: usize = puts_per_thread;
150	var prng = std.rand.DefaultPrng.init(0xdeadbeef);
151	const random = prng.random();
152	while (put_count != 0) : (put_count -= 1) {
153	std.time.sleep(1); // let the os scheduler be our fuzz
154	const x = @as(i32, @bitCast(random.int(u32)));
155	const node = ctx.allocator.create(Stack(i32).Node) catch unreachable;
156	node.* = Stack(i32).Node{
157	.next = undefined,
158	.data = x,
159	};
160	ctx.stack.push(node);
161	_ = @atomicRmw(isize, &ctx.put_sum, .Add, x, .SeqCst);
162	}
163	return 0;
164	}
165
166	fn startGets(ctx: *Context) u8 {
167	while (true) {
168	const last = @atomicLoad(bool, &ctx.puts_done, .SeqCst);
169
170	while (ctx.stack.pop()) \|node\| {
171	std.time.sleep(1); // let the os scheduler be our fuzz
172	_ = @atomicRmw(isize, &ctx.get_sum, .Add, node.data, .SeqCst);
173	_ = @atomicRmw(usize, &ctx.get_count, .Add, 1, .SeqCst);
174	}
175
176	if (last) return 0;
177	}
178	}

lib/std/child_process.zig+1-1

...	@@ -1286,7 +1286,7 @@ fn windowsMakePipeIn(rd: ?windows.HANDLE, wr: ?windows.HANDLE, sattr: *const w	...	@@ -1286,7 +1286,7 @@ fn windowsMakePipeIn(rd: ?windows.HANDLE, wr: ?windows.HANDLE, sattr: *const w
1286	wr.* = wr_h;	1286	wr.* = wr_h;
1287	}	1287	}
1288		1288
1289	var pipe_name_counter = std.atomic.Atomic(u32).init(1);	1289	var pipe_name_counter = std.atomic.Value(u32).init(1);
1290		1290
1291	fn windowsMakeAsyncPipe(rd: ?windows.HANDLE, wr: ?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {	1291	fn windowsMakeAsyncPipe(rd: ?windows.HANDLE, wr: ?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {
1292	var tmp_bufw: [128]u16 = undefined;	1292	var tmp_bufw: [128]u16 = undefined;

lib/std/debug.zig+2-2

...	@@ -375,7 +375,7 @@ pub fn panicExtra(	...	@@ -375,7 +375,7 @@ pub fn panicExtra(
375		375
376	/// Non-zero whenever the program triggered a panic.	376	/// Non-zero whenever the program triggered a panic.
377	/// The counter is incremented/decremented atomically.	377	/// The counter is incremented/decremented atomically.
378	var panicking = std.atomic.Atomic(u8).init(0);	378	var panicking = std.atomic.Value(u8).init(0);
379		379
380	// Locked to avoid interleaving panic messages from multiple threads.	380	// Locked to avoid interleaving panic messages from multiple threads.
381	var panic_mutex = std.Thread.Mutex{};	381	var panic_mutex = std.Thread.Mutex{};
...	@@ -448,7 +448,7 @@ fn waitForOtherThreadToFinishPanicking() void {	...	@@ -448,7 +448,7 @@ fn waitForOtherThreadToFinishPanicking() void {
448	if (builtin.single_threaded) unreachable;	448	if (builtin.single_threaded) unreachable;
449		449
450	// Sleep forever without hammering the CPU	450	// Sleep forever without hammering the CPU
451	var futex = std.atomic.Atomic(u32).init(0);	451	var futex = std.atomic.Value(u32).init(0);
452	while (true) std.Thread.Futex.wait(&futex, 0);	452	while (true) std.Thread.Futex.wait(&futex, 0);
453	unreachable;	453	unreachable;
454	}	454	}

lib/std/event/loop.zig+2-3

...	@@ -7,7 +7,6 @@ const os = std.os;	...	@@ -7,7 +7,6 @@ const os = std.os;
7	const windows = os.windows;	7	const windows = os.windows;
8	const maxInt = std.math.maxInt;	8	const maxInt = std.math.maxInt;
9	const Thread = std.Thread;	9	const Thread = std.Thread;
10	const Atomic = std.atomic.Atomic;
11		10
12	const is_windows = builtin.os.tag == .windows;	11	const is_windows = builtin.os.tag == .windows;
13		12
...	@@ -854,7 +853,7 @@ pub const Loop = struct {	...	@@ -854,7 +853,7 @@ pub const Loop = struct {
854	waiters: Waiters,	853	waiters: Waiters,
855	thread: std.Thread,	854	thread: std.Thread,
856	event: std.Thread.ResetEvent,	855	event: std.Thread.ResetEvent,
857	is_running: Atomic(bool),	856	is_running: std.atomic.Value(bool),
858		857
859	/// Initialize the delay queue by spawning the timer thread	858	/// Initialize the delay queue by spawning the timer thread
860	/// and starting any timer resources.	859	/// and starting any timer resources.
...	@@ -866,7 +865,7 @@ pub const Loop = struct {	...	@@ -866,7 +865,7 @@ pub const Loop = struct {
866	},	865	},
867	.thread = undefined,	866	.thread = undefined,
868	.event = .{},	867	.event = .{},
869	.is_running = Atomic(bool).init(true),	868	.is_running = std.atomic.Value(bool).init(true),
870	};	869	};
871		870
872	// Must be after init so that it can read the other state, such as `is_running`.	871	// Must be after init so that it can read the other state, such as `is_running`.

lib/std/os.zig+1-1

...	@@ -6461,7 +6461,7 @@ pub const CopyFileRangeError = error{	...	@@ -6461,7 +6461,7 @@ pub const CopyFileRangeError = error{
6461	CorruptedData,	6461	CorruptedData,
6462	} \|\| PReadError \|\| PWriteError \|\| UnexpectedError;	6462	} \|\| PReadError \|\| PWriteError \|\| UnexpectedError;
6463		6463
6464	var has_copy_file_range_syscall = std.atomic.Atomic(bool).init(true);	6464	var has_copy_file_range_syscall = std.atomic.Value(bool).init(true);
6465		6465
6466	/// Transfer data between file descriptors at specified offsets.	6466	/// Transfer data between file descriptors at specified offsets.
6467	/// Returns the number of bytes written, which can less than requested.	6467	/// Returns the number of bytes written, which can less than requested.

src/crash_report.zig+2-2

...	@@ -322,7 +322,7 @@ const PanicSwitch = struct {	...	@@ -322,7 +322,7 @@ const PanicSwitch = struct {
322	/// Updated atomically before taking the panic_mutex.	322	/// Updated atomically before taking the panic_mutex.
323	/// In recoverable cases, the program will not abort	323	/// In recoverable cases, the program will not abort
324	/// until all panicking threads have dumped their traces.	324	/// until all panicking threads have dumped their traces.
325	var panicking = std.atomic.Atomic(u8).init(0);	325	var panicking = std.atomic.Value(u8).init(0);
326		326
327	// Locked to avoid interleaving panic messages from multiple threads.	327	// Locked to avoid interleaving panic messages from multiple threads.
328	var panic_mutex = std.Thread.Mutex{};	328	var panic_mutex = std.Thread.Mutex{};
...	@@ -477,7 +477,7 @@ const PanicSwitch = struct {	...	@@ -477,7 +477,7 @@ const PanicSwitch = struct {
477	// and call abort()	477	// and call abort()
478		478
479	// Sleep forever without hammering the CPU	479	// Sleep forever without hammering the CPU
480	var futex = std.atomic.Atomic(u32).init(0);	480	var futex = std.atomic.Value(u32).init(0);
481	while (true) std.Thread.Futex.wait(&futex, 0);	481	while (true) std.Thread.Futex.wait(&futex, 0);
482		482
483	// This should be unreachable, recurse into recoverAbort.	483	// This should be unreachable, recurse into recoverAbort.

author	Andrew Kelley <andrew@ziglang.org> 2023-11-23 04:55:28-05:00
committer	GitHub <noreply@github.com> 2023-11-23 04:55:28-05:00
log	2bffd810157a8e7c0b2500a13921dd8c45694b8a
tree	2f8cfbc0403c5bd492ca68f1aa7ba6b02886ef2c
parent	115ec25f2e4eed5033f34eaee8bf3477ff417ecc
parent	70931dbdea96d92feb60406c827e39e566317863
signature	Signed by PGP key 4AEE18F83AFDEB23

...	@@ -50,7 +50,6 @@ const Mutex = std.Thread.Mutex;	...	@@ -50,7 +50,6 @@ const Mutex = std.Thread.Mutex;
50	const os = std.os;	50	const os = std.os;
51	const assert = std.debug.assert;	51	const assert = std.debug.assert;
52	const testing = std.testing;	52	const testing = std.testing;
53	const Atomic = std.atomic.Atomic;
54	const Futex = std.Thread.Futex;	53	const Futex = std.Thread.Futex;
55		54
56	impl: Impl = .{},	55	impl: Impl = .{},
...	@@ -193,8 +192,8 @@ const WindowsImpl = struct {	...	@@ -193,8 +192,8 @@ const WindowsImpl = struct {
193	};	192	};
194		193
195	const FutexImpl = struct {	194	const FutexImpl = struct {
196	state: Atomic(u32) = Atomic(u32).init(0),	195	state: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
197	epoch: Atomic(u32) = Atomic(u32).init(0),	196	epoch: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
198		197
199	const one_waiter = 1;	198	const one_waiter = 1;
200	const waiter_mask = 0xffff;	199	const waiter_mask = 0xffff;
...	@@ -232,12 +231,12 @@ const FutexImpl = struct {	...	@@ -232,12 +231,12 @@ const FutexImpl = struct {
232	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.	231	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
233	while (state & signal_mask != 0) {	232	while (state & signal_mask != 0) {
234	const new_state = state - one_waiter - one_signal;	233	const new_state = state - one_waiter - one_signal;
235	state = self.state.tryCompareAndSwap(state, new_state, .Acquire, .Monotonic) orelse return;	234	state = self.state.cmpxchgWeak(state, new_state, .Acquire, .Monotonic) orelse return;
236	}	235	}
237		236
238	// Remove the waiter we added and officially return timed out.	237	// Remove the waiter we added and officially return timed out.
239	const new_state = state - one_waiter;	238	const new_state = state - one_waiter;
240	state = self.state.tryCompareAndSwap(state, new_state, .Monotonic, .Monotonic) orelse return err;	239	state = self.state.cmpxchgWeak(state, new_state, .Monotonic, .Monotonic) orelse return err;
241	}	240	}
242	},	241	},
243	};	242	};
...	@@ -249,7 +248,7 @@ const FutexImpl = struct {	...	@@ -249,7 +248,7 @@ const FutexImpl = struct {
249	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.	248	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
250	while (state & signal_mask != 0) {	249	while (state & signal_mask != 0) {
251	const new_state = state - one_waiter - one_signal;	250	const new_state = state - one_waiter - one_signal;
252	state = self.state.tryCompareAndSwap(state, new_state, .Acquire, .Monotonic) orelse return;	251	state = self.state.cmpxchgWeak(state, new_state, .Acquire, .Monotonic) orelse return;
253	}	252	}
254	}	253	}
255	}	254	}
...	@@ -276,7 +275,7 @@ const FutexImpl = struct {	...	@@ -276,7 +275,7 @@ const FutexImpl = struct {
276	// Reserve the amount of waiters to wake by incrementing the signals count.	275	// Reserve the amount of waiters to wake by incrementing the signals count.
277	// Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.	276	// Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.
278	const new_state = state + (one_signal * to_wake);	277	const new_state = state + (one_signal * to_wake);
279	state = self.state.tryCompareAndSwap(state, new_state, .Release, .Monotonic) orelse {	278	state = self.state.cmpxchgWeak(state, new_state, .Release, .Monotonic) orelse {
280	// Wake up the waiting threads we reserved above by changing the epoch value.	279	// Wake up the waiting threads we reserved above by changing the epoch value.
281	// NOTE: a waiting thread could miss a wake up if exactly ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.	280	// NOTE: a waiting thread could miss a wake up if exactly ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.
282	// This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.	281	// This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.

...	@@ -10,7 +10,7 @@ const Futex = @This();	...	@@ -10,7 +10,7 @@ const Futex = @This();
10	const os = std.os;	10	const os = std.os;
11	const assert = std.debug.assert;	11	const assert = std.debug.assert;
12	const testing = std.testing;	12	const testing = std.testing;
13	const Atomic = std.atomic.Atomic;	13	const atomic = std.atomic;
14		14
15	/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:	15	/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
16	/// - The value at `ptr` is no longer equal to `expect`.	16	/// - The value at `ptr` is no longer equal to `expect`.
...	@@ -19,7 +19,7 @@ const Atomic = std.atomic.Atomic;	...	@@ -19,7 +19,7 @@ const Atomic = std.atomic.Atomic;
19	///	19	///
20	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically	20	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
21	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.	21	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
22	pub fn wait(ptr: *const Atomic(u32), expect: u32) void {	22	pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void {
23	@setCold(true);	23	@setCold(true);
24		24
25	Impl.wait(ptr, expect, null) catch \|err\| switch (err) {	25	Impl.wait(ptr, expect, null) catch \|err\| switch (err) {
...	@@ -35,7 +35,7 @@ pub fn wait(ptr: *const Atomic(u32), expect: u32) void {	...	@@ -35,7 +35,7 @@ pub fn wait(ptr: *const Atomic(u32), expect: u32) void {
35	///	35	///
36	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically	36	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
37	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.	37	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
38	pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {	38	pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {
39	@setCold(true);	39	@setCold(true);
40		40
41	// Avoid calling into the OS for no-op timeouts.	41	// Avoid calling into the OS for no-op timeouts.
...	@@ -48,7 +48,7 @@ pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Ti	...	@@ -48,7 +48,7 @@ pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Ti
48	}	48	}
49		49
50	/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.	50	/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.
51	pub fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	51	pub fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
52	@setCold(true);	52	@setCold(true);
53		53
54	// Avoid calling into the OS if there's nothing to wake up.	54	// Avoid calling into the OS if there's nothing to wake up.
...	@@ -83,11 +83,11 @@ else	...	@@ -83,11 +83,11 @@ else
83	/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.	83	/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.
84	/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.	84	/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.
85	const UnsupportedImpl = struct {	85	const UnsupportedImpl = struct {
86	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	86	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
87	return unsupported(.{ ptr, expect, timeout });	87	return unsupported(.{ ptr, expect, timeout });
88	}	88	}
89		89
90	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	90	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
91	return unsupported(.{ ptr, max_waiters });	91	return unsupported(.{ ptr, max_waiters });
92	}	92	}
93		93
...	@@ -98,8 +98,8 @@ const UnsupportedImpl = struct {	...	@@ -98,8 +98,8 @@ const UnsupportedImpl = struct {
98	};	98	};
99		99
100	const SingleThreadedImpl = struct {	100	const SingleThreadedImpl = struct {
101	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	101	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
102	if (ptr.loadUnchecked() != expect) {	102	if (ptr.raw != expect) {
103	return;	103	return;
104	}	104	}
105		105
...	@@ -113,7 +113,7 @@ const SingleThreadedImpl = struct {	...	@@ -113,7 +113,7 @@ const SingleThreadedImpl = struct {
113	return error.Timeout;	113	return error.Timeout;
114	}	114	}
115		115
116	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	116	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
117	// There are no other threads to possibly wake up	117	// There are no other threads to possibly wake up
118	_ = ptr;	118	_ = ptr;
119	_ = max_waiters;	119	_ = max_waiters;
...	@@ -123,7 +123,7 @@ const SingleThreadedImpl = struct {	...	@@ -123,7 +123,7 @@ const SingleThreadedImpl = struct {
123	// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll	123	// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll
124	// as it's generally already a linked target and is autoloaded into all processes anyway.	124	// as it's generally already a linked target and is autoloaded into all processes anyway.
125	const WindowsImpl = struct {	125	const WindowsImpl = struct {
126	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	126	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
127	var timeout_value: os.windows.LARGE_INTEGER = undefined;	127	var timeout_value: os.windows.LARGE_INTEGER = undefined;
128	var timeout_ptr: ?*const os.windows.LARGE_INTEGER = null;	128	var timeout_ptr: ?*const os.windows.LARGE_INTEGER = null;
129		129
...	@@ -152,7 +152,7 @@ const WindowsImpl = struct {	...	@@ -152,7 +152,7 @@ const WindowsImpl = struct {
152	}	152	}
153	}	153	}
154		154
155	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	155	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
156	const address: ?*const anyopaque = ptr;	156	const address: ?*const anyopaque = ptr;
157	assert(max_waiters != 0);	157	assert(max_waiters != 0);
158		158
...	@@ -164,7 +164,7 @@ const WindowsImpl = struct {	...	@@ -164,7 +164,7 @@ const WindowsImpl = struct {
164	};	164	};
165		165
166	const DarwinImpl = struct {	166	const DarwinImpl = struct {
167	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	167	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
168	// Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:	168	// Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:
169	// https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6	169	// https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6
170	//	170	//
...	@@ -220,7 +220,7 @@ const DarwinImpl = struct {	...	@@ -220,7 +220,7 @@ const DarwinImpl = struct {
220	}	220	}
221	}	221	}
222		222
223	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	223	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
224	var flags: u32 = os.darwin.UL_COMPARE_AND_WAIT \| os.darwin.ULF_NO_ERRNO;	224	var flags: u32 = os.darwin.UL_COMPARE_AND_WAIT \| os.darwin.ULF_NO_ERRNO;
225	if (max_waiters > 1) {	225	if (max_waiters > 1) {
226	flags \|= os.darwin.ULF_WAKE_ALL;	226	flags \|= os.darwin.ULF_WAKE_ALL;
...	@@ -244,7 +244,7 @@ const DarwinImpl = struct {	...	@@ -244,7 +244,7 @@ const DarwinImpl = struct {
244		244
245	// https://man7.org/linux/man-pages/man2/futex.2.html	245	// https://man7.org/linux/man-pages/man2/futex.2.html
246	const LinuxImpl = struct {	246	const LinuxImpl = struct {
247	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	247	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
248	var ts: os.timespec = undefined;	248	var ts: os.timespec = undefined;
249	if (timeout) \|timeout_ns\| {	249	if (timeout) \|timeout_ns\| {
250	ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));	250	ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));
...	@@ -252,7 +252,7 @@ const LinuxImpl = struct {	...	@@ -252,7 +252,7 @@ const LinuxImpl = struct {
252	}	252	}
253		253
254	const rc = os.linux.futex_wait(	254	const rc = os.linux.futex_wait(
255	@as(*const i32, @ptrCast(&ptr.value)),	255	@as(*const i32, @ptrCast(&ptr.raw)),
256	os.linux.FUTEX.PRIVATE_FLAG \| os.linux.FUTEX.WAIT,	256	os.linux.FUTEX.PRIVATE_FLAG \| os.linux.FUTEX.WAIT,
257	@as(i32, @bitCast(expect)),	257	@as(i32, @bitCast(expect)),
258	if (timeout != null) &ts else null,	258	if (timeout != null) &ts else null,
...	@@ -272,9 +272,9 @@ const LinuxImpl = struct {	...	@@ -272,9 +272,9 @@ const LinuxImpl = struct {
272	}	272	}
273	}	273	}
274		274
275	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	275	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
276	const rc = os.linux.futex_wake(	276	const rc = os.linux.futex_wake(
277	@as(*const i32, @ptrCast(&ptr.value)),	277	@as(*const i32, @ptrCast(&ptr.raw)),
278	os.linux.FUTEX.PRIVATE_FLAG \| os.linux.FUTEX.WAKE,	278	os.linux.FUTEX.PRIVATE_FLAG \| os.linux.FUTEX.WAKE,
279	std.math.cast(i32, max_waiters) orelse std.math.maxInt(i32),	279	std.math.cast(i32, max_waiters) orelse std.math.maxInt(i32),
280	);	280	);
...	@@ -290,7 +290,7 @@ const LinuxImpl = struct {	...	@@ -290,7 +290,7 @@ const LinuxImpl = struct {
290		290
291	// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1	291	// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1
292	const FreebsdImpl = struct {	292	const FreebsdImpl = struct {
293	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	293	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
294	var tm_size: usize = 0;	294	var tm_size: usize = 0;
295	var tm: os.freebsd._umtx_time = undefined;	295	var tm: os.freebsd._umtx_time = undefined;
296	var tm_ptr: ?*const os.freebsd._umtx_time = null;	296	var tm_ptr: ?*const os.freebsd._umtx_time = null;
...	@@ -326,7 +326,7 @@ const FreebsdImpl = struct {	...	@@ -326,7 +326,7 @@ const FreebsdImpl = struct {
326	}	326	}
327	}	327	}
328		328
329	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	329	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
330	const rc = os.freebsd._umtx_op(	330	const rc = os.freebsd._umtx_op(
331	@intFromPtr(&ptr.value),	331	@intFromPtr(&ptr.value),
332	@intFromEnum(os.freebsd.UMTX_OP.WAKE_PRIVATE),	332	@intFromEnum(os.freebsd.UMTX_OP.WAKE_PRIVATE),
...	@@ -346,7 +346,7 @@ const FreebsdImpl = struct {	...	@@ -346,7 +346,7 @@ const FreebsdImpl = struct {
346		346
347	// https://man.openbsd.org/futex.2	347	// https://man.openbsd.org/futex.2
348	const OpenbsdImpl = struct {	348	const OpenbsdImpl = struct {
349	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	349	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
350	var ts: os.timespec = undefined;	350	var ts: os.timespec = undefined;
351	if (timeout) \|timeout_ns\| {	351	if (timeout) \|timeout_ns\| {
352	ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));	352	ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));
...	@@ -377,7 +377,7 @@ const OpenbsdImpl = struct {	...	@@ -377,7 +377,7 @@ const OpenbsdImpl = struct {
377	}	377	}
378	}	378	}
379		379
380	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	380	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
381	const rc = os.openbsd.futex(	381	const rc = os.openbsd.futex(
382	@as(*const volatile u32, @ptrCast(&ptr.value)),	382	@as(*const volatile u32, @ptrCast(&ptr.value)),
383	os.openbsd.FUTEX_WAKE \| os.openbsd.FUTEX_PRIVATE_FLAG,	383	os.openbsd.FUTEX_WAKE \| os.openbsd.FUTEX_PRIVATE_FLAG,
...	@@ -393,7 +393,7 @@ const OpenbsdImpl = struct {	...	@@ -393,7 +393,7 @@ const OpenbsdImpl = struct {
393		393
394	// https://man.dragonflybsd.org/?command=umtx&section=2	394	// https://man.dragonflybsd.org/?command=umtx&section=2
395	const DragonflyImpl = struct {	395	const DragonflyImpl = struct {
396	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	396	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
397	// Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.	397	// Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.
398	// It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.	398	// It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.
399	var timeout_us: c_int = 0;	399	var timeout_us: c_int = 0;
...	@@ -435,7 +435,7 @@ const DragonflyImpl = struct {	...	@@ -435,7 +435,7 @@ const DragonflyImpl = struct {
435	}	435	}
436	}	436	}
437		437
438	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	438	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
439	// A count of zero means wake all waiters.	439	// A count of zero means wake all waiters.
440	assert(max_waiters != 0);	440	assert(max_waiters != 0);
441	const to_wake = std.math.cast(c_int, max_waiters) orelse 0;	441	const to_wake = std.math.cast(c_int, max_waiters) orelse 0;
...	@@ -449,7 +449,7 @@ const DragonflyImpl = struct {	...	@@ -449,7 +449,7 @@ const DragonflyImpl = struct {
449	};	449	};
450		450
451	const WasmImpl = struct {	451	const WasmImpl = struct {
452	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	452	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
453	if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {	453	if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {
454	@compileError("WASI target missing cpu feature 'atomics'");	454	@compileError("WASI target missing cpu feature 'atomics'");
455	}	455	}
...	@@ -473,7 +473,7 @@ const WasmImpl = struct {	...	@@ -473,7 +473,7 @@ const WasmImpl = struct {
473	}	473	}
474	}	474	}
475		475
476	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	476	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
477	if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {	477	if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {
478	@compileError("WASI target missing cpu feature 'atomics'");	478	@compileError("WASI target missing cpu feature 'atomics'");
479	}	479	}
...	@@ -732,8 +732,8 @@ const PosixImpl = struct {	...	@@ -732,8 +732,8 @@ const PosixImpl = struct {
732	};	732	};
733		733
734	const Bucket = struct {	734	const Bucket = struct {
735	mutex: std.c.pthread_mutex_t align(std.atomic.cache_line) = .{},	735	mutex: std.c.pthread_mutex_t align(atomic.cache_line) = .{},
736	pending: Atomic(usize) = Atomic(usize).init(0),	736	pending: atomic.Value(usize) = atomic.Value(usize).init(0),
737	treap: Treap = .{},	737	treap: Treap = .{},
738		738
739	// Global array of buckets that addresses map to.	739	// Global array of buckets that addresses map to.
...	@@ -757,9 +757,9 @@ const PosixImpl = struct {	...	@@ -757,9 +757,9 @@ const PosixImpl = struct {
757	};	757	};
758		758
759	const Address = struct {	759	const Address = struct {
760	fn from(ptr: *const Atomic(u32)) usize {	760	fn from(ptr: *const atomic.Value(u32)) usize {
761	// Get the alignment of the pointer.	761	// Get the alignment of the pointer.
762	const alignment = @alignOf(Atomic(u32));	762	const alignment = @alignOf(atomic.Value(u32));
763	comptime assert(std.math.isPowerOfTwo(alignment));	763	comptime assert(std.math.isPowerOfTwo(alignment));
764		764
765	// Make sure the pointer is aligned,	765	// Make sure the pointer is aligned,
...	@@ -770,7 +770,7 @@ const PosixImpl = struct {	...	@@ -770,7 +770,7 @@ const PosixImpl = struct {
770	}	770	}
771	};	771	};
772		772
773	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {	773	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
774	const address = Address.from(ptr);	774	const address = Address.from(ptr);
775	const bucket = Bucket.from(address);	775	const bucket = Bucket.from(address);
776		776
...	@@ -831,7 +831,7 @@ const PosixImpl = struct {	...	@@ -831,7 +831,7 @@ const PosixImpl = struct {
831	};	831	};
832	}	832	}
833		833
834	fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {	834	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
835	const address = Address.from(ptr);	835	const address = Address.from(ptr);
836	const bucket = Bucket.from(address);	836	const bucket = Bucket.from(address);
837		837
...	@@ -882,7 +882,7 @@ const PosixImpl = struct {	...	@@ -882,7 +882,7 @@ const PosixImpl = struct {
882	};	882	};
883		883
884	test "Futex - smoke test" {	884	test "Futex - smoke test" {
885	var value = Atomic(u32).init(0);	885	var value = atomic.Value(u32).init(0);
886		886
887	// Try waits with invalid values.	887	// Try waits with invalid values.
888	Futex.wait(&value, 0xdeadbeef);	888	Futex.wait(&value, 0xdeadbeef);
...	@@ -908,7 +908,7 @@ test "Futex - signaling" {	...	@@ -908,7 +908,7 @@ test "Futex - signaling" {
908	const num_iterations = 4;	908	const num_iterations = 4;
909		909
910	const Paddle = struct {	910	const Paddle = struct {
911	value: Atomic(u32) = Atomic(u32).init(0),	911	value: atomic.Value(u32) = atomic.Value(u32).init(0),
912	current: u32 = 0,	912	current: u32 = 0,
913		913
914	fn hit(self: *@This()) void {	914	fn hit(self: *@This()) void {
...	@@ -962,8 +962,8 @@ test "Futex - broadcasting" {	...	@@ -962,8 +962,8 @@ test "Futex - broadcasting" {
962	const num_iterations = 4;	962	const num_iterations = 4;
963		963
964	const Barrier = struct {	964	const Barrier = struct {
965	count: Atomic(u32) = Atomic(u32).init(num_threads),	965	count: atomic.Value(u32) = atomic.Value(u32).init(num_threads),
966	futex: Atomic(u32) = Atomic(u32).init(0),	966	futex: atomic.Value(u32) = atomic.Value(u32).init(0),
967		967
968	fn wait(self: *@This()) !void {	968	fn wait(self: *@This()) !void {
969	// Decrement the counter.	969	// Decrement the counter.
...	@@ -1036,7 +1036,7 @@ pub const Deadline = struct {	...	@@ -1036,7 +1036,7 @@ pub const Deadline = struct {
1036	/// - `Futex.wake()` is called on the `ptr`.	1036	/// - `Futex.wake()` is called on the `ptr`.
1037	/// - A spurious wake occurs.	1037	/// - A spurious wake occurs.
1038	/// - The deadline expires; In which case `error.Timeout` is returned.	1038	/// - The deadline expires; In which case `error.Timeout` is returned.
1039	pub fn wait(self: Deadline, ptr: const Atomic(u32), expect: u32) error{Timeout}!void {	1039	pub fn wait(self: Deadline, ptr: const atomic.Value(u32), expect: u32) error{Timeout}!void {
1040	@setCold(true);	1040	@setCold(true);
1041		1041
1042	// Check if we actually have a timeout to wait until.	1042	// Check if we actually have a timeout to wait until.
...	@@ -1056,7 +1056,7 @@ pub const Deadline = struct {	...	@@ -1056,7 +1056,7 @@ pub const Deadline = struct {
1056		1056
1057	test "Futex - Deadline" {	1057	test "Futex - Deadline" {
1058	var deadline = Deadline.init(100 * std.time.ns_per_ms);	1058	var deadline = Deadline.init(100 * std.time.ns_per_ms);
1059	var futex_word = Atomic(u32).init(0);	1059	var futex_word = atomic.Value(u32).init(0);
1060		1060
1061	while (true) {	1061	while (true) {
1062	deadline.wait(&futex_word, 0) catch break;	1062	deadline.wait(&futex_word, 0) catch break;

Merge pull request #18085 from ziglang/std-atomics