authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-12 18:55:16-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-03-12 18:55:16-04:00
logf51bec321b1ef9196b143295d47a9ea919cdba28
treeec4871b071e28a1b4381399c166b5be7f77e1532
parentaa49f972d655eb61ca899f76ba37933254f780a2
parent71d776c3be91f6b4e982b45fbfe03e3696a397f5
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4707 from Vexu/small-atomics

Support atomic operations with bools and non power of two integers

11 files changed, 415 insertions(+), 176 deletions(-)

doc/langref.html.in+10-35
...@@ -6728,17 +6728,8 @@ async fn func(y: *i32) void {...@@ -6728,17 +6728,8 @@ async fn func(y: *i32) void {
6728 This builtin function atomically dereferences a pointer and returns the value.6728 This builtin function atomically dereferences a pointer and returns the value.
6729 </p>6729 </p>
6730 <p>6730 <p>
6731 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,6731 {#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
6732 an integer whose bit count meets these requirements:6732 an integer or an enum.
6733 </p>
6734 <ul>
6735 <li>At least 8</li>
6736 <li>At most the same as usize</li>
6737 <li>Power of 2</li>
6738 </ul> or an enum with a valid integer tag type.
6739 <p>
6740 TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe
6741 we can remove this restriction
6742 </p>6733 </p>
6743 {#header_close#}6734 {#header_close#}
6744 {#header_open|@atomicRmw#}6735 {#header_open|@atomicRmw#}
...@@ -6747,17 +6738,8 @@ async fn func(y: *i32) void {...@@ -6747,17 +6738,8 @@ async fn func(y: *i32) void {
6747 This builtin function atomically modifies memory and then returns the previous value.6738 This builtin function atomically modifies memory and then returns the previous value.
6748 </p>6739 </p>
6749 <p>6740 <p>
6750 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#},6741 {#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
6751 or an integer whose bit count meets these requirements:6742 an integer or an enum.
6752 </p>
6753 <ul>
6754 <li>At least 8</li>
6755 <li>At most the same as usize</li>
6756 <li>Power of 2</li>
6757 </ul>
6758 <p>
6759 TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe
6760 we can remove this restriction
6761 </p>6743 </p>
6762 <p>6744 <p>
6763 Supported operations:6745 Supported operations:
...@@ -6782,17 +6764,8 @@ async fn func(y: *i32) void {...@@ -6782,17 +6764,8 @@ async fn func(y: *i32) void {
6782 This builtin function atomically stores a value.6764 This builtin function atomically stores a value.
6783 </p>6765 </p>
6784 <p>6766 <p>
6785 {#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,6767 {#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
6786 an integer whose bit count meets these requirements:6768 an integer or an enum.
6787 </p>
6788 <ul>
6789 <li>At least 8</li>
6790 <li>At most the same as usize</li>
6791 <li>Power of 2</li>
6792 </ul> or an enum with a valid integer tag type.
6793 <p>
6794 TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe
6795 we can remove this restriction
6796 </p>6769 </p>
6797 {#header_close#}6770 {#header_close#}
6798 {#header_open|@bitCast#}6771 {#header_open|@bitCast#}
...@@ -7108,7 +7081,8 @@ fn cmpxchgStrongButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_v...@@ -7108,7 +7081,8 @@ fn cmpxchgStrongButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_v
7108 more efficiently in machine instructions.7081 more efficiently in machine instructions.
7109 </p>7082 </p>
7110 <p>7083 <p>
7111 {#syntax#}AtomicOrder{#endsyntax#} can be found with {#syntax#}@import("builtin").AtomicOrder{#endsyntax#}.7084 {#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
7085 an integer or an enum.
7112 </p>7086 </p>
7113 <p>{#syntax#}@TypeOf(ptr).alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>7087 <p>{#syntax#}@TypeOf(ptr).alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>
7114 {#see_also|Compile Variables|cmpxchgWeak#}7088 {#see_also|Compile Variables|cmpxchgWeak#}
...@@ -7136,7 +7110,8 @@ fn cmpxchgWeakButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_val...@@ -7136,7 +7110,8 @@ fn cmpxchgWeakButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_val
7136 However if you need a stronger guarantee, use {#link|@cmpxchgStrong#}.7110 However if you need a stronger guarantee, use {#link|@cmpxchgStrong#}.
7137 </p>7111 </p>
7138 <p>7112 <p>
7139 {#syntax#}AtomicOrder{#endsyntax#} can be found with {#syntax#}@import("builtin").AtomicOrder{#endsyntax#}.7113 {#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
7114 an integer or an enum.
7140 </p>7115 </p>
7141 <p>{#syntax#}@TypeOf(ptr).alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>7116 <p>{#syntax#}@TypeOf(ptr).alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>
7142 {#see_also|Compile Variables|cmpxchgStrong#}7117 {#see_also|Compile Variables|cmpxchgStrong#}
lib/std/atomic/int.zig+5-8
...@@ -1,6 +1,3 @@...@@ -1,6 +1,3 @@
1const builtin = @import("builtin");
2const AtomicOrder = builtin.AtomicOrder;
3
4/// Thread-safe, lock-free integer1/// Thread-safe, lock-free integer
5pub fn Int(comptime T: type) type {2pub fn Int(comptime T: type) type {
6 return struct {3 return struct {
...@@ -14,16 +11,16 @@ pub fn Int(comptime T: type) type {...@@ -14,16 +11,16 @@ pub fn Int(comptime T: type) type {
1411
15 /// Returns previous value12 /// Returns previous value
16 pub fn incr(self: *Self) T {13 pub fn incr(self: *Self) T {
17 return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);14 return @atomicRmw(T, &self.unprotected_value, .Add, 1, .SeqCst);
18 }15 }
1916
20 /// Returns previous value17 /// Returns previous value
21 pub fn decr(self: *Self) T {18 pub fn decr(self: *Self) T {
22 return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);19 return @atomicRmw(T, &self.unprotected_value, .Sub, 1, .SeqCst);
23 }20 }
2421
25 pub fn get(self: *Self) T {22 pub fn get(self: *Self) T {
26 return @atomicLoad(T, &self.unprotected_value, AtomicOrder.SeqCst);23 return @atomicLoad(T, &self.unprotected_value, .SeqCst);
27 }24 }
2825
29 pub fn set(self: *Self, new_value: T) void {26 pub fn set(self: *Self, new_value: T) void {
...@@ -31,11 +28,11 @@ pub fn Int(comptime T: type) type {...@@ -31,11 +28,11 @@ pub fn Int(comptime T: type) type {
31 }28 }
3229
33 pub fn xchg(self: *Self, new_value: T) T {30 pub fn xchg(self: *Self, new_value: T) T {
34 return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Xchg, new_value, AtomicOrder.SeqCst);31 return @atomicRmw(T, &self.unprotected_value, .Xchg, new_value, .SeqCst);
35 }32 }
3633
37 pub fn fetchAdd(self: *Self, op: T) T {34 pub fn fetchAdd(self: *Self, op: T) T {
38 return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, op, AtomicOrder.SeqCst);35 return @atomicRmw(T, &self.unprotected_value, .Add, op, .SeqCst);
39 }36 }
40 };37 };
41}38}
lib/std/atomic/queue.zig+9-11
...@@ -1,7 +1,5 @@...@@ -1,7 +1,5 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const AtomicOrder = builtin.AtomicOrder;
4const AtomicRmwOp = builtin.AtomicRmwOp;
5const assert = std.debug.assert;3const assert = std.debug.assert;
6const expect = std.testing.expect;4const expect = std.testing.expect;
75
...@@ -145,7 +143,7 @@ const Context = struct {...@@ -145,7 +143,7 @@ const Context = struct {
145 put_sum: isize,143 put_sum: isize,
146 get_sum: isize,144 get_sum: isize,
147 get_count: usize,145 get_count: usize,
148 puts_done: u8, // TODO make this a bool146 puts_done: bool,
149};147};
150148
151// TODO add lazy evaluated build options and then put puts_per_thread behind149// TODO add lazy evaluated build options and then put puts_per_thread behind
...@@ -169,7 +167,7 @@ test "std.atomic.Queue" {...@@ -169,7 +167,7 @@ test "std.atomic.Queue" {
169 .queue = &queue,167 .queue = &queue,
170 .put_sum = 0,168 .put_sum = 0,
171 .get_sum = 0,169 .get_sum = 0,
172 .puts_done = 0,170 .puts_done = false,
173 .get_count = 0,171 .get_count = 0,
174 };172 };
175173
...@@ -182,7 +180,7 @@ test "std.atomic.Queue" {...@@ -182,7 +180,7 @@ test "std.atomic.Queue" {
182 }180 }
183 }181 }
184 expect(!context.queue.isEmpty());182 expect(!context.queue.isEmpty());
185 context.puts_done = 1;183 context.puts_done = true;
186 {184 {
187 var i: usize = 0;185 var i: usize = 0;
188 while (i < put_thread_count) : (i += 1) {186 while (i < put_thread_count) : (i += 1) {
...@@ -204,7 +202,7 @@ test "std.atomic.Queue" {...@@ -204,7 +202,7 @@ test "std.atomic.Queue" {
204202
205 for (putters) |t|203 for (putters) |t|
206 t.wait();204 t.wait();
207 @atomicStore(u8, &context.puts_done, 1, AtomicOrder.SeqCst);205 @atomicStore(bool, &context.puts_done, true, .SeqCst);
208 for (getters) |t|206 for (getters) |t|
209 t.wait();207 t.wait();
210208
...@@ -231,25 +229,25 @@ fn startPuts(ctx: *Context) u8 {...@@ -231,25 +229,25 @@ fn startPuts(ctx: *Context) u8 {
231 std.time.sleep(1); // let the os scheduler be our fuzz229 std.time.sleep(1); // let the os scheduler be our fuzz
232 const x = @bitCast(i32, r.random.scalar(u32));230 const x = @bitCast(i32, r.random.scalar(u32));
233 const node = ctx.allocator.create(Queue(i32).Node) catch unreachable;231 const node = ctx.allocator.create(Queue(i32).Node) catch unreachable;
234 node.* = Queue(i32).Node{232 node.* = .{
235 .prev = undefined,233 .prev = undefined,
236 .next = undefined,234 .next = undefined,
237 .data = x,235 .data = x,
238 };236 };
239 ctx.queue.put(node);237 ctx.queue.put(node);
240 _ = @atomicRmw(isize, &ctx.put_sum, builtin.AtomicRmwOp.Add, x, AtomicOrder.SeqCst);238 _ = @atomicRmw(isize, &ctx.put_sum, .Add, x, .SeqCst);
241 }239 }
242 return 0;240 return 0;
243}241}
244242
245fn startGets(ctx: *Context) u8 {243fn startGets(ctx: *Context) u8 {
246 while (true) {244 while (true) {
247 const last = @atomicLoad(u8, &ctx.puts_done, builtin.AtomicOrder.SeqCst) == 1;245 const last = @atomicLoad(bool, &ctx.puts_done, .SeqCst);
248246
249 while (ctx.queue.get()) |node| {247 while (ctx.queue.get()) |node| {
250 std.time.sleep(1); // let the os scheduler be our fuzz248 std.time.sleep(1); // let the os scheduler be our fuzz
251 _ = @atomicRmw(isize, &ctx.get_sum, builtin.AtomicRmwOp.Add, node.data, builtin.AtomicOrder.SeqCst);249 _ = @atomicRmw(isize, &ctx.get_sum, .Add, node.data, .SeqCst);
252 _ = @atomicRmw(usize, &ctx.get_count, builtin.AtomicRmwOp.Add, 1, builtin.AtomicOrder.SeqCst);250 _ = @atomicRmw(usize, &ctx.get_count, .Add, 1, .SeqCst);
253 }251 }
254252
255 if (last) return 0;253 if (last) return 0;
lib/std/atomic/stack.zig+15-16
...@@ -1,6 +1,5 @@...@@ -1,6 +1,5 @@
1const assert = std.debug.assert;1const assert = std.debug.assert;
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const AtomicOrder = builtin.AtomicOrder;
4const expect = std.testing.expect;3const expect = std.testing.expect;
54
6/// Many reader, many writer, non-allocating, thread-safe5/// Many reader, many writer, non-allocating, thread-safe
...@@ -11,7 +10,7 @@ pub fn Stack(comptime T: type) type {...@@ -11,7 +10,7 @@ pub fn Stack(comptime T: type) type {
11 root: ?*Node,10 root: ?*Node,
12 lock: @TypeOf(lock_init),11 lock: @TypeOf(lock_init),
1312
14 const lock_init = if (builtin.single_threaded) {} else @as(u8, 0);13 const lock_init = if (builtin.single_threaded) {} else false;
1514
16 pub const Self = @This();15 pub const Self = @This();
1716
...@@ -31,7 +30,7 @@ pub fn Stack(comptime T: type) type {...@@ -31,7 +30,7 @@ pub fn Stack(comptime T: type) type {
31 /// being the first item in the stack, returns the other item that was there.30 /// being the first item in the stack, returns the other item that was there.
32 pub fn pushFirst(self: *Self, node: *Node) ?*Node {31 pub fn pushFirst(self: *Self, node: *Node) ?*Node {
33 node.next = null;32 node.next = null;
34 return @cmpxchgStrong(?*Node, &self.root, null, node, AtomicOrder.SeqCst, AtomicOrder.SeqCst);33 return @cmpxchgStrong(?*Node, &self.root, null, node, .SeqCst, .SeqCst);
35 }34 }
3635
37 pub fn push(self: *Self, node: *Node) void {36 pub fn push(self: *Self, node: *Node) void {
...@@ -39,8 +38,8 @@ pub fn Stack(comptime T: type) type {...@@ -39,8 +38,8 @@ pub fn Stack(comptime T: type) type {
39 node.next = self.root;38 node.next = self.root;
40 self.root = node;39 self.root = node;
41 } else {40 } else {
42 while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {}41 while (@atomicRmw(bool, &self.lock, .Xchg, true, .SeqCst)) {}
43 defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1);42 defer assert(@atomicRmw(bool, &self.lock, .Xchg, false, .SeqCst));
4443
45 node.next = self.root;44 node.next = self.root;
46 self.root = node;45 self.root = node;
...@@ -53,8 +52,8 @@ pub fn Stack(comptime T: type) type {...@@ -53,8 +52,8 @@ pub fn Stack(comptime T: type) type {
53 self.root = root.next;52 self.root = root.next;
54 return root;53 return root;
55 } else {54 } else {
56 while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {}55 while (@atomicRmw(bool, &self.lock, .Xchg, true, .SeqCst)) {}
57 defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1);56 defer assert(@atomicRmw(bool, &self.lock, .Xchg, false, .SeqCst));
5857
59 const root = self.root orelse return null;58 const root = self.root orelse return null;
60 self.root = root.next;59 self.root = root.next;
...@@ -63,7 +62,7 @@ pub fn Stack(comptime T: type) type {...@@ -63,7 +62,7 @@ pub fn Stack(comptime T: type) type {
63 }62 }
6463
65 pub fn isEmpty(self: *Self) bool {64 pub fn isEmpty(self: *Self) bool {
66 return @atomicLoad(?*Node, &self.root, AtomicOrder.SeqCst) == null;65 return @atomicLoad(?*Node, &self.root, .SeqCst) == null;
67 }66 }
68 };67 };
69}68}
...@@ -75,7 +74,7 @@ const Context = struct {...@@ -75,7 +74,7 @@ const Context = struct {
75 put_sum: isize,74 put_sum: isize,
76 get_sum: isize,75 get_sum: isize,
77 get_count: usize,76 get_count: usize,
78 puts_done: u8, // TODO make this a bool77 puts_done: bool,
79};78};
80// TODO add lazy evaluated build options and then put puts_per_thread behind79// TODO add lazy evaluated build options and then put puts_per_thread behind
81// some option such as: "AggressiveMultithreadedFuzzTest". In the AppVeyor80// some option such as: "AggressiveMultithreadedFuzzTest". In the AppVeyor
...@@ -98,7 +97,7 @@ test "std.atomic.stack" {...@@ -98,7 +97,7 @@ test "std.atomic.stack" {
98 .stack = &stack,97 .stack = &stack,
99 .put_sum = 0,98 .put_sum = 0,
100 .get_sum = 0,99 .get_sum = 0,
101 .puts_done = 0,100 .puts_done = false,
102 .get_count = 0,101 .get_count = 0,
103 };102 };
104103
...@@ -109,7 +108,7 @@ test "std.atomic.stack" {...@@ -109,7 +108,7 @@ test "std.atomic.stack" {
109 expect(startPuts(&context) == 0);108 expect(startPuts(&context) == 0);
110 }109 }
111 }110 }
112 context.puts_done = 1;111 context.puts_done = true;
113 {112 {
114 var i: usize = 0;113 var i: usize = 0;
115 while (i < put_thread_count) : (i += 1) {114 while (i < put_thread_count) : (i += 1) {
...@@ -128,7 +127,7 @@ test "std.atomic.stack" {...@@ -128,7 +127,7 @@ test "std.atomic.stack" {
128127
129 for (putters) |t|128 for (putters) |t|
130 t.wait();129 t.wait();
131 @atomicStore(u8, &context.puts_done, 1, AtomicOrder.SeqCst);130 @atomicStore(bool, &context.puts_done, true, .SeqCst);
132 for (getters) |t|131 for (getters) |t|
133 t.wait();132 t.wait();
134 }133 }
...@@ -158,19 +157,19 @@ fn startPuts(ctx: *Context) u8 {...@@ -158,19 +157,19 @@ fn startPuts(ctx: *Context) u8 {
158 .data = x,157 .data = x,
159 };158 };
160 ctx.stack.push(node);159 ctx.stack.push(node);
161 _ = @atomicRmw(isize, &ctx.put_sum, builtin.AtomicRmwOp.Add, x, AtomicOrder.SeqCst);160 _ = @atomicRmw(isize, &ctx.put_sum, .Add, x, .SeqCst);
162 }161 }
163 return 0;162 return 0;
164}163}
165164
166fn startGets(ctx: *Context) u8 {165fn startGets(ctx: *Context) u8 {
167 while (true) {166 while (true) {
168 const last = @atomicLoad(u8, &ctx.puts_done, builtin.AtomicOrder.SeqCst) == 1;167 const last = @atomicLoad(bool, &ctx.puts_done, .SeqCst);
169168
170 while (ctx.stack.pop()) |node| {169 while (ctx.stack.pop()) |node| {
171 std.time.sleep(1); // let the os scheduler be our fuzz170 std.time.sleep(1); // let the os scheduler be our fuzz
172 _ = @atomicRmw(isize, &ctx.get_sum, builtin.AtomicRmwOp.Add, node.data, builtin.AtomicOrder.SeqCst);171 _ = @atomicRmw(isize, &ctx.get_sum, .Add, node.data, .SeqCst);
173 _ = @atomicRmw(usize, &ctx.get_count, builtin.AtomicRmwOp.Add, 1, builtin.AtomicOrder.SeqCst);172 _ = @atomicRmw(usize, &ctx.get_count, .Add, 1, .SeqCst);
174 }173 }
175174
176 if (last) return 0;175 if (last) return 0;
lib/std/event/channel.zig+10-13
...@@ -14,8 +14,8 @@ pub fn Channel(comptime T: type) type {...@@ -14,8 +14,8 @@ pub fn Channel(comptime T: type) type {
14 putters: std.atomic.Queue(PutNode),14 putters: std.atomic.Queue(PutNode),
15 get_count: usize,15 get_count: usize,
16 put_count: usize,16 put_count: usize,
17 dispatch_lock: u8, // TODO make this a bool17 dispatch_lock: bool,
18 need_dispatch: u8, // TODO make this a bool18 need_dispatch: bool,
1919
20 // simple fixed size ring buffer20 // simple fixed size ring buffer
21 buffer_nodes: []T,21 buffer_nodes: []T,
...@@ -62,8 +62,8 @@ pub fn Channel(comptime T: type) type {...@@ -62,8 +62,8 @@ pub fn Channel(comptime T: type) type {
62 .buffer_len = 0,62 .buffer_len = 0,
63 .buffer_nodes = buffer,63 .buffer_nodes = buffer,
64 .buffer_index = 0,64 .buffer_index = 0,
65 .dispatch_lock = 0,65 .dispatch_lock = false,
66 .need_dispatch = 0,66 .need_dispatch = false,
67 .getters = std.atomic.Queue(GetNode).init(),67 .getters = std.atomic.Queue(GetNode).init(),
68 .putters = std.atomic.Queue(PutNode).init(),68 .putters = std.atomic.Queue(PutNode).init(),
69 .or_null_queue = std.atomic.Queue(*std.atomic.Queue(GetNode).Node).init(),69 .or_null_queue = std.atomic.Queue(*std.atomic.Queue(GetNode).Node).init(),
...@@ -165,15 +165,14 @@ pub fn Channel(comptime T: type) type {...@@ -165,15 +165,14 @@ pub fn Channel(comptime T: type) type {
165165
166 fn dispatch(self: *SelfChannel) void {166 fn dispatch(self: *SelfChannel) void {
167 // set the "need dispatch" flag167 // set the "need dispatch" flag
168 @atomicStore(u8, &self.need_dispatch, 1, .SeqCst);168 @atomicStore(bool, &self.need_dispatch, true, .SeqCst);
169169
170 lock: while (true) {170 lock: while (true) {
171 // set the lock flag171 // set the lock flag
172 const prev_lock = @atomicRmw(u8, &self.dispatch_lock, .Xchg, 1, .SeqCst);172 if (@atomicRmw(bool, &self.dispatch_lock, .Xchg, true, .SeqCst)) return;
173 if (prev_lock != 0) return;
174173
175 // clear the need_dispatch flag since we're about to do it174 // clear the need_dispatch flag since we're about to do it
176 @atomicStore(u8, &self.need_dispatch, 0, .SeqCst);175 @atomicStore(bool, &self.need_dispatch, false, .SeqCst);
177176
178 while (true) {177 while (true) {
179 one_dispatch: {178 one_dispatch: {
...@@ -250,14 +249,12 @@ pub fn Channel(comptime T: type) type {...@@ -250,14 +249,12 @@ pub fn Channel(comptime T: type) type {
250 }249 }
251250
252 // clear need-dispatch flag251 // clear need-dispatch flag
253 const need_dispatch = @atomicRmw(u8, &self.need_dispatch, .Xchg, 0, .SeqCst);252 if (@atomicRmw(bool, &self.need_dispatch, .Xchg, false, .SeqCst)) continue;
254 if (need_dispatch != 0) continue;
255253
256 const my_lock = @atomicRmw(u8, &self.dispatch_lock, .Xchg, 0, .SeqCst);254 assert(@atomicRmw(bool, &self.dispatch_lock, .Xchg, false, .SeqCst));
257 assert(my_lock != 0);
258255
259 // we have to check again now that we unlocked256 // we have to check again now that we unlocked
260 if (@atomicLoad(u8, &self.need_dispatch, .SeqCst) != 0) continue :lock;257 if (@atomicLoad(bool, &self.need_dispatch, .SeqCst)) continue :lock;
261258
262 return;259 return;
263 }260 }
lib/std/event/lock.zig+17-19
...@@ -11,9 +11,9 @@ const Loop = std.event.Loop;...@@ -11,9 +11,9 @@ const Loop = std.event.Loop;
11/// Allows only one actor to hold the lock.11/// Allows only one actor to hold the lock.
12/// TODO: make this API also work in blocking I/O mode.12/// TODO: make this API also work in blocking I/O mode.
13pub const Lock = struct {13pub const Lock = struct {
14 shared_bit: u8, // TODO make this a bool14 shared: bool,
15 queue: Queue,15 queue: Queue,
16 queue_empty_bit: u8, // TODO make this a bool16 queue_empty: bool,
1717
18 const Queue = std.atomic.Queue(anyframe);18 const Queue = std.atomic.Queue(anyframe);
1919
...@@ -31,20 +31,19 @@ pub const Lock = struct {...@@ -31,20 +31,19 @@ pub const Lock = struct {
31 }31 }
3232
33 // We need to release the lock.33 // We need to release the lock.
34 @atomicStore(u8, &self.lock.queue_empty_bit, 1, .SeqCst);34 @atomicStore(bool, &self.lock.queue_empty, true, .SeqCst);
35 @atomicStore(u8, &self.lock.shared_bit, 0, .SeqCst);35 @atomicStore(bool, &self.lock.shared, false, .SeqCst);
3636
37 // There might be a queue item. If we know the queue is empty, we can be done,37 // There might be a queue item. If we know the queue is empty, we can be done,
38 // because the other actor will try to obtain the lock.38 // because the other actor will try to obtain the lock.
39 // But if there's a queue item, we are the actor which must loop and attempt39 // But if there's a queue item, we are the actor which must loop and attempt
40 // to grab the lock again.40 // to grab the lock again.
41 if (@atomicLoad(u8, &self.lock.queue_empty_bit, .SeqCst) == 1) {41 if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) {
42 return;42 return;
43 }43 }
4444
45 while (true) {45 while (true) {
46 const old_bit = @atomicRmw(u8, &self.lock.shared_bit, .Xchg, 1, .SeqCst);46 if (@atomicRmw(bool, &self.lock.shared, .Xchg, true, .SeqCst)) {
47 if (old_bit != 0) {
48 // We did not obtain the lock. Great, the queue is someone else's problem.47 // We did not obtain the lock. Great, the queue is someone else's problem.
49 return;48 return;
50 }49 }
...@@ -56,11 +55,11 @@ pub const Lock = struct {...@@ -56,11 +55,11 @@ pub const Lock = struct {
56 }55 }
5756
58 // Release the lock again.57 // Release the lock again.
59 @atomicStore(u8, &self.lock.queue_empty_bit, 1, .SeqCst);58 @atomicStore(bool, &self.lock.queue_empty, true, .SeqCst);
60 @atomicStore(u8, &self.lock.shared_bit, 0, .SeqCst);59 @atomicStore(bool, &self.lock.shared, false, .SeqCst);
6160
62 // Find out if we can be done.61 // Find out if we can be done.
63 if (@atomicLoad(u8, &self.lock.queue_empty_bit, .SeqCst) == 1) {62 if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) {
64 return;63 return;
65 }64 }
66 }65 }
...@@ -69,24 +68,24 @@ pub const Lock = struct {...@@ -69,24 +68,24 @@ pub const Lock = struct {
6968
70 pub fn init() Lock {69 pub fn init() Lock {
71 return Lock{70 return Lock{
72 .shared_bit = 0,71 .shared = false,
73 .queue = Queue.init(),72 .queue = Queue.init(),
74 .queue_empty_bit = 1,73 .queue_empty = true,
75 };74 };
76 }75 }
7776
78 pub fn initLocked() Lock {77 pub fn initLocked() Lock {
79 return Lock{78 return Lock{
80 .shared_bit = 1,79 .shared = true,
81 .queue = Queue.init(),80 .queue = Queue.init(),
82 .queue_empty_bit = 1,81 .queue_empty = true,
83 };82 };
84 }83 }
8584
86 /// Must be called when not locked. Not thread safe.85 /// Must be called when not locked. Not thread safe.
87 /// All calls to acquire() and release() must complete before calling deinit().86 /// All calls to acquire() and release() must complete before calling deinit().
88 pub fn deinit(self: *Lock) void {87 pub fn deinit(self: *Lock) void {
89 assert(self.shared_bit == 0);88 assert(!self.shared);
90 while (self.queue.get()) |node| resume node.data;89 while (self.queue.get()) |node| resume node.data;
91 }90 }
9291
...@@ -99,12 +98,11 @@ pub const Lock = struct {...@@ -99,12 +98,11 @@ pub const Lock = struct {
9998
100 // At this point, we are in the queue, so we might have already been resumed.99 // At this point, we are in the queue, so we might have already been resumed.
101100
102 // We set this bit so that later we can rely on the fact, that if queue_empty_bit is 1, some actor101 // We set this bit so that later we can rely on the fact, that if queue_empty == true, some actor
103 // will attempt to grab the lock.102 // will attempt to grab the lock.
104 @atomicStore(u8, &self.queue_empty_bit, 0, .SeqCst);103 @atomicStore(bool, &self.queue_empty, false, .SeqCst);
105104
106 const old_bit = @atomicRmw(u8, &self.shared_bit, .Xchg, 1, .SeqCst);105 if (!@atomicRmw(bool, &self.shared, .Xchg, true, .SeqCst)) {
107 if (old_bit == 0) {
108 if (self.queue.get()) |node| {106 if (self.queue.get()) |node| {
109 // Whether this node is us or someone else, we tail resume it.107 // Whether this node is us or someone else, we tail resume it.
110 resume node.data;108 resume node.data;
lib/std/event/rwlock.zig+16-16
...@@ -16,8 +16,8 @@ pub const RwLock = struct {...@@ -16,8 +16,8 @@ pub const RwLock = struct {
16 shared_state: State,16 shared_state: State,
17 writer_queue: Queue,17 writer_queue: Queue,
18 reader_queue: Queue,18 reader_queue: Queue,
19 writer_queue_empty_bit: u8, // TODO make this a bool19 writer_queue_empty: bool,
20 reader_queue_empty_bit: u8, // TODO make this a bool20 reader_queue_empty: bool,
21 reader_lock_count: usize,21 reader_lock_count: usize,
2222
23 const State = enum(u8) {23 const State = enum(u8) {
...@@ -40,7 +40,7 @@ pub const RwLock = struct {...@@ -40,7 +40,7 @@ pub const RwLock = struct {
40 return;40 return;
41 }41 }
4242
43 @atomicStore(u8, &self.lock.reader_queue_empty_bit, 1, .SeqCst);43 @atomicStore(bool, &self.lock.reader_queue_empty, true, .SeqCst);
44 if (@cmpxchgStrong(State, &self.lock.shared_state, .ReadLock, .Unlocked, .SeqCst, .SeqCst) != null) {44 if (@cmpxchgStrong(State, &self.lock.shared_state, .ReadLock, .Unlocked, .SeqCst, .SeqCst) != null) {
45 // Didn't unlock. Someone else's problem.45 // Didn't unlock. Someone else's problem.
46 return;46 return;
...@@ -62,7 +62,7 @@ pub const RwLock = struct {...@@ -62,7 +62,7 @@ pub const RwLock = struct {
62 }62 }
6363
64 // We need to release the write lock. Check if any readers are waiting to grab the lock.64 // We need to release the write lock. Check if any readers are waiting to grab the lock.
65 if (@atomicLoad(u8, &self.lock.reader_queue_empty_bit, .SeqCst) == 0) {65 if (!@atomicLoad(bool, &self.lock.reader_queue_empty, .SeqCst)) {
66 // Switch to a read lock.66 // Switch to a read lock.
67 @atomicStore(State, &self.lock.shared_state, .ReadLock, .SeqCst);67 @atomicStore(State, &self.lock.shared_state, .ReadLock, .SeqCst);
68 while (self.lock.reader_queue.get()) |node| {68 while (self.lock.reader_queue.get()) |node| {
...@@ -71,7 +71,7 @@ pub const RwLock = struct {...@@ -71,7 +71,7 @@ pub const RwLock = struct {
71 return;71 return;
72 }72 }
7373
74 @atomicStore(u8, &self.lock.writer_queue_empty_bit, 1, .SeqCst);74 @atomicStore(bool, &self.lock.writer_queue_empty, true, .SeqCst);
75 @atomicStore(State, &self.lock.shared_state, .Unlocked, .SeqCst);75 @atomicStore(State, &self.lock.shared_state, .Unlocked, .SeqCst);
7676
77 self.lock.commonPostUnlock();77 self.lock.commonPostUnlock();
...@@ -79,12 +79,12 @@ pub const RwLock = struct {...@@ -79,12 +79,12 @@ pub const RwLock = struct {
79 };79 };
8080
81 pub fn init() RwLock {81 pub fn init() RwLock {
82 return RwLock{82 return .{
83 .shared_state = .Unlocked,83 .shared_state = .Unlocked,
84 .writer_queue = Queue.init(),84 .writer_queue = Queue.init(),
85 .writer_queue_empty_bit = 1,85 .writer_queue_empty = true,
86 .reader_queue = Queue.init(),86 .reader_queue = Queue.init(),
87 .reader_queue_empty_bit = 1,87 .reader_queue_empty = true,
88 .reader_lock_count = 0,88 .reader_lock_count = 0,
89 };89 };
90 }90 }
...@@ -111,9 +111,9 @@ pub const RwLock = struct {...@@ -111,9 +111,9 @@ pub const RwLock = struct {
111111
112 // At this point, we are in the reader_queue, so we might have already been resumed.112 // At this point, we are in the reader_queue, so we might have already been resumed.
113113
114 // We set this bit so that later we can rely on the fact, that if reader_queue_empty_bit is 1,114 // We set this bit so that later we can rely on the fact, that if reader_queue_empty == true,
115 // some actor will attempt to grab the lock.115 // some actor will attempt to grab the lock.
116 @atomicStore(u8, &self.reader_queue_empty_bit, 0, .SeqCst);116 @atomicStore(bool, &self.reader_queue_empty, false, .SeqCst);
117117
118 // Here we don't care if we are the one to do the locking or if it was already locked for reading.118 // Here we don't care if we are the one to do the locking or if it was already locked for reading.
119 const have_read_lock = if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .ReadLock, .SeqCst, .SeqCst)) |old_state| old_state == .ReadLock else true;119 const have_read_lock = if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .ReadLock, .SeqCst, .SeqCst)) |old_state| old_state == .ReadLock else true;
...@@ -142,9 +142,9 @@ pub const RwLock = struct {...@@ -142,9 +142,9 @@ pub const RwLock = struct {
142142
143 // At this point, we are in the writer_queue, so we might have already been resumed.143 // At this point, we are in the writer_queue, so we might have already been resumed.
144144
145 // We set this bit so that later we can rely on the fact, that if writer_queue_empty_bit is 1,145 // We set this bit so that later we can rely on the fact, that if writer_queue_empty == true,
146 // some actor will attempt to grab the lock.146 // some actor will attempt to grab the lock.
147 @atomicStore(u8, &self.writer_queue_empty_bit, 0, .SeqCst);147 @atomicStore(bool, &self.writer_queue_empty, false, .SeqCst);
148148
149 // Here we must be the one to acquire the write lock. It cannot already be locked.149 // Here we must be the one to acquire the write lock. It cannot already be locked.
150 if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .WriteLock, .SeqCst, .SeqCst) == null) {150 if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .WriteLock, .SeqCst, .SeqCst) == null) {
...@@ -165,7 +165,7 @@ pub const RwLock = struct {...@@ -165,7 +165,7 @@ pub const RwLock = struct {
165 // obtain the lock.165 // obtain the lock.
166 // But if there's a writer_queue item or a reader_queue item,166 // But if there's a writer_queue item or a reader_queue item,
167 // we are the actor which must loop and attempt to grab the lock again.167 // we are the actor which must loop and attempt to grab the lock again.
168 if (@atomicLoad(u8, &self.writer_queue_empty_bit, .SeqCst) == 0) {168 if (!@atomicLoad(bool, &self.writer_queue_empty, .SeqCst)) {
169 if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .WriteLock, .SeqCst, .SeqCst) != null) {169 if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .WriteLock, .SeqCst, .SeqCst) != null) {
170 // We did not obtain the lock. Great, the queues are someone else's problem.170 // We did not obtain the lock. Great, the queues are someone else's problem.
171 return;171 return;
...@@ -176,12 +176,12 @@ pub const RwLock = struct {...@@ -176,12 +176,12 @@ pub const RwLock = struct {
176 return;176 return;
177 }177 }
178 // Release the lock again.178 // Release the lock again.
179 @atomicStore(u8, &self.writer_queue_empty_bit, 1, .SeqCst);179 @atomicStore(bool, &self.writer_queue_empty, true, .SeqCst);
180 @atomicStore(State, &self.shared_state, .Unlocked, .SeqCst);180 @atomicStore(State, &self.shared_state, .Unlocked, .SeqCst);
181 continue;181 continue;
182 }182 }
183183
184 if (@atomicLoad(u8, &self.reader_queue_empty_bit, .SeqCst) == 0) {184 if (!@atomicLoad(bool, &self.reader_queue_empty, .SeqCst)) {
185 if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .ReadLock, .SeqCst, .SeqCst) != null) {185 if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .ReadLock, .SeqCst, .SeqCst) != null) {
186 // We did not obtain the lock. Great, the queues are someone else's problem.186 // We did not obtain the lock. Great, the queues are someone else's problem.
187 return;187 return;
...@@ -195,7 +195,7 @@ pub const RwLock = struct {...@@ -195,7 +195,7 @@ pub const RwLock = struct {
195 return;195 return;
196 }196 }
197 // Release the lock again.197 // Release the lock again.
198 @atomicStore(u8, &self.reader_queue_empty_bit, 1, .SeqCst);198 @atomicStore(bool, &self.reader_queue_empty, true, .SeqCst);
199 if (@cmpxchgStrong(State, &self.shared_state, .ReadLock, .Unlocked, .SeqCst, .SeqCst) != null) {199 if (@cmpxchgStrong(State, &self.shared_state, .ReadLock, .Unlocked, .SeqCst, .SeqCst) != null) {
200 // Didn't unlock. Someone else's problem.200 // Didn't unlock. Someone else's problem.
201 return;201 return;
src/codegen.cpp+89
...@@ -5251,11 +5251,55 @@ static enum ZigLLVM_AtomicRMWBinOp to_ZigLLVMAtomicRMWBinOp(AtomicRmwOp op, bool...@@ -5251,11 +5251,55 @@ static enum ZigLLVM_AtomicRMWBinOp to_ZigLLVMAtomicRMWBinOp(AtomicRmwOp op, bool
5251 zig_unreachable();5251 zig_unreachable();
5252}5252}
52535253
5254static LLVMTypeRef get_atomic_abi_type(CodeGen *g, IrInstGen *instruction) {
5255 // If the operand type of an atomic operation is not a power of two sized
5256 // we need to widen it before using it and then truncate the result.
5257
5258 ir_assert(instruction->value->type->id == ZigTypeIdPointer, instruction);
5259 ZigType *operand_type = instruction->value->type->data.pointer.child_type;
5260 if (operand_type->id == ZigTypeIdInt || operand_type->id == ZigTypeIdEnum) {
5261 if (operand_type->id == ZigTypeIdEnum) {
5262 operand_type = operand_type->data.enumeration.tag_int_type;
5263 }
5264 auto bit_count = operand_type->data.integral.bit_count;
5265 bool is_signed = operand_type->data.integral.is_signed;
5266
5267 ir_assert(bit_count != 0, instruction);
5268 if (bit_count == 1 || !is_power_of_2(bit_count)) {
5269 return get_llvm_type(g, get_int_type(g, is_signed, operand_type->abi_size * 8));
5270 } else {
5271 return nullptr;
5272 }
5273 } else if (operand_type->id == ZigTypeIdFloat) {
5274 return nullptr;
5275 } else if (operand_type->id == ZigTypeIdBool) {
5276 return g->builtin_types.entry_u8->llvm_type;
5277 } else {
5278 ir_assert(get_codegen_ptr_type_bail(g, operand_type) != nullptr, instruction);
5279 return nullptr;
5280 }
5281}
5282
5254static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, IrInstGenCmpxchg *instruction) {5283static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, IrInstGenCmpxchg *instruction) {
5255 LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr);5284 LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr);
5256 LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value);5285 LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value);
5257 LLVMValueRef new_val = ir_llvm_value(g, instruction->new_value);5286 LLVMValueRef new_val = ir_llvm_value(g, instruction->new_value);
52585287
5288 ZigType *operand_type = instruction->new_value->value->type;
5289 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
5290 if (actual_abi_type != nullptr) {
5291 // operand needs widening and truncating
5292 ptr_val = LLVMBuildBitCast(g->builder, ptr_val,
5293 LLVMPointerType(actual_abi_type, 0), "");
5294 if (operand_type->data.integral.is_signed) {
5295 cmp_val = LLVMBuildSExt(g->builder, cmp_val, actual_abi_type, "");
5296 new_val = LLVMBuildSExt(g->builder, new_val, actual_abi_type, "");
5297 } else {
5298 cmp_val = LLVMBuildZExt(g->builder, cmp_val, actual_abi_type, "");
5299 new_val = LLVMBuildZExt(g->builder, new_val, actual_abi_type, "");
5300 }
5301 }
5302
5259 LLVMAtomicOrdering success_order = to_LLVMAtomicOrdering(instruction->success_order);5303 LLVMAtomicOrdering success_order = to_LLVMAtomicOrdering(instruction->success_order);
5260 LLVMAtomicOrdering failure_order = to_LLVMAtomicOrdering(instruction->failure_order);5304 LLVMAtomicOrdering failure_order = to_LLVMAtomicOrdering(instruction->failure_order);
52615305
...@@ -5268,6 +5312,9 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I...@@ -5268,6 +5312,9 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I
52685312
5269 if (!handle_is_ptr(g, optional_type)) {5313 if (!handle_is_ptr(g, optional_type)) {
5270 LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");5314 LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
5315 if (actual_abi_type != nullptr) {
5316 payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), "");
5317 }
5271 LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");5318 LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");
5272 return LLVMBuildSelect(g->builder, success_bit, LLVMConstNull(get_llvm_type(g, child_type)), payload_val, "");5319 return LLVMBuildSelect(g->builder, success_bit, LLVMConstNull(get_llvm_type(g, child_type)), payload_val, "");
5273 }5320 }
...@@ -5282,6 +5329,9 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I...@@ -5282,6 +5329,9 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I
5282 ir_assert(type_has_bits(g, child_type), &instruction->base);5329 ir_assert(type_has_bits(g, child_type), &instruction->base);
52835330
5284 LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");5331 LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
5332 if (actual_abi_type != nullptr) {
5333 payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), "");
5334 }
5285 LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");5335 LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");
5286 gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);5336 gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);
52875337
...@@ -5859,6 +5909,22 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutableGen *executable...@@ -5859,6 +5909,22 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutableGen *executable
5859 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);5909 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
5860 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);5910 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
58615911
5912 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
5913 if (actual_abi_type != nullptr) {
5914 // operand needs widening and truncating
5915 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
5916 LLVMPointerType(actual_abi_type, 0), "");
5917 LLVMValueRef casted_operand;
5918 if (operand_type->data.integral.is_signed) {
5919 casted_operand = LLVMBuildSExt(g->builder, operand, actual_abi_type, "");
5920 } else {
5921 casted_operand = LLVMBuildZExt(g->builder, operand, actual_abi_type, "");
5922 }
5923 LLVMValueRef uncasted_result = ZigLLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
5924 g->is_single_threaded);
5925 return LLVMBuildTrunc(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
5926 }
5927
5862 if (get_codegen_ptr_type_bail(g, operand_type) == nullptr) {5928 if (get_codegen_ptr_type_bail(g, operand_type) == nullptr) {
5863 return ZigLLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);5929 return ZigLLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
5864 }5930 }
...@@ -5877,6 +5943,17 @@ static LLVMValueRef ir_render_atomic_load(CodeGen *g, IrExecutableGen *executabl...@@ -5877,6 +5943,17 @@ static LLVMValueRef ir_render_atomic_load(CodeGen *g, IrExecutableGen *executabl
5877{5943{
5878 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);5944 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
5879 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);5945 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
5946
5947 ZigType *operand_type = instruction->ptr->value->type->data.pointer.child_type;
5948 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
5949 if (actual_abi_type != nullptr) {
5950 // operand needs widening and truncating
5951 ptr = LLVMBuildBitCast(g->builder, ptr,
5952 LLVMPointerType(actual_abi_type, 0), "");
5953 LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");
5954 LLVMSetOrdering(load_inst, ordering);
5955 return LLVMBuildTrunc(g->builder, load_inst, get_llvm_type(g, operand_type), "");
5956 }
5880 LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");5957 LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");
5881 LLVMSetOrdering(load_inst, ordering);5958 LLVMSetOrdering(load_inst, ordering);
5882 return load_inst;5959 return load_inst;
...@@ -5888,6 +5965,18 @@ static LLVMValueRef ir_render_atomic_store(CodeGen *g, IrExecutableGen *executab...@@ -5888,6 +5965,18 @@ static LLVMValueRef ir_render_atomic_store(CodeGen *g, IrExecutableGen *executab
5888 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);5965 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
5889 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);5966 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
5890 LLVMValueRef value = ir_llvm_value(g, instruction->value);5967 LLVMValueRef value = ir_llvm_value(g, instruction->value);
5968
5969 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
5970 if (actual_abi_type != nullptr) {
5971 // operand needs widening
5972 ptr = LLVMBuildBitCast(g->builder, ptr,
5973 LLVMPointerType(actual_abi_type, 0), "");
5974 if (instruction->value->value->type->data.integral.is_signed) {
5975 value = LLVMBuildSExt(g->builder, value, actual_abi_type, "");
5976 } else {
5977 value = LLVMBuildZExt(g->builder, value, actual_abi_type, "");
5978 }
5979 }
5891 LLVMValueRef store_inst = gen_store(g, value, ptr, instruction->ptr->value->type);5980 LLVMValueRef store_inst = gen_store(g, value, ptr, instruction->ptr->value->type);
5892 LLVMSetOrdering(store_inst, ordering);5981 LLVMSetOrdering(store_inst, ordering);
5893 return nullptr;5982 return nullptr;
src/ir.cpp+163-43
...@@ -25208,12 +25208,50 @@ static IrInstGen *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstSrcCmpxch...@@ -25208,12 +25208,50 @@ static IrInstGen *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstSrcCmpxch
25208 return ira->codegen->invalid_inst_gen;25208 return ira->codegen->invalid_inst_gen;
25209 }25209 }
2521025210
25211 ZigType *result_type = get_optional_type(ira->codegen, operand_type);
25212
25213 // special case zero bit types
25214 switch (type_has_one_possible_value(ira->codegen, operand_type)) {
25215 case OnePossibleValueInvalid:
25216 return ira->codegen->invalid_inst_gen;
25217 case OnePossibleValueYes: {
25218 IrInstGen *result = ir_const(ira, &instruction->base.base, result_type);
25219 set_optional_value_to_null(result->value);
25220 return result;
25221 }
25222 case OnePossibleValueNo:
25223 break;
25224 }
25225
25211 if (instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut != ConstPtrMutRuntimeVar &&25226 if (instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut != ConstPtrMutRuntimeVar &&
25212 instr_is_comptime(casted_cmp_value) && instr_is_comptime(casted_new_value)) {25227 instr_is_comptime(casted_cmp_value) && instr_is_comptime(casted_new_value)) {
25213 zig_panic("TODO compile-time execution of cmpxchg");25228 ZigValue *ptr_val = ir_resolve_const(ira, casted_ptr, UndefBad);
25229 if (ptr_val == nullptr)
25230 return ira->codegen->invalid_inst_gen;
25231
25232 ZigValue *stored_val = const_ptr_pointee(ira, ira->codegen, ptr_val, instruction->base.base.source_node);
25233 if (stored_val == nullptr)
25234 return ira->codegen->invalid_inst_gen;
25235
25236 ZigValue *expected_val = ir_resolve_const(ira, casted_cmp_value, UndefBad);
25237 if (expected_val == nullptr)
25238 return ira->codegen->invalid_inst_gen;
25239
25240 ZigValue *new_val = ir_resolve_const(ira, casted_new_value, UndefBad);
25241 if (new_val == nullptr)
25242 return ira->codegen->invalid_inst_gen;
25243
25244 bool eql = const_values_equal(ira->codegen, stored_val, expected_val);
25245 IrInstGen *result = ir_const(ira, &instruction->base.base, result_type);
25246 if (eql) {
25247 copy_const_val(ira->codegen, stored_val, new_val);
25248 set_optional_value_to_null(result->value);
25249 } else {
25250 set_optional_payload(result->value, stored_val);
25251 }
25252 return result;
25214 }25253 }
2521525254
25216 ZigType *result_type = get_optional_type(ira->codegen, operand_type);
25217 IrInstGen *result_loc;25255 IrInstGen *result_loc;
25218 if (handle_is_ptr(ira->codegen, result_type)) {25256 if (handle_is_ptr(ira->codegen, result_type)) {
25219 result_loc = ir_resolve_result(ira, &instruction->base.base, instruction->result_loc,25257 result_loc = ir_resolve_result(ira, &instruction->base.base, instruction->result_loc,
...@@ -28324,43 +28362,20 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op) {...@@ -28324,43 +28362,20 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op) {
28324 if (type_is_invalid(operand_type))28362 if (type_is_invalid(operand_type))
28325 return ira->codegen->builtin_types.entry_invalid;28363 return ira->codegen->builtin_types.entry_invalid;
2832628364
28327 if (operand_type->id == ZigTypeIdInt) {28365 if (operand_type->id == ZigTypeIdInt || operand_type->id == ZigTypeIdEnum) {
28328 if (operand_type->data.integral.bit_count < 8) {28366 ZigType *int_type;
28329 ir_add_error(ira, &op->base,28367 if (operand_type->id == ZigTypeIdEnum) {
28330 buf_sprintf("expected integer type 8 bits or larger, found %" PRIu32 "-bit integer type",28368 int_type = operand_type->data.enumeration.tag_int_type;
28331 operand_type->data.integral.bit_count));28369 } else {
28332 return ira->codegen->builtin_types.entry_invalid;28370 int_type = operand_type;
28333 }28371 }
28372 auto bit_count = int_type->data.integral.bit_count;
28334 uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);28373 uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);
28335 if (operand_type->data.integral.bit_count > max_atomic_bits) {28374
28375 if (bit_count > max_atomic_bits) {
28336 ir_add_error(ira, &op->base,28376 ir_add_error(ira, &op->base,
28337 buf_sprintf("expected %" PRIu32 "-bit integer type or smaller, found %" PRIu32 "-bit integer type",28377 buf_sprintf("expected %" PRIu32 "-bit integer type or smaller, found %" PRIu32 "-bit integer type",
28338 max_atomic_bits, operand_type->data.integral.bit_count));28378 max_atomic_bits, bit_count));
28339 return ira->codegen->builtin_types.entry_invalid;
28340 }
28341 if (!is_power_of_2(operand_type->data.integral.bit_count)) {
28342 ir_add_error(ira, &op->base,
28343 buf_sprintf("%" PRIu32 "-bit integer type is not a power of 2", operand_type->data.integral.bit_count));
28344 return ira->codegen->builtin_types.entry_invalid;
28345 }
28346 } else if (operand_type->id == ZigTypeIdEnum) {
28347 ZigType *int_type = operand_type->data.enumeration.tag_int_type;
28348 if (int_type->data.integral.bit_count < 8) {
28349 ir_add_error(ira, &op->base,
28350 buf_sprintf("expected enum tag type 8 bits or larger, found %" PRIu32 "-bit tag type",
28351 int_type->data.integral.bit_count));
28352 return ira->codegen->builtin_types.entry_invalid;
28353 }
28354 uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);
28355 if (int_type->data.integral.bit_count > max_atomic_bits) {
28356 ir_add_error(ira, &op->base,
28357 buf_sprintf("expected %" PRIu32 "-bit enum tag type or smaller, found %" PRIu32 "-bit tag type",
28358 max_atomic_bits, int_type->data.integral.bit_count));
28359 return ira->codegen->builtin_types.entry_invalid;
28360 }
28361 if (!is_power_of_2(int_type->data.integral.bit_count)) {
28362 ir_add_error(ira, &op->base,
28363 buf_sprintf("%" PRIu32 "-bit enum tag type is not a power of 2", int_type->data.integral.bit_count));
28364 return ira->codegen->builtin_types.entry_invalid;28379 return ira->codegen->builtin_types.entry_invalid;
28365 }28380 }
28366 } else if (operand_type->id == ZigTypeIdFloat) {28381 } else if (operand_type->id == ZigTypeIdFloat) {
...@@ -28371,6 +28386,8 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op) {...@@ -28371,6 +28386,8 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op) {
28371 max_atomic_bits, (uint32_t) operand_type->data.floating.bit_count));28386 max_atomic_bits, (uint32_t) operand_type->data.floating.bit_count));
28372 return ira->codegen->builtin_types.entry_invalid;28387 return ira->codegen->builtin_types.entry_invalid;
28373 }28388 }
28389 } else if (operand_type->id == ZigTypeIdBool) {
28390 // will be treated as u8
28374 } else {28391 } else {
28375 Error err;28392 Error err;
28376 ZigType *operand_ptr_type;28393 ZigType *operand_ptr_type;
...@@ -28409,11 +28426,15 @@ static IrInstGen *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstSrcAto...@@ -28409,11 +28426,15 @@ static IrInstGen *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstSrcAto
2840928426
28410 if (operand_type->id == ZigTypeIdEnum && op != AtomicRmwOp_xchg) {28427 if (operand_type->id == ZigTypeIdEnum && op != AtomicRmwOp_xchg) {
28411 ir_add_error(ira, &instruction->op->base,28428 ir_add_error(ira, &instruction->op->base,
28412 buf_sprintf("@atomicRmw on enum only works with .Xchg"));28429 buf_sprintf("@atomicRmw with enum only allowed with .Xchg"));
28430 return ira->codegen->invalid_inst_gen;
28431 } else if (operand_type->id == ZigTypeIdBool && op != AtomicRmwOp_xchg) {
28432 ir_add_error(ira, &instruction->op->base,
28433 buf_sprintf("@atomicRmw with bool only allowed with .Xchg"));
28413 return ira->codegen->invalid_inst_gen;28434 return ira->codegen->invalid_inst_gen;
28414 } else if (operand_type->id == ZigTypeIdFloat && op > AtomicRmwOp_sub) {28435 } else if (operand_type->id == ZigTypeIdFloat && op > AtomicRmwOp_sub) {
28415 ir_add_error(ira, &instruction->op->base,28436 ir_add_error(ira, &instruction->op->base,
28416 buf_sprintf("@atomicRmw with float only works with .Xchg, .Add and .Sub"));28437 buf_sprintf("@atomicRmw with float only allowed with .Xchg, .Add and .Sub"));
28417 return ira->codegen->invalid_inst_gen;28438 return ira->codegen->invalid_inst_gen;
28418 }28439 }
2841928440
...@@ -28434,14 +28455,103 @@ static IrInstGen *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstSrcAto...@@ -28434,14 +28455,103 @@ static IrInstGen *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstSrcAto
28434 return ira->codegen->invalid_inst_gen;28455 return ira->codegen->invalid_inst_gen;
28435 }28456 }
2843628457
28437 if (instr_is_comptime(casted_operand) && instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut == ConstPtrMutComptimeVar)28458 // special case zero bit types
28438 {28459 switch (type_has_one_possible_value(ira->codegen, operand_type)) {
28439 ir_add_error(ira, &instruction->base.base,28460 case OnePossibleValueInvalid:
28440 buf_sprintf("compiler bug: TODO compile-time execution of @atomicRmw"));28461 return ira->codegen->invalid_inst_gen;
28441 return ira->codegen->invalid_inst_gen;28462 case OnePossibleValueYes:
28463 return ir_const_move(ira, &instruction->base.base, get_the_one_possible_value(ira->codegen, operand_type));
28464 case OnePossibleValueNo:
28465 break;
28466 }
28467
28468 IrInst *source_inst = &instruction->base.base;
28469 if (instr_is_comptime(casted_operand) && instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut == ConstPtrMutComptimeVar) {
28470 ZigValue *ptr_val = ir_resolve_const(ira, casted_ptr, UndefBad);
28471 if (ptr_val == nullptr)
28472 return ira->codegen->invalid_inst_gen;
28473
28474 ZigValue *op1_val = const_ptr_pointee(ira, ira->codegen, ptr_val, instruction->base.base.source_node);
28475 if (op1_val == nullptr)
28476 return ira->codegen->invalid_inst_gen;
28477
28478 ZigValue *op2_val = ir_resolve_const(ira, casted_operand, UndefBad);
28479 if (op2_val == nullptr)
28480 return ira->codegen->invalid_inst_gen;
28481
28482 IrInstGen *result = ir_const(ira, source_inst, operand_type);
28483 copy_const_val(ira->codegen, result->value, op1_val);
28484 if (op == AtomicRmwOp_xchg) {
28485 copy_const_val(ira->codegen, op1_val, op2_val);
28486 return result;
28487 }
28488
28489 if (operand_type->id == ZigTypeIdPointer || operand_type->id == ZigTypeIdOptional) {
28490 ir_add_error(ira, &instruction->ordering->base,
28491 buf_sprintf("TODO comptime @atomicRmw with pointers other than .Xchg"));
28492 return ira->codegen->invalid_inst_gen;
28493 }
28494
28495 ErrorMsg *msg;
28496 if (op == AtomicRmwOp_min || op == AtomicRmwOp_max) {
28497 IrBinOp bin_op;
28498 if (op == AtomicRmwOp_min)
28499 // store op2 if op2 < op1
28500 bin_op = IrBinOpCmpGreaterThan;
28501 else
28502 // store op2 if op2 > op1
28503 bin_op = IrBinOpCmpLessThan;
28504
28505 IrInstGen *dummy_value = ir_const(ira, source_inst, operand_type);
28506 msg = ir_eval_bin_op_cmp_scalar(ira, source_inst, op1_val, bin_op, op2_val, dummy_value->value);
28507 if (msg != nullptr) {
28508 return ira->codegen->invalid_inst_gen;
28509 }
28510 if (dummy_value->value->data.x_bool)
28511 copy_const_val(ira->codegen, op1_val, op2_val);
28512 } else {
28513 IrBinOp bin_op;
28514 switch (op) {
28515 case AtomicRmwOp_xchg:
28516 case AtomicRmwOp_max:
28517 case AtomicRmwOp_min:
28518 zig_unreachable();
28519 case AtomicRmwOp_add:
28520 if (operand_type->id == ZigTypeIdFloat)
28521 bin_op = IrBinOpAdd;
28522 else
28523 bin_op = IrBinOpAddWrap;
28524 break;
28525 case AtomicRmwOp_sub:
28526 if (operand_type->id == ZigTypeIdFloat)
28527 bin_op = IrBinOpSub;
28528 else
28529 bin_op = IrBinOpSubWrap;
28530 break;
28531 case AtomicRmwOp_and:
28532 case AtomicRmwOp_nand:
28533 bin_op = IrBinOpBinAnd;
28534 break;
28535 case AtomicRmwOp_or:
28536 bin_op = IrBinOpBinOr;
28537 break;
28538 case AtomicRmwOp_xor:
28539 bin_op = IrBinOpBinXor;
28540 break;
28541 }
28542 msg = ir_eval_math_op_scalar(ira, source_inst, operand_type, op1_val, bin_op, op2_val, op1_val);
28543 if (msg != nullptr) {
28544 return ira->codegen->invalid_inst_gen;
28545 }
28546 if (op == AtomicRmwOp_nand) {
28547 bigint_not(&op1_val->data.x_bigint, &op1_val->data.x_bigint,
28548 operand_type->data.integral.bit_count, operand_type->data.integral.is_signed);
28549 }
28550 }
28551 return result;
28442 }28552 }
2844328553
28444 return ir_build_atomic_rmw_gen(ira, &instruction->base.base, casted_ptr, casted_operand, op,28554 return ir_build_atomic_rmw_gen(ira, source_inst, casted_ptr, casted_operand, op,
28445 ordering, operand_type);28555 ordering, operand_type);
28446}28556}
2844728557
...@@ -28513,6 +28623,16 @@ static IrInstGen *ir_analyze_instruction_atomic_store(IrAnalyze *ira, IrInstSrcA...@@ -28513,6 +28623,16 @@ static IrInstGen *ir_analyze_instruction_atomic_store(IrAnalyze *ira, IrInstSrcA
28513 return ira->codegen->invalid_inst_gen;28623 return ira->codegen->invalid_inst_gen;
28514 }28624 }
2851528625
28626 // special case zero bit types
28627 switch (type_has_one_possible_value(ira->codegen, operand_type)) {
28628 case OnePossibleValueInvalid:
28629 return ira->codegen->invalid_inst_gen;
28630 case OnePossibleValueYes:
28631 return ir_const_void(ira, &instruction->base.base);
28632 case OnePossibleValueNo:
28633 break;
28634 }
28635
28516 if (instr_is_comptime(casted_value) && instr_is_comptime(casted_ptr)) {28636 if (instr_is_comptime(casted_value) && instr_is_comptime(casted_ptr)) {
28517 IrInstGen *result = ir_analyze_store_ptr(ira, &instruction->base.base, casted_ptr, value, false);28637 IrInstGen *result = ir_analyze_store_ptr(ira, &instruction->base.base, casted_ptr, value, false);
28518 result->value->type = ira->codegen->builtin_types.entry_void;28638 result->value->type = ira->codegen->builtin_types.entry_void;
test/compile_errors.zig+11-2
...@@ -49,6 +49,15 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -49,6 +49,15 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
49 "tmp.zig:5:9: error: resume in noasync scope",49 "tmp.zig:5:9: error: resume in noasync scope",
50 });50 });
5151
52 cases.add("atomicrmw with bool op not .Xchg",
53 \\export fn entry() void {
54 \\ var x = false;
55 \\ _ = @atomicRmw(bool, &x, .Add, true, .SeqCst);
56 \\}
57 , &[_][]const u8{
58 "tmp.zig:3:30: error: @atomicRmw with bool only allowed with .Xchg",
59 });
60
52 cases.addTest("@TypeOf with no arguments",61 cases.addTest("@TypeOf with no arguments",
53 \\export fn entry() void {62 \\export fn entry() void {
54 \\ _ = @TypeOf();63 \\ _ = @TypeOf();
...@@ -357,7 +366,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -357,7 +366,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
357 \\ _ = @atomicRmw(f32, &x, .And, 2, .SeqCst);366 \\ _ = @atomicRmw(f32, &x, .And, 2, .SeqCst);
358 \\}367 \\}
359 , &[_][]const u8{368 , &[_][]const u8{
360 "tmp.zig:3:29: error: @atomicRmw with float only works with .Xchg, .Add and .Sub",369 "tmp.zig:3:29: error: @atomicRmw with float only allowed with .Xchg, .Add and .Sub",
361 });370 });
362371
363 cases.add("intToPtr with misaligned address",372 cases.add("intToPtr with misaligned address",
...@@ -574,7 +583,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -574,7 +583,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
574 \\ _ = @atomicRmw(E, &x, .Add, .b, .SeqCst);583 \\ _ = @atomicRmw(E, &x, .Add, .b, .SeqCst);
575 \\}584 \\}
576 , &[_][]const u8{585 , &[_][]const u8{
577 "tmp.zig:9:27: error: @atomicRmw on enum only works with .Xchg",586 "tmp.zig:9:27: error: @atomicRmw with enum only allowed with .Xchg",
578 });587 });
579588
580 cases.add("disallow coercion from non-null-terminated pointer to null-terminated pointer",589 cases.add("disallow coercion from non-null-terminated pointer to null-terminated pointer",
test/stage1/behavior/atomics.zig+70-13
...@@ -2,29 +2,32 @@ const std = @import("std");...@@ -2,29 +2,32 @@ const std = @import("std");
2const expect = std.testing.expect;2const expect = std.testing.expect;
3const expectEqual = std.testing.expectEqual;3const expectEqual = std.testing.expectEqual;
4const builtin = @import("builtin");4const builtin = @import("builtin");
5const AtomicRmwOp = builtin.AtomicRmwOp;
6const AtomicOrder = builtin.AtomicOrder;
75
8test "cmpxchg" {6test "cmpxchg" {
7 testCmpxchg();
8 comptime testCmpxchg();
9}
10
11fn testCmpxchg() void {
9 var x: i32 = 1234;12 var x: i32 = 1234;
10 if (@cmpxchgWeak(i32, &x, 99, 5678, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) |x1| {13 if (@cmpxchgWeak(i32, &x, 99, 5678, .SeqCst, .SeqCst)) |x1| {
11 expect(x1 == 1234);14 expect(x1 == 1234);
12 } else {15 } else {
13 @panic("cmpxchg should have failed");16 @panic("cmpxchg should have failed");
14 }17 }
1518
16 while (@cmpxchgWeak(i32, &x, 1234, 5678, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) |x1| {19 while (@cmpxchgWeak(i32, &x, 1234, 5678, .SeqCst, .SeqCst)) |x1| {
17 expect(x1 == 1234);20 expect(x1 == 1234);
18 }21 }
19 expect(x == 5678);22 expect(x == 5678);
2023
21 expect(@cmpxchgStrong(i32, &x, 5678, 42, AtomicOrder.SeqCst, AtomicOrder.SeqCst) == null);24 expect(@cmpxchgStrong(i32, &x, 5678, 42, .SeqCst, .SeqCst) == null);
22 expect(x == 42);25 expect(x == 42);
23}26}
2427
25test "fence" {28test "fence" {
26 var x: i32 = 1234;29 var x: i32 = 1234;
27 @fence(AtomicOrder.SeqCst);30 @fence(.SeqCst);
28 x = 5678;31 x = 5678;
29}32}
3033
...@@ -36,18 +39,18 @@ test "atomicrmw and atomicload" {...@@ -36,18 +39,18 @@ test "atomicrmw and atomicload" {
36}39}
3740
38fn testAtomicRmw(ptr: *u8) void {41fn testAtomicRmw(ptr: *u8) void {
39 const prev_value = @atomicRmw(u8, ptr, AtomicRmwOp.Xchg, 42, AtomicOrder.SeqCst);42 const prev_value = @atomicRmw(u8, ptr, .Xchg, 42, .SeqCst);
40 expect(prev_value == 200);43 expect(prev_value == 200);
41 comptime {44 comptime {
42 var x: i32 = 1234;45 var x: i32 = 1234;
43 const y: i32 = 12345;46 const y: i32 = 12345;
44 expect(@atomicLoad(i32, &x, AtomicOrder.SeqCst) == 1234);47 expect(@atomicLoad(i32, &x, .SeqCst) == 1234);
45 expect(@atomicLoad(i32, &y, AtomicOrder.SeqCst) == 12345);48 expect(@atomicLoad(i32, &y, .SeqCst) == 12345);
46 }49 }
47}50}
4851
49fn testAtomicLoad(ptr: *u8) void {52fn testAtomicLoad(ptr: *u8) void {
50 const x = @atomicLoad(u8, ptr, AtomicOrder.SeqCst);53 const x = @atomicLoad(u8, ptr, .SeqCst);
51 expect(x == 42);54 expect(x == 42);
52}55}
5356
...@@ -56,18 +59,18 @@ test "cmpxchg with ptr" {...@@ -56,18 +59,18 @@ test "cmpxchg with ptr" {
56 var data2: i32 = 5678;59 var data2: i32 = 5678;
57 var data3: i32 = 9101;60 var data3: i32 = 9101;
58 var x: *i32 = &data1;61 var x: *i32 = &data1;
59 if (@cmpxchgWeak(*i32, &x, &data2, &data3, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) |x1| {62 if (@cmpxchgWeak(*i32, &x, &data2, &data3, .SeqCst, .SeqCst)) |x1| {
60 expect(x1 == &data1);63 expect(x1 == &data1);
61 } else {64 } else {
62 @panic("cmpxchg should have failed");65 @panic("cmpxchg should have failed");
63 }66 }
6467
65 while (@cmpxchgWeak(*i32, &x, &data1, &data3, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) |x1| {68 while (@cmpxchgWeak(*i32, &x, &data1, &data3, .SeqCst, .SeqCst)) |x1| {
66 expect(x1 == &data1);69 expect(x1 == &data1);
67 }70 }
68 expect(x == &data3);71 expect(x == &data3);
6972
70 expect(@cmpxchgStrong(*i32, &x, &data3, &data2, AtomicOrder.SeqCst, AtomicOrder.SeqCst) == null);73 expect(@cmpxchgStrong(*i32, &x, &data3, &data2, .SeqCst, .SeqCst) == null);
71 expect(x == &data2);74 expect(x == &data2);
72}75}
7376
...@@ -146,9 +149,11 @@ fn testAtomicStore() void {...@@ -146,9 +149,11 @@ fn testAtomicStore() void {
146}149}
147150
148test "atomicrmw with floats" {151test "atomicrmw with floats" {
152 // TODO https://github.com/ziglang/zig/issues/4457
149 if (builtin.arch == .aarch64 or builtin.arch == .arm or builtin.arch == .riscv64)153 if (builtin.arch == .aarch64 or builtin.arch == .arm or builtin.arch == .riscv64)
150 return error.SkipZigTest;154 return error.SkipZigTest;
151 testAtomicRmwFloat();155 testAtomicRmwFloat();
156 comptime testAtomicRmwFloat();
152}157}
153158
154fn testAtomicRmwFloat() void {159fn testAtomicRmwFloat() void {
...@@ -161,3 +166,55 @@ fn testAtomicRmwFloat() void {...@@ -161,3 +166,55 @@ fn testAtomicRmwFloat() void {
161 _ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst);166 _ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst);
162 expect(x == 4);167 expect(x == 4);
163}168}
169
170test "atomicrmw with ints" {
171 testAtomicRmwInt();
172 comptime testAtomicRmwInt();
173}
174
175fn testAtomicRmwInt() void {
176 var x: u8 = 1;
177 var res = @atomicRmw(u8, &x, .Xchg, 3, .SeqCst);
178 expect(x == 3 and res == 1);
179 _ = @atomicRmw(u8, &x, .Add, 3, .SeqCst);
180 expect(x == 6);
181 _ = @atomicRmw(u8, &x, .Sub, 1, .SeqCst);
182 expect(x == 5);
183 _ = @atomicRmw(u8, &x, .And, 4, .SeqCst);
184 expect(x == 4);
185 _ = @atomicRmw(u8, &x, .Nand, 4, .SeqCst);
186 expect(x == 0xfb);
187 _ = @atomicRmw(u8, &x, .Or, 6, .SeqCst);
188 expect(x == 0xff);
189 _ = @atomicRmw(u8, &x, .Xor, 2, .SeqCst);
190 expect(x == 0xfd);
191
192 // TODO https://github.com/ziglang/zig/issues/4724
193 if (builtin.arch == .mipsel) return;
194 _ = @atomicRmw(u8, &x, .Max, 1, .SeqCst);
195 expect(x == 0xfd);
196 _ = @atomicRmw(u8, &x, .Min, 1, .SeqCst);
197 expect(x == 1);
198}
199
200
201test "atomics with different types" {
202 testAtomicsWithType(bool, true, false);
203 inline for (.{ u1, i5, u15 }) |T| {
204 var x: T = 0;
205 testAtomicsWithType(T, 0, 1);
206 }
207 testAtomicsWithType(u0, 0, 0);
208 testAtomicsWithType(i0, 0, 0);
209}
210
211fn testAtomicsWithType(comptime T: type, a: T, b: T) void {
212 var x: T = b;
213 @atomicStore(T, &x, a, .SeqCst);
214 expect(x == a);
215 expect(@atomicLoad(T, &x, .SeqCst) == a);
216 expect(@atomicRmw(T, &x, .Xchg, b, .SeqCst) == a);
217 expect(@cmpxchgStrong(T, &x, b, a, .SeqCst, .SeqCst) == null);
218 if (@sizeOf(T) != 0)
219 expect(@cmpxchgStrong(T, &x, b, a, .SeqCst, .SeqCst).? == a);
220}