authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-19 22:51:15-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-21 16:36:30-04:00
logd98974e826791e4546a72599a1a5fb24692d5acf
tree4cc008134300a20dca6083ab523857c41cbcaaa2
parentc5cf611516ea5587dbefcf70a2f35e2b8f7f70b5

cbe: fix issues with atomic floats

Since the Zig language documentation claims support for `.Min` and `.Max` in `@atomicRmw` with floats, allow in Sema and implement for both the llvm and C backends.

7 files changed, 305 insertions(+), 144 deletions(-)

lib/zig.h+138-57
......@@ -255,42 +255,90 @@ typedef char bool;
255255
256256#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
257257#include <stdatomic.h>
258#define zig_atomic(type) _Atomic(type)
259#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
260#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail)
261#define zig_atomicrmw_xchg(obj, arg, order, type) atomic_exchange_explicit (obj, arg, order)
262#define zig_atomicrmw_add(obj, arg, order, type) atomic_fetch_add_explicit (obj, arg, order)
263#define zig_atomicrmw_sub(obj, arg, order, type) atomic_fetch_sub_explicit (obj, arg, order)
264#define zig_atomicrmw_or(obj, arg, order, type) atomic_fetch_or_explicit (obj, arg, order)
265#define zig_atomicrmw_xor(obj, arg, order, type) atomic_fetch_xor_explicit (obj, arg, order)
266#define zig_atomicrmw_and(obj, arg, order, type) atomic_fetch_and_explicit (obj, arg, order)
267#define zig_atomicrmw_nand(obj, arg, order, type) __atomic_fetch_nand (obj, arg, order)
268#define zig_atomicrmw_min(obj, arg, order, type) __atomic_fetch_min (obj, arg, order)
269#define zig_atomicrmw_max(obj, arg, order, type) __atomic_fetch_max (obj, arg, order)
270#define zig_atomic_store(obj, arg, order, type) atomic_store_explicit (obj, arg, order)
271#define zig_atomic_load(obj, order, type) atomic_load_explicit (obj, order)
258typedef enum memory_order zig_memory_order;
259#define zig_atomic(Type) _Atomic(Type)
260#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
261#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail)
262#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = atomic_exchange_explicit (obj, arg, order)
263#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = atomic_fetch_add_explicit (obj, arg, order)
264#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = atomic_fetch_sub_explicit (obj, arg, order)
265#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = atomic_fetch_or_explicit (obj, arg, order)
266#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = atomic_fetch_xor_explicit (obj, arg, order)
267#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = atomic_fetch_and_explicit (obj, arg, order)
268#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order)
269#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order)
270#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order)
271#define zig_atomic_store( obj, arg, order, Type, ReprType) atomic_store_explicit (obj, arg, order)
272#define zig_atomic_load(res, obj, order, Type, ReprType) res = atomic_load_explicit (obj, order)
273#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
274#define zig_atomicrmw_add_float zig_atomicrmw_add
275#define zig_atomicrmw_sub_float zig_atomicrmw_sub
276#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \
277 zig_##Type zig_atomicrmw_desired; \
278 zig_atomic_load(res, obj, order, Type, ReprType); \
279 do { \
280 zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(res, arg); \
281 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
282} while (0)
283#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \
284 zig_##Type zig_atomicrmw_desired; \
285 zig_atomic_load(res, obj, order, Type, ReprType); \
286 do { \
287 zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(res, arg); \
288 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
289} while (0)
272290#define zig_fence(order) atomic_thread_fence(order)
273291#elif defined(__GNUC__)
292typedef int zig_memory_order;
274293#define memory_order_relaxed __ATOMIC_RELAXED
275294#define memory_order_consume __ATOMIC_CONSUME
276295#define memory_order_acquire __ATOMIC_ACQUIRE
277296#define memory_order_release __ATOMIC_RELEASE
278297#define memory_order_acq_rel __ATOMIC_ACQ_REL
279298#define memory_order_seq_cst __ATOMIC_SEQ_CST
280#define zig_atomic(type) type
281#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) __atomic_compare_exchange_n(obj, &(expected), desired, false, succ, fail)
282#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) __atomic_compare_exchange_n(obj, &(expected), desired, true , succ, fail)
283#define zig_atomicrmw_xchg(obj, arg, order, type) __atomic_exchange_n(obj, arg, order)
284#define zig_atomicrmw_add(obj, arg, order, type) __atomic_fetch_add (obj, arg, order)
285#define zig_atomicrmw_sub(obj, arg, order, type) __atomic_fetch_sub (obj, arg, order)
286#define zig_atomicrmw_or(obj, arg, order, type) __atomic_fetch_or (obj, arg, order)
287#define zig_atomicrmw_xor(obj, arg, order, type) __atomic_fetch_xor (obj, arg, order)
288#define zig_atomicrmw_and(obj, arg, order, type) __atomic_fetch_and (obj, arg, order)
289#define zig_atomicrmw_nand(obj, arg, order, type) __atomic_fetch_nand(obj, arg, order)
290#define zig_atomicrmw_min(obj, arg, order, type) __atomic_fetch_min (obj, arg, order)
291#define zig_atomicrmw_max(obj, arg, order, type) __atomic_fetch_max (obj, arg, order)
292#define zig_atomic_store(obj, arg, order, type) __atomic_store_n (obj, arg, order)
293#define zig_atomic_load(obj, order, type) __atomic_load_n (obj, order)
299#define zig_atomic(Type) Type
300#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), false, succ, fail)
301#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), true, succ, fail)
302#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) __atomic_exchange(obj, &(arg), &(res), order)
303#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_add (obj, arg, order)
304#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_sub (obj, arg, order)
305#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_or (obj, arg, order)
306#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_xor (obj, arg, order)
307#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_and (obj, arg, order)
308#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order)
309#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order)
310#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order)
311#define zig_atomic_store( obj, arg, order, Type, ReprType) __atomic_store (obj, &(arg), order)
312#define zig_atomic_load(res, obj, order, Type, ReprType) __atomic_load (obj, &(res), order)
313#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
314#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) do { \
315 zig_##Type zig_atomicrmw_desired; \
316 zig_atomic_load(res, obj, order, Type, ReprType); \
317 do { \
318 zig_atomicrmw_desired = (res) + (arg); \
319 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
320} while (0)
321#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) do { \
322 zig_##Type zig_atomicrmw_desired; \
323 zig_atomic_load(res, obj, order, Type, ReprType); \
324 do { \
325 zig_atomicrmw_desired = (res) - (arg); \
326 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
327} while (0)
328#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \
329 zig_##Type zig_atomicrmw_desired; \
330 zig_atomic_load(res, obj, order, Type, ReprType); \
331 do { \
332 zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(res, arg); \
333 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
334} while (0)
335#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \
336 zig_##Type zig_atomicrmw_desired; \
337 zig_atomic_load(res, obj, order, Type, ReprType); \
338 do { \
339 zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(res, arg); \
340 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
341} while (0)
294342#define zig_fence(order) __atomic_thread_fence(order)
295343#elif _MSC_VER && (_M_IX86 || _M_X64)
296344#define memory_order_relaxed 0
......@@ -299,20 +347,25 @@ typedef char bool;
299347#define memory_order_release 3
300348#define memory_order_acq_rel 4
301349#define memory_order_seq_cst 5
302#define zig_atomic(type) type
303#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) zig_expand_concat(zig_msvc_cmpxchg_, type)(obj, &(expected), desired)
304#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) zig_cmpxchg_strong(obj, expected, desired, succ, fail, type)
305#define zig_atomicrmw_xchg(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_xchg_, type)(obj, arg)
306#define zig_atomicrmw_add(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_add_, type)(obj, arg)
307#define zig_atomicrmw_sub(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_sub_, type)(obj, arg)
308#define zig_atomicrmw_or(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_or_, type)(obj, arg)
309#define zig_atomicrmw_xor(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_xor_, type)(obj, arg)
310#define zig_atomicrmw_and(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_and_, type)(obj, arg)
311#define zig_atomicrmw_nand(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_nand_, type)(obj, arg)
312#define zig_atomicrmw_min(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_min_, type)(obj, arg)
313#define zig_atomicrmw_max(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_max_, type)(obj, arg)
314#define zig_atomic_store(obj, arg, order, type) zig_expand_concat(zig_msvc_atomic_store_, type)(obj, arg)
315#define zig_atomic_load(obj, order, type) zig_expand_concat(zig_msvc_atomic_load_, type)(obj)
350#define zig_atomic(Type) Type
351#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) res = zig_msvc_cmpxchg_##Type(obj, &(expected), desired)
352#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_cmpxchg_strong(res, obj, expected, desired, succ, fail, Type)
353#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xchg_##Type(obj, arg)
354#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_add_ ##Type(obj, arg)
355#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_sub_ ##Type(obj, arg)
356#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_or_ ##Type(obj, arg)
357#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xor_ ##Type(obj, arg)
358#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_and_ ##Type(obj, arg)
359#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_nand_##Type(obj, arg)
360#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_min_ ##Type(obj, arg)
361#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_max_ ##Type(obj, arg)
362#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_msvc_atomic_store_ ##Type(obj, arg)
363#define zig_atomic_load(res, obj, order, Type, ReprType) res = zig_msvc_atomic_load_ ##Type(obj)
364#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
365#define zig_atomicrmw_add_float zig_atomicrmw_add
366#define zig_atomicrmw_sub_float zig_atomicrmw_sub
367#define zig_atomicrmw_min_float zig_atomicrmw_min
368#define zig_atomicrmw_max_float zig_atomicrmw_max
316369#if _M_X64
317370#define zig_fence(order) __faststorefence()
318371#else
......@@ -327,21 +380,25 @@ typedef char bool;
327380#define memory_order_release 3
328381#define memory_order_acq_rel 4
329382#define memory_order_seq_cst 5
330#define zig_atomic(type) type
331#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) zig_unimplemented()
332#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) zig_unimplemented()
333#define zig_atomicrmw_xchg(obj, arg, order, type) zig_unimplemented()
334#define zig_atomicrmw_add(obj, arg, order, type) zig_unimplemented()
335#define zig_atomicrmw_sub(obj, arg, order, type) zig_unimplemented()
336#define zig_atomicrmw_or(obj, arg, order, type) zig_unimplemented()
337#define zig_atomicrmw_xor(obj, arg, order, type) zig_unimplemented()
338#define zig_atomicrmw_and(obj, arg, order, type) zig_unimplemented()
339#define zig_atomicrmw_nand(obj, arg, order, type) zig_unimplemented()
340#define zig_atomicrmw_min(obj, arg, order, type) zig_unimplemented()
341#define zig_atomicrmw_max(obj, arg, order, type) zig_unimplemented()
342#define zig_atomic_store(obj, arg, order, type) zig_unimplemented()
343#define zig_atomic_load(obj, order, type) zig_unimplemented()
344#define zig_fence(order) zig_unimplemented()
383#define zig_atomic(Type) Type
384#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable
385#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable
386#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
387#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
388#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
389#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
390#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
391#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
392#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
393#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
394#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
395#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_atomics_unavailable
396#define zig_atomic_load(res, obj, order, Type, ReprType) zig_atomics_unavailable
397#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
398#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
399#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
400#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
401#define zig_fence(order) zig_fence_unavailable
345402#endif
346403
347404#if __STDC_VERSION__ >= 201112L
......@@ -3461,6 +3518,8 @@ zig_float_builtins(64)
34613518zig_float_builtins(80)
34623519zig_float_builtins(128)
34633520
3521/* ============================ Atomics Support ============================= */
3522
34643523#if _MSC_VER && (_M_IX86 || _M_X64)
34653524
34663525// TODO: zig_msvc_atomic_load should load 32 bit without interlocked on x86, and load 64 bit without interlocked on x64
......@@ -3596,6 +3655,28 @@ zig_msvc_atomics(i64, int64_t, __int64, 64)
35963655 desired = expected - value; \
35973656 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
35983657 return expected; \
3658 } \
3659 static inline zig_##Type zig_msvc_atomicrmw_min_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3660 ReprType repr; \
3661 zig_##Type expected; \
3662 zig_##Type desired; \
3663 repr = *(ReprType volatile*)obj; \
3664 memcpy(&expected, &repr, sizeof(expected)); \
3665 do { \
3666 desired = zig_libc_name_##Type(fmin)(expected, value); \
3667 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
3668 return expected; \
3669 } \
3670 static inline zig_##Type zig_msvc_atomicrmw_max_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3671 ReprType repr; \
3672 zig_##Type expected; \
3673 zig_##Type desired; \
3674 repr = *(ReprType volatile*)obj; \
3675 memcpy(&expected, &repr, sizeof(expected)); \
3676 do { \
3677 desired = zig_libc_name_##Type(fmax)(expected, value); \
3678 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
3679 return expected; \
35993680 }
36003681
36013682zig_msvc_flt_atomics(f32, long, )
src/Sema.zig+2-2
......@@ -21319,8 +21319,8 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
2131921319 return sema.fail(block, op_src, "@atomicRmw with bool only allowed with .Xchg", .{});
2132021320 },
2132121321 .Float => switch (op) {
21322 .Xchg, .Add, .Sub => {},
21323 else => return sema.fail(block, op_src, "@atomicRmw with float only allowed with .Xchg, .Add, and .Sub", .{}),
21322 .Xchg, .Add, .Sub, .Max, .Min => {},
21323 else => return sema.fail(block, op_src, "@atomicRmw with float only allowed with .Xchg, .Add, .Sub, .Max, and .Min", .{}),
2132421324 },
2132521325 else => {},
2132621326 }
src/codegen/c.zig+154-63
......@@ -45,9 +45,6 @@ pub const CValue = union(enum) {
4545 identifier: []const u8,
4646 /// Render the slice as an payload.identifier (using fmtIdent)
4747 payload_identifier: []const u8,
48 /// Render these bytes literally.
49 /// TODO make this a [*:0]const u8 to save memory
50 bytes: []const u8,
5148};
5249
5350const BlockData = struct {
......@@ -1770,7 +1767,6 @@ pub const DeclGen = struct {
17701767 fmtIdent("payload"),
17711768 fmtIdent(ident),
17721769 }),
1773 .bytes => |bytes| return w.writeAll(bytes),
17741770 }
17751771 }
17761772
......@@ -1799,11 +1795,6 @@ pub const DeclGen = struct {
17991795 fmtIdent("payload"),
18001796 fmtIdent(ident),
18011797 }),
1802 .bytes => |bytes| {
1803 try w.writeAll("(*");
1804 try w.writeAll(bytes);
1805 return w.writeByte(')');
1806 },
18071798 }
18081799 }
18091800
......@@ -1816,7 +1807,7 @@ pub const DeclGen = struct {
18161807 fn writeCValueDerefMember(dg: *DeclGen, writer: anytype, c_value: CValue, member: CValue) !void {
18171808 switch (c_value) {
18181809 .none, .constant, .field, .undef => unreachable,
1819 .new_local, .local, .arg, .arg_array, .decl, .identifier, .payload_identifier, .bytes => {
1810 .new_local, .local, .arg, .arg_array, .decl, .identifier, .payload_identifier => {
18201811 try dg.writeCValue(writer, c_value);
18211812 try writer.writeAll("->");
18221813 },
......@@ -5959,15 +5950,29 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue
59595950 try reap(f, inst, &.{ extra.ptr, extra.expected_value, extra.new_value });
59605951 const writer = f.object.writer();
59615952 const ptr_ty = f.air.typeOf(extra.ptr);
5953 const ty = ptr_ty.childType();
5954
5955 const target = f.object.dg.module.getTarget();
5956 var repr_pl = Type.Payload.Bits{
5957 .base = .{ .tag = .int_unsigned },
5958 .data = @intCast(u16, ty.abiSize(target) * 8),
5959 };
5960 const repr_ty = if (ty.isRuntimeFloat()) Type.initPayload(&repr_pl.base) else ty;
5961
5962 const new_value_mat = try Materialize.start(f, inst, writer, ty, new_value);
59625963 const local = try f.allocLocal(inst, inst_ty);
59635964 if (inst_ty.isPtrLikeOptional()) {
5964 try f.writeCValue(writer, local, .Other);
5965 try writer.writeAll(" = ");
5966 try f.writeCValue(writer, expected_value, .Initializer);
5967 try writer.writeAll(";\n");
5965 {
5966 const a = try Assignment.start(f, writer, ty);
5967 try f.writeCValue(writer, local, .Other);
5968 try a.assign(f, writer);
5969 try f.writeCValue(writer, expected_value, .Other);
5970 try a.end(f, writer);
5971 }
5972
59685973 try writer.writeAll("if (");
59695974 try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor});
5970 try f.renderType(writer, ptr_ty.childType());
5975 try f.renderType(writer, ty);
59715976 try writer.writeByte(')');
59725977 if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile");
59735978 try writer.writeAll(" *)");
......@@ -5975,45 +5980,62 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue
59755980 try writer.writeAll(", ");
59765981 try f.writeCValue(writer, local, .FunctionArgument);
59775982 try writer.writeAll(", ");
5978 try f.writeCValue(writer, new_value, .FunctionArgument);
5983 try new_value_mat.mat(f, writer);
59795984 try writer.writeAll(", ");
59805985 try writeMemoryOrder(writer, extra.successOrder());
59815986 try writer.writeAll(", ");
59825987 try writeMemoryOrder(writer, extra.failureOrder());
59835988 try writer.writeAll(", ");
5984 try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType());
5989 try f.object.dg.renderTypeForBuiltinFnName(writer, ty);
5990 try writer.writeAll(", ");
5991 try f.object.dg.renderType(writer, repr_ty);
59855992 try writer.writeByte(')');
59865993 try writer.writeAll(") {\n");
59875994 f.object.indent_writer.pushIndent();
5988 try f.writeCValue(writer, local, .Other);
5989 try writer.writeAll(" = NULL;\n");
5995 {
5996 const a = try Assignment.start(f, writer, ty);
5997 try f.writeCValue(writer, local, .Other);
5998 try a.assign(f, writer);
5999 try writer.writeAll("NULL");
6000 try a.end(f, writer);
6001 }
59906002 f.object.indent_writer.popIndent();
59916003 try writer.writeAll("}\n");
59926004 } else {
5993 try f.writeCValue(writer, local, .Other);
5994 try writer.writeAll(".payload = ");
5995 try f.writeCValue(writer, expected_value, .Other);
5996 try writer.writeAll(";\n");
5997 try f.writeCValue(writer, local, .Other);
5998 try writer.print(".is_null = zig_cmpxchg_{s}((zig_atomic(", .{flavor});
5999 try f.renderType(writer, ptr_ty.childType());
6000 try writer.writeByte(')');
6001 if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile");
6002 try writer.writeAll(" *)");
6003 try f.writeCValue(writer, ptr, .Other);
6004 try writer.writeAll(", ");
6005 try f.writeCValueMember(writer, local, .{ .identifier = "payload" });
6006 try writer.writeAll(", ");
6007 try f.writeCValue(writer, new_value, .FunctionArgument);
6008 try writer.writeAll(", ");
6009 try writeMemoryOrder(writer, extra.successOrder());
6010 try writer.writeAll(", ");
6011 try writeMemoryOrder(writer, extra.failureOrder());
6012 try writer.writeAll(", ");
6013 try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType());
6014 try writer.writeByte(')');
6015 try writer.writeAll(";\n");
6005 {
6006 const a = try Assignment.start(f, writer, ty);
6007 try f.writeCValueMember(writer, local, .{ .identifier = "payload" });
6008 try a.assign(f, writer);
6009 try f.writeCValue(writer, expected_value, .Other);
6010 try a.end(f, writer);
6011 }
6012 {
6013 const a = try Assignment.start(f, writer, Type.bool);
6014 try f.writeCValueMember(writer, local, .{ .identifier = "is_null" });
6015 try a.assign(f, writer);
6016 try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor});
6017 try f.renderType(writer, ty);
6018 try writer.writeByte(')');
6019 if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile");
6020 try writer.writeAll(" *)");
6021 try f.writeCValue(writer, ptr, .Other);
6022 try writer.writeAll(", ");
6023 try f.writeCValueMember(writer, local, .{ .identifier = "payload" });
6024 try writer.writeAll(", ");
6025 try new_value_mat.mat(f, writer);
6026 try writer.writeAll(", ");
6027 try writeMemoryOrder(writer, extra.successOrder());
6028 try writer.writeAll(", ");
6029 try writeMemoryOrder(writer, extra.failureOrder());
6030 try writer.writeAll(", ");
6031 try f.object.dg.renderTypeForBuiltinFnName(writer, ty);
6032 try writer.writeAll(", ");
6033 try f.object.dg.renderType(writer, repr_ty);
6034 try writer.writeByte(')');
6035 try a.end(f, writer);
6036 }
60166037 }
6038 try new_value_mat.end(f, inst);
60176039
60186040 if (f.liveness.isUnused(inst)) {
60196041 try freeLocal(f, inst, local.new_local, 0);
......@@ -6028,35 +6050,42 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue {
60286050 const extra = f.air.extraData(Air.AtomicRmw, pl_op.payload).data;
60296051 const inst_ty = f.air.typeOfIndex(inst);
60306052 const ptr_ty = f.air.typeOf(pl_op.operand);
6053 const ty = ptr_ty.childType();
60316054 const ptr = try f.resolveInst(pl_op.operand);
60326055 const operand = try f.resolveInst(extra.operand);
60336056 try reap(f, inst, &.{ pl_op.operand, extra.operand });
60346057 const writer = f.object.writer();
60356058 const local = try f.allocLocal(inst, inst_ty);
60366059
6060 const target = f.object.dg.module.getTarget();
6061 var repr_pl = Type.Payload.Bits{
6062 .base = .{ .tag = .int_unsigned },
6063 .data = @intCast(u16, ty.abiSize(target) * 8),
6064 };
6065 const is_float = ty.isRuntimeFloat();
6066 const repr_ty = if (is_float) Type.initPayload(&repr_pl.base) else ty;
6067
6068 const operand_mat = try Materialize.start(f, inst, writer, ty, operand);
6069 try writer.print("zig_atomicrmw_{s}", .{toAtomicRmwSuffix(extra.op())});
6070 if (is_float) try writer.writeAll("_float");
6071 try writer.writeByte('(');
60376072 try f.writeCValue(writer, local, .Other);
6038 try writer.print(" = zig_atomicrmw_{s}((", .{toAtomicRmwSuffix(extra.op())});
6039 switch (extra.op()) {
6040 else => {
6041 try writer.writeAll("zig_atomic(");
6042 try f.renderType(writer, ptr_ty.elemType());
6043 try writer.writeByte(')');
6044 },
6045 .Nand, .Min, .Max => {
6046 // These are missing from stdatomic.h, so no atomic types for now.
6047 try f.renderType(writer, ptr_ty.elemType());
6048 },
6049 }
6073 try writer.writeAll(", (zig_atomic(");
6074 try f.renderType(writer, ty);
6075 try writer.writeByte(')');
60506076 if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile");
60516077 try writer.writeAll(" *)");
60526078 try f.writeCValue(writer, ptr, .Other);
60536079 try writer.writeAll(", ");
6054 try f.writeCValue(writer, operand, .FunctionArgument);
6080 try operand_mat.mat(f, writer);
60556081 try writer.writeAll(", ");
60566082 try writeMemoryOrder(writer, extra.ordering());
60576083 try writer.writeAll(", ");
6058 try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType());
6084 try f.object.dg.renderTypeForBuiltinFnName(writer, ty);
6085 try writer.writeAll(", ");
6086 try f.object.dg.renderType(writer, repr_ty);
60596087 try writer.writeAll(");\n");
6088 try operand_mat.end(f, inst);
60606089
60616090 if (f.liveness.isUnused(inst)) {
60626091 try freeLocal(f, inst, local.new_local, 0);
......@@ -6071,14 +6100,23 @@ fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue {
60716100 const ptr = try f.resolveInst(atomic_load.ptr);
60726101 try reap(f, inst, &.{atomic_load.ptr});
60736102 const ptr_ty = f.air.typeOf(atomic_load.ptr);
6103 const ty = ptr_ty.childType();
6104
6105 const target = f.object.dg.module.getTarget();
6106 var repr_pl = Type.Payload.Bits{
6107 .base = .{ .tag = .int_unsigned },
6108 .data = @intCast(u16, ty.abiSize(target) * 8),
6109 };
6110 const repr_ty = if (ty.isRuntimeFloat()) Type.initPayload(&repr_pl.base) else ty;
60746111
60756112 const inst_ty = f.air.typeOfIndex(inst);
60766113 const writer = f.object.writer();
60776114 const local = try f.allocLocal(inst, inst_ty);
6078 try f.writeCValue(writer, local, .Other);
60796115
6080 try writer.writeAll(" = zig_atomic_load((zig_atomic(");
6081 try f.renderType(writer, ptr_ty.elemType());
6116 try writer.writeAll("zig_atomic_load(");
6117 try f.writeCValue(writer, local, .Other);
6118 try writer.writeAll(", (zig_atomic(");
6119 try f.renderType(writer, ty);
60826120 try writer.writeByte(')');
60836121 if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile");
60846122 try writer.writeAll(" *)");
......@@ -6086,7 +6124,9 @@ fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue {
60866124 try writer.writeAll(", ");
60876125 try writeMemoryOrder(writer, atomic_load.order);
60886126 try writer.writeAll(", ");
6089 try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType());
6127 try f.object.dg.renderTypeForBuiltinFnName(writer, ty);
6128 try writer.writeAll(", ");
6129 try f.object.dg.renderType(writer, repr_ty);
60906130 try writer.writeAll(");\n");
60916131
60926132 return local;
......@@ -6095,22 +6135,34 @@ fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue {
60956135fn airAtomicStore(f: *Function, inst: Air.Inst.Index, order: [*:0]const u8) !CValue {
60966136 const bin_op = f.air.instructions.items(.data)[inst].bin_op;
60976137 const ptr_ty = f.air.typeOf(bin_op.lhs);
6138 const ty = ptr_ty.childType();
60986139 const ptr = try f.resolveInst(bin_op.lhs);
60996140 const element = try f.resolveInst(bin_op.rhs);
61006141 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
61016142 const writer = f.object.writer();
61026143
6144 const target = f.object.dg.module.getTarget();
6145 var repr_pl = Type.Payload.Bits{
6146 .base = .{ .tag = .int_unsigned },
6147 .data = @intCast(u16, ty.abiSize(target) * 8),
6148 };
6149 const repr_ty = if (ty.isRuntimeFloat()) Type.initPayload(&repr_pl.base) else ty;
6150
6151 const element_mat = try Materialize.start(f, inst, writer, ty, element);
61036152 try writer.writeAll("zig_atomic_store((zig_atomic(");
6104 try f.renderType(writer, ptr_ty.elemType());
6153 try f.renderType(writer, ty);
61056154 try writer.writeByte(')');
61066155 if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile");
61076156 try writer.writeAll(" *)");
61086157 try f.writeCValue(writer, ptr, .Other);
61096158 try writer.writeAll(", ");
6110 try f.writeCValue(writer, element, .FunctionArgument);
6159 try element_mat.mat(f, writer);
61116160 try writer.print(", {s}, ", .{order});
6112 try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType());
6161 try f.object.dg.renderTypeForBuiltinFnName(writer, ty);
6162 try writer.writeAll(", ");
6163 try f.object.dg.renderType(writer, repr_ty);
61136164 try writer.writeAll(");\n");
6165 try element_mat.end(f, inst);
61146166
61156167 return .none;
61166168}
......@@ -7370,6 +7422,45 @@ fn formatIntLiteral(
73707422 try data.cty.renderLiteralSuffix(writer);
73717423}
73727424
7425const Materialize = struct {
7426 local: CValue,
7427
7428 pub fn start(
7429 f: *Function,
7430 inst: Air.Inst.Index,
7431 writer: anytype,
7432 ty: Type,
7433 value: CValue,
7434 ) !Materialize {
7435 switch (value) {
7436 .local_ref, .constant, .decl_ref, .undef => {
7437 const local = try f.allocLocal(inst, ty);
7438
7439 const a = try Assignment.start(f, writer, ty);
7440 try f.writeCValue(writer, local, .Other);
7441 try a.assign(f, writer);
7442 try f.writeCValue(writer, value, .Other);
7443 try a.end(f, writer);
7444
7445 return .{ .local = local };
7446 },
7447 .new_local => |local| return .{ .local = .{ .local = local } },
7448 else => return .{ .local = value },
7449 }
7450 }
7451
7452 pub fn mat(self: Materialize, f: *Function, writer: anytype) !void {
7453 try f.writeCValue(writer, self.local, .Other);
7454 }
7455
7456 pub fn end(self: Materialize, f: *Function, inst: Air.Inst.Index) !void {
7457 switch (self.local) {
7458 .new_local => |local| try freeLocal(f, inst, local, 0),
7459 else => {},
7460 }
7461 }
7462};
7463
73737464const Assignment = struct {
73747465 cty: CType.Index,
73757466
src/codegen/llvm.zig+4-4
......@@ -10135,14 +10135,14 @@ fn toLlvmAtomicRmwBinOp(
1013510135) llvm.AtomicRMWBinOp {
1013610136 return switch (op) {
1013710137 .Xchg => .Xchg,
10138 .Add => if (is_float) llvm.AtomicRMWBinOp.FAdd else return .Add,
10139 .Sub => if (is_float) llvm.AtomicRMWBinOp.FSub else return .Sub,
10138 .Add => if (is_float) .FAdd else return .Add,
10139 .Sub => if (is_float) .FSub else return .Sub,
1014010140 .And => .And,
1014110141 .Nand => .Nand,
1014210142 .Or => .Or,
1014310143 .Xor => .Xor,
10144 .Max => if (is_signed) llvm.AtomicRMWBinOp.Max else return .UMax,
10145 .Min => if (is_signed) llvm.AtomicRMWBinOp.Min else return .UMin,
10144 .Max => if (is_float) .FMax else if (is_signed) .Max else return .UMax,
10145 .Min => if (is_float) .FMin else if (is_signed) .Min else return .UMin,
1014610146 };
1014710147}
1014810148
src/codegen/llvm/bindings.zig+2
......@@ -1436,6 +1436,8 @@ pub const AtomicRMWBinOp = enum(c_int) {
14361436 UMin,
14371437 FAdd,
14381438 FSub,
1439 FMax,
1440 FMin,
14391441};
14401442
14411443pub const TypeKind = enum(c_int) {
test/behavior/atomics.zig+5-17
......@@ -209,15 +209,7 @@ test "atomicrmw with floats" {
209209 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
210210 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
211211
212 if (builtin.zig_backend == .stage2_c) {
213 // TODO: test.c:34929:7: error: address argument to atomic operation must be a pointer to integer or pointer ('zig_f32 *' (aka 'float *') invalid
214 // when compiling with -std=c99 -pedantic
215 return error.SkipZigTest;
216 }
217
218 if ((builtin.zig_backend == .stage2_llvm or builtin.zig_backend == .stage2_c) and
219 builtin.cpu.arch == .aarch64)
220 {
212 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
221213 // https://github.com/ziglang/zig/issues/10627
222214 return error.SkipZigTest;
223215 }
......@@ -234,6 +226,10 @@ fn testAtomicRmwFloat() !void {
234226 try expect(x == 6);
235227 _ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst);
236228 try expect(x == 4);
229 _ = @atomicRmw(f32, &x, .Max, 13, .SeqCst);
230 try expect(x == 13);
231 _ = @atomicRmw(f32, &x, .Min, 42, .SeqCst);
232 try expect(x == 13);
237233}
238234
239235test "atomicrmw with ints" {
......@@ -242,10 +238,6 @@ test "atomicrmw with ints" {
242238 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
243239 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
244240
245 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch == .aarch64) {
246 return error.SkipZigTest;
247 }
248
249241 try testAtomicRmwInts();
250242 comptime try testAtomicRmwInts();
251243}
......@@ -390,10 +382,6 @@ test "atomics with different types" {
390382 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
391383 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
392384
393 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch == .aarch64) {
394 return error.SkipZigTest;
395 }
396
397385 try testAtomicsWithType(bool, true, false);
398386
399387 try testAtomicsWithType(u1, 0, 1);
test/behavior/builtin_functions_returning_void_or_noreturn.zig-1
......@@ -7,7 +7,6 @@ var x: u8 = 1;
77// This excludes builtin functions that return void or noreturn that cannot be tested.
88test {
99 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1110 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1211 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1312 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO