authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-21 15:58:40-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-21 16:36:31-04:00
logd56c6c77913a6d560b7d3ebd6283cbf71d5a0222
tree11af28d08220a48515a75f073bd18127f6cebd7a
parent5afaaf865f93965f2a13069d4890c1491664652f

cbe: implement 128-bit atomics support

* Disable 128-bit atomics for x86_64 generic (currently also baseline) because they require heavy abi agreement to correctly lower. ** This is a breaking change ** * Enable 128-bit atomics for aarch64 in Sema since it just works.

4 files changed, 258 insertions(+), 264 deletions(-)

lib/zig.h+232-247
......@@ -253,154 +253,6 @@ typedef char bool;
253253#define zig_concat(lhs, rhs) lhs##rhs
254254#define zig_expand_concat(lhs, rhs) zig_concat(lhs, rhs)
255255
256#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
257#include <stdatomic.h>
258typedef enum memory_order zig_memory_order;
259#define zig_atomic(Type) _Atomic(Type)
260#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
261#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail)
262#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = atomic_exchange_explicit (obj, arg, order)
263#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = atomic_fetch_add_explicit (obj, arg, order)
264#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = atomic_fetch_sub_explicit (obj, arg, order)
265#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = atomic_fetch_or_explicit (obj, arg, order)
266#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = atomic_fetch_xor_explicit (obj, arg, order)
267#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = atomic_fetch_and_explicit (obj, arg, order)
268#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order)
269#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order)
270#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order)
271#define zig_atomic_store( obj, arg, order, Type, ReprType) atomic_store_explicit (obj, arg, order)
272#define zig_atomic_load(res, obj, order, Type, ReprType) res = atomic_load_explicit (obj, order)
273#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
274#define zig_atomicrmw_add_float zig_atomicrmw_add
275#define zig_atomicrmw_sub_float zig_atomicrmw_sub
276#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \
277 zig_##Type zig_atomicrmw_desired; \
278 zig_atomic_load(res, obj, order, Type, ReprType); \
279 do { \
280 zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(res, arg); \
281 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
282} while (0)
283#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \
284 zig_##Type zig_atomicrmw_desired; \
285 zig_atomic_load(res, obj, order, Type, ReprType); \
286 do { \
287 zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(res, arg); \
288 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
289} while (0)
290#define zig_fence(order) atomic_thread_fence(order)
291#elif defined(__GNUC__)
292typedef int zig_memory_order;
293#define memory_order_relaxed __ATOMIC_RELAXED
294#define memory_order_consume __ATOMIC_CONSUME
295#define memory_order_acquire __ATOMIC_ACQUIRE
296#define memory_order_release __ATOMIC_RELEASE
297#define memory_order_acq_rel __ATOMIC_ACQ_REL
298#define memory_order_seq_cst __ATOMIC_SEQ_CST
299#define zig_atomic(Type) Type
300#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), false, succ, fail)
301#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), true, succ, fail)
302#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) __atomic_exchange(obj, &(arg), &(res), order)
303#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_add (obj, arg, order)
304#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_sub (obj, arg, order)
305#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_or (obj, arg, order)
306#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_xor (obj, arg, order)
307#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_and (obj, arg, order)
308#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order)
309#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order)
310#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order)
311#define zig_atomic_store( obj, arg, order, Type, ReprType) __atomic_store (obj, &(arg), order)
312#define zig_atomic_load(res, obj, order, Type, ReprType) __atomic_load (obj, &(res), order)
313#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
314#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) do { \
315 zig_##Type zig_atomicrmw_desired; \
316 zig_atomic_load(res, obj, order, Type, ReprType); \
317 do { \
318 zig_atomicrmw_desired = (res) + (arg); \
319 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
320} while (0)
321#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) do { \
322 zig_##Type zig_atomicrmw_desired; \
323 zig_atomic_load(res, obj, order, Type, ReprType); \
324 do { \
325 zig_atomicrmw_desired = (res) - (arg); \
326 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
327} while (0)
328#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \
329 zig_##Type zig_atomicrmw_desired; \
330 zig_atomic_load(res, obj, order, Type, ReprType); \
331 do { \
332 zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(res, arg); \
333 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
334} while (0)
335#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \
336 zig_##Type zig_atomicrmw_desired; \
337 zig_atomic_load(res, obj, order, Type, ReprType); \
338 do { \
339 zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(res, arg); \
340 } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
341} while (0)
342#define zig_fence(order) __atomic_thread_fence(order)
343#elif _MSC_VER && (_M_IX86 || _M_X64)
344#define memory_order_relaxed 0
345#define memory_order_consume 1
346#define memory_order_acquire 2
347#define memory_order_release 3
348#define memory_order_acq_rel 4
349#define memory_order_seq_cst 5
350#define zig_atomic(Type) Type
351#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_msvc_cmpxchg_##Type(obj, &(expected), desired)
352#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_cmpxchg_strong(obj, expected, desired, succ, fail, Type, ReprType)
353#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xchg_##Type(obj, arg)
354#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_add_ ##Type(obj, arg)
355#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_sub_ ##Type(obj, arg)
356#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_or_ ##Type(obj, arg)
357#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xor_ ##Type(obj, arg)
358#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_and_ ##Type(obj, arg)
359#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_nand_##Type(obj, arg)
360#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_min_ ##Type(obj, arg)
361#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_max_ ##Type(obj, arg)
362#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_msvc_atomic_store_ ##Type(obj, arg)
363#define zig_atomic_load(res, obj, order, Type, ReprType) res = zig_msvc_atomic_load_ ##Type(obj)
364#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
365#define zig_atomicrmw_add_float zig_atomicrmw_add
366#define zig_atomicrmw_sub_float zig_atomicrmw_sub
367#define zig_atomicrmw_min_float zig_atomicrmw_min
368#define zig_atomicrmw_max_float zig_atomicrmw_max
369#if _M_X64
370#define zig_fence(order) __faststorefence()
371#else
372#define zig_fence(order) zig_msvc_atomic_barrier()
373#endif
374
375// TODO: _MSC_VER && (_M_ARM || _M_ARM64)
376#else
377#define memory_order_relaxed 0
378#define memory_order_consume 1
379#define memory_order_acquire 2
380#define memory_order_release 3
381#define memory_order_acq_rel 4
382#define memory_order_seq_cst 5
383#define zig_atomic(Type) Type
384#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable
385#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable
386#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
387#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
388#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
389#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
390#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
391#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
392#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
393#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
394#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
395#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_atomics_unavailable
396#define zig_atomic_load(res, obj, order, Type, ReprType) zig_atomics_unavailable
397#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
398#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
399#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
400#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
401#define zig_fence(order) zig_fence_unavailable
402#endif
403
404256#if __STDC_VERSION__ >= 201112L
405257#define zig_noreturn _Noreturn
406258#elif zig_has_attribute(noreturn) || defined(zig_gnuc)
......@@ -1517,12 +1369,12 @@ static zig_u128 zig_mul_u128(zig_u128 lhs, zig_u128 rhs) {
15171369zig_extern zig_u128 __udivti3(zig_u128 lhs, zig_u128 rhs);
15181370static zig_u128 zig_div_trunc_u128(zig_u128 lhs, zig_u128 rhs) {
15191371 return __udivti3(lhs, rhs);
1520};
1372}
15211373
15221374zig_extern zig_i128 __divti3(zig_i128 lhs, zig_i128 rhs);
15231375static zig_i128 zig_div_trunc_i128(zig_i128 lhs, zig_i128 rhs) {
15241376 return __divti3(lhs, rhs);
1525};
1377}
15261378
15271379zig_extern zig_u128 __umodti3(zig_u128 lhs, zig_u128 rhs);
15281380static zig_u128 zig_rem_u128(zig_u128 lhs, zig_u128 rhs) {
......@@ -1548,10 +1400,6 @@ static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) {
15481400#define zig_div_floor_u128 zig_div_trunc_u128
15491401#define zig_mod_u128 zig_rem_u128
15501402
1551static inline zig_u128 zig_nand_u128(zig_u128 lhs, zig_u128 rhs) {
1552 return zig_not_u128(zig_and_u128(lhs, rhs), 128);
1553}
1554
15551403static inline zig_u128 zig_min_u128(zig_u128 lhs, zig_u128 rhs) {
15561404 return zig_cmp_u128(lhs, rhs) < INT32_C(0) ? lhs : rhs;
15571405}
......@@ -3307,12 +3155,12 @@ typedef zig_repr_f128 zig_f128;
33073155#endif
33083156
33093157#if !_MSC_VER && defined(ZIG_TARGET_ABI_MSVC)
3310// Emulate msvc abi on a gnu compiler
3158/* Emulate msvc abi on a gnu compiler */
33113159#define zig_bitSizeOf_c_longdouble 64
33123160typedef zig_repr_f64 zig_repr_c_longdouble;
33133161typedef zig_f64 zig_c_longdouble;
33143162#elif _MSC_VER && !defined(ZIG_TARGET_ABI_MSVC)
3315// Emulate gnu abi on an msvc compiler
3163/* Emulate gnu abi on an msvc compiler */
33163164#define zig_bitSizeOf_c_longdouble 128
33173165typedef zig_repr_f128 zig_repr_c_longdouble;
33183166typedef zig_f128 zig_c_longdouble;
......@@ -3501,9 +3349,236 @@ zig_float_builtins(128)
35013349
35023350/* ============================ Atomics Support ============================= */
35033351
3352/* Note that atomics should be implemented as macros because most
3353 compilers silently discard runtime atomic order information. */
3354
3355/* Define fallback implementations first that can later be undef'd on compilers with builtin support. */
3356/* Note that zig_atomicrmw_expected is needed to handle aliasing between res and arg. */
3357#define zig_atomicrmw_xchg_float(res, obj, arg, order, Type, ReprType) do { \
3358 zig_##Type zig_atomicrmw_expected; \
3359 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3360 while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, arg, order, memory_order_relaxed, Type, ReprType)); \
3361 res = zig_atomicrmw_expected; \
3362} while (0)
3363#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) do { \
3364 zig_##Type zig_atomicrmw_expected; \
3365 zig_##Type zig_atomicrmw_desired; \
3366 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3367 do { \
3368 zig_atomicrmw_desired = zig_add_##Type(zig_atomicrmw_expected, arg); \
3369 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3370 res = zig_atomicrmw_expected; \
3371} while (0)
3372#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) do { \
3373 zig_##Type zig_atomicrmw_expected; \
3374 zig_##Type zig_atomicrmw_desired; \
3375 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3376 do { \
3377 zig_atomicrmw_desired = zig_sub_##Type(zig_atomicrmw_expected, arg); \
3378 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3379 res = zig_atomicrmw_expected; \
3380} while (0)
3381#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \
3382 zig_##Type zig_atomicrmw_expected; \
3383 zig_##Type zig_atomicrmw_desired; \
3384 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3385 do { \
3386 zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(zig_atomicrmw_expected, arg); \
3387 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3388 res = zig_atomicrmw_expected; \
3389} while (0)
3390#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \
3391 zig_##Type zig_atomicrmw_expected; \
3392 zig_##Type zig_atomicrmw_desired; \
3393 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3394 do { \
3395 zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(zig_atomicrmw_expected, arg); \
3396 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3397 res = zig_atomicrmw_expected; \
3398} while (0)
3399
3400#define zig_atomicrmw_xchg_int128(res, obj, arg, order, Type, ReprType) do { \
3401 zig_##Type zig_atomicrmw_expected; \
3402 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3403 while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, arg, order, memory_order_relaxed, Type, ReprType)); \
3404 res = zig_atomicrmw_expected; \
3405} while (0)
3406#define zig_atomicrmw_add_int128(res, obj, arg, order, Type, ReprType) do { \
3407 zig_##Type zig_atomicrmw_expected; \
3408 zig_##Type zig_atomicrmw_desired; \
3409 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3410 do { \
3411 zig_atomicrmw_desired = zig_add_##Type(zig_atomicrmw_expected, arg); \
3412 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3413 res = zig_atomicrmw_expected; \
3414} while (0)
3415#define zig_atomicrmw_sub_int128(res, obj, arg, order, Type, ReprType) do { \
3416 zig_##Type zig_atomicrmw_expected; \
3417 zig_##Type zig_atomicrmw_desired; \
3418 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3419 do { \
3420 zig_atomicrmw_desired = zig_sub_##Type(zig_atomicrmw_expected, arg); \
3421 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3422 res = zig_atomicrmw_expected; \
3423} while (0)
3424#define zig_atomicrmw_and_int128(res, obj, arg, order, Type, ReprType) do { \
3425 zig_##Type zig_atomicrmw_expected; \
3426 zig_##Type zig_atomicrmw_desired; \
3427 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3428 do { \
3429 zig_atomicrmw_desired = zig_and_##Type(zig_atomicrmw_expected, arg); \
3430 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3431 res = zig_atomicrmw_expected; \
3432} while (0)
3433#define zig_atomicrmw_nand_int128(res, obj, arg, order, Type, ReprType) do { \
3434 zig_##Type zig_atomicrmw_expected; \
3435 zig_##Type zig_atomicrmw_desired; \
3436 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3437 do { \
3438 zig_atomicrmw_desired = zig_not_##Type(zig_and_##Type(zig_atomicrmw_expected, arg), 128); \
3439 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3440 res = zig_atomicrmw_expected; \
3441} while (0)
3442#define zig_atomicrmw_or_int128(res, obj, arg, order, Type, ReprType) do { \
3443 zig_##Type zig_atomicrmw_expected; \
3444 zig_##Type zig_atomicrmw_desired; \
3445 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3446 do { \
3447 zig_atomicrmw_desired = zig_or_##Type(zig_atomicrmw_expected, arg); \
3448 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3449 res = zig_atomicrmw_expected; \
3450} while (0)
3451#define zig_atomicrmw_xor_int128(res, obj, arg, order, Type, ReprType) do { \
3452 zig_##Type zig_atomicrmw_expected; \
3453 zig_##Type zig_atomicrmw_desired; \
3454 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3455 do { \
3456 zig_atomicrmw_desired = zig_xor_##Type(zig_atomicrmw_expected, arg); \
3457 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3458 res = zig_atomicrmw_expected; \
3459} while (0)
3460#define zig_atomicrmw_min_int128(res, obj, arg, order, Type, ReprType) do { \
3461 zig_##Type zig_atomicrmw_expected; \
3462 zig_##Type zig_atomicrmw_desired; \
3463 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3464 do { \
3465 zig_atomicrmw_desired = zig_min_##Type(zig_atomicrmw_expected, arg); \
3466 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3467 res = zig_atomicrmw_expected; \
3468} while (0)
3469#define zig_atomicrmw_max_int128(res, obj, arg, order, Type, ReprType) do { \
3470 zig_##Type zig_atomicrmw_expected; \
3471 zig_##Type zig_atomicrmw_desired; \
3472 zig_atomic_load(zig_atomicrmw_expected, obj, memory_order_relaxed, Type, ReprType); \
3473 do { \
3474 zig_atomicrmw_desired = zig_max_##Type(zig_atomicrmw_expected, arg); \
3475 } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \
3476 res = zig_atomicrmw_expected; \
3477} while (0)
3478
3479#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
3480#include <stdatomic.h>
3481typedef enum memory_order zig_memory_order;
3482#define zig_atomic(Type) _Atomic(Type)
3483#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
3484#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail)
3485#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = atomic_exchange_explicit (obj, arg, order)
3486#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = atomic_fetch_add_explicit (obj, arg, order)
3487#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = atomic_fetch_sub_explicit (obj, arg, order)
3488#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = atomic_fetch_or_explicit (obj, arg, order)
3489#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = atomic_fetch_xor_explicit (obj, arg, order)
3490#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = atomic_fetch_and_explicit (obj, arg, order)
3491#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order)
3492#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order)
3493#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order)
3494#define zig_atomic_store( obj, arg, order, Type, ReprType) atomic_store_explicit (obj, arg, order)
3495#define zig_atomic_load(res, obj, order, Type, ReprType) res = atomic_load_explicit (obj, order)
3496#undef zig_atomicrmw_xchg_float
3497#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
3498#undef zig_atomicrmw_add_float
3499#define zig_atomicrmw_add_float zig_atomicrmw_add
3500#undef zig_atomicrmw_sub_float
3501#define zig_atomicrmw_sub_float zig_atomicrmw_sub
3502#define zig_fence(order) atomic_thread_fence(order)
3503#elif defined(__GNUC__)
3504typedef int zig_memory_order;
3505#define memory_order_relaxed __ATOMIC_RELAXED
3506#define memory_order_consume __ATOMIC_CONSUME
3507#define memory_order_acquire __ATOMIC_ACQUIRE
3508#define memory_order_release __ATOMIC_RELEASE
3509#define memory_order_acq_rel __ATOMIC_ACQ_REL
3510#define memory_order_seq_cst __ATOMIC_SEQ_CST
3511#define zig_atomic(Type) Type
3512#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), false, succ, fail)
3513#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), true, succ, fail)
3514#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) __atomic_exchange(obj, &(arg), &(res), order)
3515#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_add (obj, arg, order)
3516#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_sub (obj, arg, order)
3517#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_or (obj, arg, order)
3518#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_xor (obj, arg, order)
3519#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_and (obj, arg, order)
3520#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order)
3521#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order)
3522#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order)
3523#define zig_atomic_store( obj, arg, order, Type, ReprType) __atomic_store (obj, &(arg), order)
3524#define zig_atomic_load(res, obj, order, Type, ReprType) __atomic_load (obj, &(res), order)
3525#undef zig_atomicrmw_xchg_float
3526#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg
3527#define zig_fence(order) __atomic_thread_fence(order)
3528#elif _MSC_VER && (_M_IX86 || _M_X64)
3529#define memory_order_relaxed 0
3530#define memory_order_consume 1
3531#define memory_order_acquire 2
3532#define memory_order_release 3
3533#define memory_order_acq_rel 4
3534#define memory_order_seq_cst 5
3535#define zig_atomic(Type) Type
3536#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_msvc_cmpxchg_##Type(obj, &(expected), desired)
3537#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_cmpxchg_strong(obj, expected, desired, succ, fail, Type, ReprType)
3538#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xchg_##Type(obj, arg)
3539#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_add_ ##Type(obj, arg)
3540#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_sub_ ##Type(obj, arg)
3541#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_or_ ##Type(obj, arg)
3542#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xor_ ##Type(obj, arg)
3543#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_and_ ##Type(obj, arg)
3544#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_nand_##Type(obj, arg)
3545#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_min_ ##Type(obj, arg)
3546#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_max_ ##Type(obj, arg)
3547#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_msvc_atomic_store_ ##Type(obj, arg)
3548#define zig_atomic_load(res, obj, order, Type, ReprType) res = zig_msvc_atomic_load_ ##Type(obj)
3549#if _M_X64
3550#define zig_fence(order) __faststorefence()
3551#else
3552#define zig_fence(order) zig_msvc_atomic_barrier()
3553#endif
3554/* TODO: _MSC_VER && (_M_ARM || _M_ARM64) */
3555#else
3556#define memory_order_relaxed 0
3557#define memory_order_consume 1
3558#define memory_order_acquire 2
3559#define memory_order_release 3
3560#define memory_order_acq_rel 4
3561#define memory_order_seq_cst 5
3562#define zig_atomic(Type) Type
3563#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable
3564#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable
3565#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3566#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3567#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3568#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3569#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3570#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3571#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3572#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3573#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable
3574#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_atomics_unavailable
3575#define zig_atomic_load(res, obj, order, Type, ReprType) zig_atomics_unavailable
3576#define zig_fence(order) zig_fence_unavailable
3577#endif
3578
35043579#if _MSC_VER && (_M_IX86 || _M_X64)
35053580
3506// TODO: zig_msvc_atomic_load should load 32 bit without interlocked on x86, and load 64 bit without interlocked on x64
3581/* TODO: zig_msvc_atomic_load should load 32 bit without interlocked on x86, and load 64 bit without interlocked on x64 */
35073582
35083583#define zig_msvc_atomics(ZigType, Type, SigType, suffix) \
35093584 static inline bool zig_msvc_cmpxchg_##ZigType(Type volatile* obj, Type* expected, Type desired) { \
......@@ -3605,61 +3680,7 @@ zig_msvc_atomics(i64, int64_t, __int64, 64)
36053680 success = initial == comparand; \
36063681 if (!success) memcpy(expected, &initial, sizeof(*expected)); \
36073682 return success; \
3608 } \
3609 static inline zig_##Type zig_msvc_atomicrmw_xchg_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3610 ReprType repr; \
3611 ReprType initial; \
3612 zig_##Type result; \
3613 memcpy(&repr, &value, sizeof(repr)); \
3614 initial = _InterlockedExchange##suffix((ReprType volatile*)obj, repr); \
3615 memcpy(&result, &initial, sizeof(result)); \
3616 return result; \
3617 } \
3618 static inline zig_##Type zig_msvc_atomicrmw_add_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3619 ReprType repr; \
3620 zig_##Type expected; \
3621 zig_##Type desired; \
3622 repr = *(ReprType volatile*)obj; \
3623 memcpy(&expected, &repr, sizeof(expected)); \
3624 do { \
3625 desired = expected + value; \
3626 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
3627 return expected; \
3628 } \
3629 static inline zig_##Type zig_msvc_atomicrmw_sub_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3630 ReprType repr; \
3631 zig_##Type expected; \
3632 zig_##Type desired; \
3633 repr = *(ReprType volatile*)obj; \
3634 memcpy(&expected, &repr, sizeof(expected)); \
3635 do { \
3636 desired = expected - value; \
3637 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
3638 return expected; \
3639 } \
3640 static inline zig_##Type zig_msvc_atomicrmw_min_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3641 ReprType repr; \
3642 zig_##Type expected; \
3643 zig_##Type desired; \
3644 repr = *(ReprType volatile*)obj; \
3645 memcpy(&expected, &repr, sizeof(expected)); \
3646 do { \
3647 desired = zig_libc_name_##Type(fmin)(expected, value); \
3648 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
3649 return expected; \
3650 } \
3651 static inline zig_##Type zig_msvc_atomicrmw_max_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3652 ReprType repr; \
3653 zig_##Type expected; \
3654 zig_##Type desired; \
3655 repr = *(ReprType volatile*)obj; \
3656 memcpy(&expected, &repr, sizeof(expected)); \
3657 do { \
3658 desired = zig_libc_name_##Type(fmax)(expected, value); \
3659 } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \
3660 return expected; \
36613683 }
3662
36633684zig_msvc_flt_atomics(f32, long, )
36643685#if _M_X64
36653686zig_msvc_flt_atomics(f64, int64_t, 64)
......@@ -3720,42 +3741,6 @@ static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expec
37203741static inline bool zig_msvc_cmpxchg_i128(zig_i128 volatile* obj, zig_i128* expected, zig_i128 desired) {
37213742 return _InterlockedCompareExchange128((__int64 volatile*)obj, (__int64)zig_hi_i128(desired), (__int64)zig_lo_i128(desired), (__int64*)expected);
37223743}
3723
3724#define zig_msvc_atomics_128xchg(Type) \
3725 static inline zig_##Type zig_msvc_atomicrmw_xchg_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3726 bool success = false; \
3727 zig_##Type prev; \
3728 while (!success) { \
3729 prev = *obj; \
3730 success = zig_msvc_cmpxchg_##Type(obj, &prev, value); \
3731 } \
3732 return prev; \
3733 }
3734
3735zig_msvc_atomics_128xchg(u128)
3736zig_msvc_atomics_128xchg(i128)
3737
3738#define zig_msvc_atomics_128op(Type, operation) \
3739 static inline zig_##Type zig_msvc_atomicrmw_##operation##_##Type(zig_##Type volatile* obj, zig_##Type value) { \
3740 bool success = false; \
3741 zig_##Type new; \
3742 zig_##Type prev; \
3743 while (!success) { \
3744 prev = *obj; \
3745 new = zig_##operation##_##Type(prev, value); \
3746 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
3747 } \
3748 return prev; \
3749 }
3750
3751zig_msvc_atomics_128op(u128, add)
3752zig_msvc_atomics_128op(u128, sub)
3753zig_msvc_atomics_128op(u128, or)
3754zig_msvc_atomics_128op(u128, xor)
3755zig_msvc_atomics_128op(u128, and)
3756zig_msvc_atomics_128op(u128, nand)
3757zig_msvc_atomics_128op(u128, min)
3758zig_msvc_atomics_128op(u128, max)
37593744#endif /* _M_IX86 */
37603745
37613746#endif /* _MSC_VER && (_M_IX86 || _M_X64) */
src/codegen/c.zig+4-4
......@@ -6062,19 +6062,19 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue {
60626062 .data = @intCast(u16, ty.abiSize(target) * 8),
60636063 };
60646064 const is_float = ty.isRuntimeFloat();
6065 const is_128 = repr_pl.data == 128;
60656066 const repr_ty = if (is_float) Type.initPayload(&repr_pl.base) else ty;
60666067
60676068 const operand_mat = try Materialize.start(f, inst, writer, ty, operand);
60686069 try writer.print("zig_atomicrmw_{s}", .{toAtomicRmwSuffix(extra.op())});
6069 if (is_float) try writer.writeAll("_float");
6070 if (is_float) try writer.writeAll("_float") else if (is_128) try writer.writeAll("_int128");
60706071 try writer.writeByte('(');
60716072 try f.writeCValue(writer, local, .Other);
60726073 try writer.writeAll(", (");
60736074 const use_atomic = switch (extra.op()) {
60746075 else => true,
6075 // These are missing from stdatomic.h, so no atomic types for now.
6076 .Nand => false,
6077 .Min, .Max => is_float,
6076 // These are missing from stdatomic.h, so no atomic types unless a fallback is used.
6077 .Nand, .Min, .Max => is_float or is_128,
60786078 };
60796079 if (use_atomic) try writer.writeAll("zig_atomic(");
60806080 try f.renderType(writer, ty);
src/target.zig+6-4
......@@ -575,9 +575,6 @@ pub fn atomicPtrAlignment(
575575 .xtensa,
576576 => 32,
577577
578 .aarch64,
579 .aarch64_be,
580 .aarch64_32,
581578 .amdgcn,
582579 .bpfel,
583580 .bpfeb,
......@@ -600,7 +597,12 @@ pub fn atomicPtrAlignment(
600597 .loongarch64,
601598 => 64,
602599
603 .x86_64 => 128,
600 .aarch64,
601 .aarch64_be,
602 .aarch64_32,
603 => 128,
604
605 .x86_64 => if (std.Target.x86.featureSetHas(target.cpu.features, .cx16)) 128 else 64,
604606 };
605607
606608 var buffer: Type.Payload.Bits = undefined;
test/behavior/atomics.zig+16-9
......@@ -3,6 +3,13 @@ const builtin = @import("builtin");
33const expect = std.testing.expect;
44const expectEqual = std.testing.expectEqual;
55
6const supports_128_bit_atomics = switch (builtin.cpu.arch) {
7 // TODO: Ideally this could be sync'd with the logic in Sema.
8 .aarch64, .aarch64_be, .aarch64_32 => true,
9 .x86_64 => std.Target.x86.featureSetHas(builtin.cpu.features, .cx16),
10 else => false,
11};
12
613test "cmpxchg" {
714 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
815 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
......@@ -107,15 +114,14 @@ test "cmpxchg with ignored result" {
107114}
108115
109116test "128-bit cmpxchg" {
117 if (!supports_128_bit_atomics) return error.SkipZigTest;
118
110119 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
111120 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
112121 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
113122 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
114123 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
115124
116 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
117 if (comptime !std.Target.x86.featureSetHas(builtin.cpu.features, .cx16)) return error.SkipZigTest;
118
119125 try test_u128_cmpxchg();
120126 comptime try test_u128_cmpxchg();
121127}
......@@ -303,24 +309,25 @@ fn testAtomicRmwInt(comptime signedness: std.builtin.Signedness, comptime N: usi
303309}
304310
305311test "atomicrmw with 128-bit ints" {
306 if (builtin.cpu.arch != .x86_64) {
307 // TODO: Ideally this could use target.atomicPtrAlignment and check for IntTooBig
308 return error.SkipZigTest;
309 }
312 if (!supports_128_bit_atomics) return error.SkipZigTest;
310313
314 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
311315 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
312316
313317 // TODO "ld.lld: undefined symbol: __sync_lock_test_and_set_16" on -mcpu x86_64
314 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
318 if (builtin.cpu.arch == .x86_64 and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
315319
320 try testAtomicRmwInt128(.signed);
316321 try testAtomicRmwInt128(.unsigned);
322 comptime try testAtomicRmwInt128(.signed);
317323 comptime try testAtomicRmwInt128(.unsigned);
318324}
319325
320326fn testAtomicRmwInt128(comptime signedness: std.builtin.Signedness) !void {
327 const uint = std.meta.Int(.unsigned, 128);
321328 const int = std.meta.Int(signedness, 128);
322329
323 const initial: int = 0xaaaaaaaa_bbbbbbbb_cccccccc_dddddddd;
330 const initial: int = @bitCast(int, @as(uint, 0xaaaaaaaa_bbbbbbbb_cccccccc_dddddddd));
324331 const replacement: int = 0x00000000_00000005_00000000_00000003;
325332
326333 var x: int align(16) = initial;