authorgravatar for kcbanner@gmail.comCasey Banner <kcbanner@gmail.com> 2022-12-17 14:15:37-05:00
committergravatar for kcbanner@gmail.comCasey Banner <kcbanner@gmail.com> 2023-01-01 16:44:29-05:00
logd63b8d21b36645a773d201e4b343b32859cc9927
treee12d4774db466c1f978dd5b9791b9fd2d82e9b5a
parent5b8245d35a88d3d1a0d8bd4670c735824ad40837

cbe: implement missing atomic builtings using cmpxchg loops

1 files changed, 73 insertions(+), 41 deletions(-)

lib/zig.h+73-41
......@@ -2097,49 +2097,80 @@ zig_float_builtins(c_longdouble)
20972097#include <intrin.h>
20982098
20992099// TODO: zig_msvc_atomic_load should just load 32 bit without interlocked on x86, and just load 64 bit without interlocked on x64
2100// TODO: Fix obviously broken nand / min / max, these don't exist on msvc
21012100
2102#define zig_msvc_atomics(type, suffix) \
2103 static inline bool zig_msvc_cmpxchg_##type(zig_##type volatile* obj, zig_##type* expected, zig_##type desired) { \
2104 zig_##type comparand = *expected; \
2105 zig_##type initial = _InterlockedCompareExchange##suffix(obj, desired, comparand); \
2101#define zig_msvc_atomics(Type, suffix) \
2102 static inline bool zig_msvc_cmpxchg_##Type(zig_##Type volatile* obj, zig_##Type* expected, zig_##Type desired) { \
2103 zig_##Type comparand = *expected; \
2104 zig_##Type initial = _InterlockedCompareExchange##suffix(obj, desired, comparand); \
21062105 bool exchanged = initial == comparand; \
21072106 if (!exchanged) { \
21082107 *expected = initial; \
21092108 } \
21102109 return exchanged; \
21112110 } \
2112 static inline zig_##type zig_msvc_atomicrmw_xchg_##type(zig_##type volatile* obj, zig_##type value) { \
2111 static inline zig_##Type zig_msvc_atomicrmw_xchg_##Type(zig_##Type volatile* obj, zig_##Type value) { \
21132112 return _InterlockedExchange##suffix(obj, value); \
21142113 } \
2115 static inline zig_##type zig_msvc_atomicrmw_add_##type(zig_##type volatile* obj, zig_##type value) { \
2114 static inline zig_##Type zig_msvc_atomicrmw_add_##Type(zig_##Type volatile* obj, zig_##Type value) { \
21162115 return _InterlockedExchangeAdd##suffix(obj, value); \
21172116 } \
2118 static inline zig_##type zig_msvc_atomicrmw_sub_##type(zig_##type volatile* obj, zig_##type value) { \
2119 return _InterlockedExchangeAdd##suffix(obj, -value); \
2117 static inline zig_##Type zig_msvc_atomicrmw_sub_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2118 bool success = false; \
2119 zig_##Type new; \
2120 zig_##Type prev; \
2121 while (!success) { \
2122 prev = *obj; \
2123 new = prev - value; \
2124 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2125 } \
2126 return prev; \
21202127 } \
2121 static inline zig_##type zig_msvc_atomicrmw_or_##type(zig_##type volatile* obj, zig_##type value) { \
2128 static inline zig_##Type zig_msvc_atomicrmw_or_##Type(zig_##Type volatile* obj, zig_##Type value) { \
21222129 return _InterlockedOr##suffix(obj, value); \
21232130 } \
2124 static inline zig_##type zig_msvc_atomicrmw_xor_##type(zig_##type volatile* obj, zig_##type value) { \
2131 static inline zig_##Type zig_msvc_atomicrmw_xor_##Type(zig_##Type volatile* obj, zig_##Type value) { \
21252132 return _InterlockedXor##suffix(obj, value); \
21262133 } \
2127 static inline zig_##type zig_msvc_atomicrmw_and_##type(zig_##type volatile* obj, zig_##type value) { \
2134 static inline zig_##Type zig_msvc_atomicrmw_and_##Type(zig_##Type volatile* obj, zig_##Type value) { \
21282135 return _InterlockedAnd##suffix(obj, value); \
21292136 } \
2130 static inline zig_##type zig_msvc_atomicrmw_nand_##type(zig_##type volatile* obj, zig_##type value) { \
2131 return 0; \
2137 static inline zig_##Type zig_msvc_atomicrmw_nand_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2138 bool success = false; \
2139 zig_##Type new; \
2140 zig_##Type prev; \
2141 while (!success) { \
2142 prev = *obj; \
2143 new = ~(prev & value); \
2144 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2145 } \
2146 return prev; \
21322147 } \
2133 static inline zig_##type zig_msvc_atomicrmw_min_##type(zig_##type volatile* obj, zig_##type value) { \
2134 return 0; \
2148 static inline zig_##Type zig_msvc_atomicrmw_min_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2149 bool success = false; \
2150 zig_##Type new; \
2151 zig_##Type prev; \
2152 while (!success) { \
2153 prev = *obj; \
2154 new = value < prev ? value : prev; \
2155 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2156 } \
2157 return prev; \
21352158 } \
2136 static inline zig_##type zig_msvc_atomicrmw_max_##type(zig_##type volatile* obj, zig_##type value) { \
2137 return 0; \
2159 static inline zig_##Type zig_msvc_atomicrmw_max_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2160 bool success = false; \
2161 zig_##Type new; \
2162 zig_##Type prev; \
2163 while (!success) { \
2164 prev = *obj; \
2165 new = value > prev ? value : prev; \
2166 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2167 } \
2168 return prev; \
21382169 } \
2139 static inline void zig_msvc_atomic_store_##type(zig_##type volatile* obj, zig_##type value) { \
2170 static inline void zig_msvc_atomic_store_##Type(zig_##Type volatile* obj, zig_##Type value) { \
21402171 _InterlockedExchange##suffix(obj, value); \
21412172 } \
2142 static inline zig_##type zig_msvc_atomic_load_##type(zig_##type volatile* obj) { \
2173 static inline zig_##Type zig_msvc_atomic_load_##Type(zig_##Type volatile* obj) { \
21432174 return _InterlockedOr##suffix(obj, 0); \
21442175 }
21452176
......@@ -2152,26 +2183,6 @@ zig_msvc_atomics(i32, )
21522183zig_msvc_atomics(u64, 64)
21532184zig_msvc_atomics(i64, 64)
21542185
2155static inline bool zig_msvc_cmpxchg_p32(void** obj, void** expected, void* desired) {
2156 void* comparand = *expected;
2157 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2158 bool exchanged = initial == comparand;
2159 if (!exchanged) {
2160 *expected = initial;
2161 }
2162 return exchanged;
2163}
2164
2165static inline bool zig_msvc_cmpxchg_p64(void** obj, void** expected, void* desired) {
2166 void* comparand = *expected;
2167 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2168 bool exchanged = initial == comparand;
2169 if (!exchanged) {
2170 *expected = initial;
2171 }
2172 return exchanged;
2173}
2174
21752186#if _M_IX86
21762187static inline void* zig_msvc_atomicrmw_xchg_p32(void** obj, zig_u32* arg) {
21772188 return _InterlockedExchangePointer(obj, arg);
......@@ -2184,8 +2195,18 @@ static inline void zig_msvc_atomic_store_p32(void** obj, zig_u32* arg) {
21842195static inline void* zig_msvc_atomic_load_p32(void** obj, zig_u32* arg) {
21852196 return (void*)_InterlockedOr((void*)obj, 0);
21862197}
2198
2199static inline bool zig_msvc_cmpxchg_p32(void** obj, void** expected, void* desired) {
2200 void* comparand = *expected;
2201 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2202 bool exchanged = initial == comparand;
2203 if (!exchanged) {
2204 *expected = initial;
2205 }
2206 return exchanged;
2207}
21872208#else
2188static inline void* zig_msvc_atomicrmw_xchg__p64(void** obj, zig_u64* arg) {
2209static inline void* zig_msvc_atomicrmw_xchg_p64(void** obj, zig_u64* arg) {
21892210 return _InterlockedExchangePointer(obj, arg);
21902211}
21912212
......@@ -2196,6 +2217,16 @@ static inline void zig_msvc_atomic_store_p64(void** obj, zig_u64* arg) {
21962217static inline void* zig_msvc_atomic_load_p64(void** obj) {
21972218 return (void*)_InterlockedOr64((void*)obj, 0);
21982219}
2220
2221static inline bool zig_msvc_cmpxchg_p64(void** obj, void** expected, void* desired) {
2222 void* comparand = *expected;
2223 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2224 bool exchanged = initial == comparand;
2225 if (!exchanged) {
2226 *expected = initial;
2227 }
2228 return exchanged;
2229}
21992230#endif
22002231
22012232static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expected, zig_u128 desired) {
......@@ -2205,4 +2236,5 @@ static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expec
22052236static inline bool zig_msvc_cmpxchg_i128(zig_i128 volatile* obj, zig_i128* expected, zig_i128 desired) {
22062237 return _InterlockedCompareExchange128((zig_i64 volatile*)obj, desired.hi, desired.lo, (zig_u64*)expected);
22072238}
2239
22082240#endif