authorgravatar for kcbanner@gmail.comCasey Banner <kcbanner@gmail.com> 2022-12-17 14:15:37-05:00
committergravatar for kcbanner@gmail.comCasey Banner <kcbanner@gmail.com> 2023-01-01 16:44:29-05:00
logd63b8d21b36645a773d201e4b343b32859cc9927
treee12d4774db466c1f978dd5b9791b9fd2d82e9b5a
parent5b8245d35a88d3d1a0d8bd4670c735824ad40837

cbe: implement missing atomic builtings using cmpxchg loops

1 files changed, 73 insertions(+), 41 deletions(-)

lib/zig.h+73-41
...@@ -2097,49 +2097,80 @@ zig_float_builtins(c_longdouble)...@@ -2097,49 +2097,80 @@ zig_float_builtins(c_longdouble)
2097#include <intrin.h>2097#include <intrin.h>
20982098
2099// TODO: zig_msvc_atomic_load should just load 32 bit without interlocked on x86, and just load 64 bit without interlocked on x642099// TODO: zig_msvc_atomic_load should just load 32 bit without interlocked on x86, and just load 64 bit without interlocked on x64
2100// TODO: Fix obviously broken nand / min / max, these don't exist on msvc
21012100
2102#define zig_msvc_atomics(type, suffix) \2101#define zig_msvc_atomics(Type, suffix) \
2103 static inline bool zig_msvc_cmpxchg_##type(zig_##type volatile* obj, zig_##type* expected, zig_##type desired) { \2102 static inline bool zig_msvc_cmpxchg_##Type(zig_##Type volatile* obj, zig_##Type* expected, zig_##Type desired) { \
2104 zig_##type comparand = *expected; \2103 zig_##Type comparand = *expected; \
2105 zig_##type initial = _InterlockedCompareExchange##suffix(obj, desired, comparand); \2104 zig_##Type initial = _InterlockedCompareExchange##suffix(obj, desired, comparand); \
2106 bool exchanged = initial == comparand; \2105 bool exchanged = initial == comparand; \
2107 if (!exchanged) { \2106 if (!exchanged) { \
2108 *expected = initial; \2107 *expected = initial; \
2109 } \2108 } \
2110 return exchanged; \2109 return exchanged; \
2111 } \2110 } \
2112 static inline zig_##type zig_msvc_atomicrmw_xchg_##type(zig_##type volatile* obj, zig_##type value) { \2111 static inline zig_##Type zig_msvc_atomicrmw_xchg_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2113 return _InterlockedExchange##suffix(obj, value); \2112 return _InterlockedExchange##suffix(obj, value); \
2114 } \2113 } \
2115 static inline zig_##type zig_msvc_atomicrmw_add_##type(zig_##type volatile* obj, zig_##type value) { \2114 static inline zig_##Type zig_msvc_atomicrmw_add_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2116 return _InterlockedExchangeAdd##suffix(obj, value); \2115 return _InterlockedExchangeAdd##suffix(obj, value); \
2117 } \2116 } \
2118 static inline zig_##type zig_msvc_atomicrmw_sub_##type(zig_##type volatile* obj, zig_##type value) { \2117 static inline zig_##Type zig_msvc_atomicrmw_sub_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2119 return _InterlockedExchangeAdd##suffix(obj, -value); \2118 bool success = false; \
2119 zig_##Type new; \
2120 zig_##Type prev; \
2121 while (!success) { \
2122 prev = *obj; \
2123 new = prev - value; \
2124 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2125 } \
2126 return prev; \
2120 } \2127 } \
2121 static inline zig_##type zig_msvc_atomicrmw_or_##type(zig_##type volatile* obj, zig_##type value) { \2128 static inline zig_##Type zig_msvc_atomicrmw_or_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2122 return _InterlockedOr##suffix(obj, value); \2129 return _InterlockedOr##suffix(obj, value); \
2123 } \2130 } \
2124 static inline zig_##type zig_msvc_atomicrmw_xor_##type(zig_##type volatile* obj, zig_##type value) { \2131 static inline zig_##Type zig_msvc_atomicrmw_xor_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2125 return _InterlockedXor##suffix(obj, value); \2132 return _InterlockedXor##suffix(obj, value); \
2126 } \2133 } \
2127 static inline zig_##type zig_msvc_atomicrmw_and_##type(zig_##type volatile* obj, zig_##type value) { \2134 static inline zig_##Type zig_msvc_atomicrmw_and_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2128 return _InterlockedAnd##suffix(obj, value); \2135 return _InterlockedAnd##suffix(obj, value); \
2129 } \2136 } \
2130 static inline zig_##type zig_msvc_atomicrmw_nand_##type(zig_##type volatile* obj, zig_##type value) { \2137 static inline zig_##Type zig_msvc_atomicrmw_nand_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2131 return 0; \2138 bool success = false; \
2139 zig_##Type new; \
2140 zig_##Type prev; \
2141 while (!success) { \
2142 prev = *obj; \
2143 new = ~(prev & value); \
2144 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2145 } \
2146 return prev; \
2132 } \2147 } \
2133 static inline zig_##type zig_msvc_atomicrmw_min_##type(zig_##type volatile* obj, zig_##type value) { \2148 static inline zig_##Type zig_msvc_atomicrmw_min_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2134 return 0; \2149 bool success = false; \
2150 zig_##Type new; \
2151 zig_##Type prev; \
2152 while (!success) { \
2153 prev = *obj; \
2154 new = value < prev ? value : prev; \
2155 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2156 } \
2157 return prev; \
2135 } \2158 } \
2136 static inline zig_##type zig_msvc_atomicrmw_max_##type(zig_##type volatile* obj, zig_##type value) { \2159 static inline zig_##Type zig_msvc_atomicrmw_max_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2137 return 0; \2160 bool success = false; \
2161 zig_##Type new; \
2162 zig_##Type prev; \
2163 while (!success) { \
2164 prev = *obj; \
2165 new = value > prev ? value : prev; \
2166 success = zig_msvc_cmpxchg_##Type(obj, &prev, new); \
2167 } \
2168 return prev; \
2138 } \2169 } \
2139 static inline void zig_msvc_atomic_store_##type(zig_##type volatile* obj, zig_##type value) { \2170 static inline void zig_msvc_atomic_store_##Type(zig_##Type volatile* obj, zig_##Type value) { \
2140 _InterlockedExchange##suffix(obj, value); \2171 _InterlockedExchange##suffix(obj, value); \
2141 } \2172 } \
2142 static inline zig_##type zig_msvc_atomic_load_##type(zig_##type volatile* obj) { \2173 static inline zig_##Type zig_msvc_atomic_load_##Type(zig_##Type volatile* obj) { \
2143 return _InterlockedOr##suffix(obj, 0); \2174 return _InterlockedOr##suffix(obj, 0); \
2144 }2175 }
21452176
...@@ -2152,26 +2183,6 @@ zig_msvc_atomics(i32, )...@@ -2152,26 +2183,6 @@ zig_msvc_atomics(i32, )
2152zig_msvc_atomics(u64, 64)2183zig_msvc_atomics(u64, 64)
2153zig_msvc_atomics(i64, 64)2184zig_msvc_atomics(i64, 64)
21542185
2155static inline bool zig_msvc_cmpxchg_p32(void** obj, void** expected, void* desired) {
2156 void* comparand = *expected;
2157 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2158 bool exchanged = initial == comparand;
2159 if (!exchanged) {
2160 *expected = initial;
2161 }
2162 return exchanged;
2163}
2164
2165static inline bool zig_msvc_cmpxchg_p64(void** obj, void** expected, void* desired) {
2166 void* comparand = *expected;
2167 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2168 bool exchanged = initial == comparand;
2169 if (!exchanged) {
2170 *expected = initial;
2171 }
2172 return exchanged;
2173}
2174
2175#if _M_IX862186#if _M_IX86
2176static inline void* zig_msvc_atomicrmw_xchg_p32(void** obj, zig_u32* arg) {2187static inline void* zig_msvc_atomicrmw_xchg_p32(void** obj, zig_u32* arg) {
2177 return _InterlockedExchangePointer(obj, arg);2188 return _InterlockedExchangePointer(obj, arg);
...@@ -2184,8 +2195,18 @@ static inline void zig_msvc_atomic_store_p32(void** obj, zig_u32* arg) {...@@ -2184,8 +2195,18 @@ static inline void zig_msvc_atomic_store_p32(void** obj, zig_u32* arg) {
2184static inline void* zig_msvc_atomic_load_p32(void** obj, zig_u32* arg) {2195static inline void* zig_msvc_atomic_load_p32(void** obj, zig_u32* arg) {
2185 return (void*)_InterlockedOr((void*)obj, 0);2196 return (void*)_InterlockedOr((void*)obj, 0);
2186}2197}
2198
2199static inline bool zig_msvc_cmpxchg_p32(void** obj, void** expected, void* desired) {
2200 void* comparand = *expected;
2201 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2202 bool exchanged = initial == comparand;
2203 if (!exchanged) {
2204 *expected = initial;
2205 }
2206 return exchanged;
2207}
2187#else2208#else
2188static inline void* zig_msvc_atomicrmw_xchg__p64(void** obj, zig_u64* arg) {2209static inline void* zig_msvc_atomicrmw_xchg_p64(void** obj, zig_u64* arg) {
2189 return _InterlockedExchangePointer(obj, arg);2210 return _InterlockedExchangePointer(obj, arg);
2190}2211}
21912212
...@@ -2196,6 +2217,16 @@ static inline void zig_msvc_atomic_store_p64(void** obj, zig_u64* arg) {...@@ -2196,6 +2217,16 @@ static inline void zig_msvc_atomic_store_p64(void** obj, zig_u64* arg) {
2196static inline void* zig_msvc_atomic_load_p64(void** obj) {2217static inline void* zig_msvc_atomic_load_p64(void** obj) {
2197 return (void*)_InterlockedOr64((void*)obj, 0);2218 return (void*)_InterlockedOr64((void*)obj, 0);
2198}2219}
2220
2221static inline bool zig_msvc_cmpxchg_p64(void** obj, void** expected, void* desired) {
2222 void* comparand = *expected;
2223 void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand);
2224 bool exchanged = initial == comparand;
2225 if (!exchanged) {
2226 *expected = initial;
2227 }
2228 return exchanged;
2229}
2199#endif2230#endif
22002231
2201static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expected, zig_u128 desired) {2232static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expected, zig_u128 desired) {
...@@ -2205,4 +2236,5 @@ static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expec...@@ -2205,4 +2236,5 @@ static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expec
2205static inline bool zig_msvc_cmpxchg_i128(zig_i128 volatile* obj, zig_i128* expected, zig_i128 desired) {2236static inline bool zig_msvc_cmpxchg_i128(zig_i128 volatile* obj, zig_i128* expected, zig_i128 desired) {
2206 return _InterlockedCompareExchange128((zig_i64 volatile*)obj, desired.hi, desired.lo, (zig_u64*)expected);2237 return _InterlockedCompareExchange128((zig_i64 volatile*)obj, desired.hi, desired.lo, (zig_u64*)expected);
2207}2238}
2239
2208#endif2240#endif