| ... | @@ -8,6 +8,31 @@ const builtin = std.builtin; | ... | @@ -8,6 +8,31 @@ const builtin = std.builtin; |
| 8 | | 8 | |
| 9 | const linkage: builtin.GlobalLinkage = if (builtin.is_test) .Internal else .Weak; | 9 | const linkage: builtin.GlobalLinkage = if (builtin.is_test) .Internal else .Weak; |
| 10 | | 10 | |
| | 11 | // This parameter is true iff the target architecture supports the bare minimum |
| | 12 | // to implement the atomic load/store intrinsics. |
| | 13 | // Some architectures support atomic load/stores but no CAS, but we ignore this |
| | 14 | // detail to keep the export logic clean and because we need some kind of CAS to |
| | 15 | // implement the spinlocks. |
| | 16 | const supports_atomic_ops = switch (builtin.arch) { |
| | 17 | .msp430, .avr => false, |
| | 18 | .arm, .armeb, .thumb, .thumbeb => |
| | 19 | // The ARM v6m ISA has no ldrex/strex and so it's impossible to do CAS |
| | 20 | // operations (unless we're targeting Linux, the kernel provides a way to |
| | 21 | // perform CAS operations). |
| | 22 | // XXX: The Linux code path is not implemented yet. |
| | 23 | !std.Target.arm.featureSetHas(std.Target.current.cpu.features, .has_v6m), |
| | 24 | else => true, |
| | 25 | }; |
| | 26 | |
| | 27 | // The size (in bytes) of the biggest object that the architecture can |
| | 28 | // load/store atomically. |
| | 29 | // Objects bigger than this threshold require the use of a lock. |
| | 30 | const largest_atomic_size = switch (builtin.arch) { |
| | 31 | // XXX: On x86/x86_64 we could check the presence of cmpxchg8b/cmpxchg16b |
| | 32 | // and set this parameter accordingly. |
| | 33 | else => @sizeOf(usize), |
| | 34 | }; |
| | 35 | |
| 11 | const cache_line_size = 64; | 36 | const cache_line_size = 64; |
| 12 | | 37 | |
| 13 | const SpinlockTable = struct { | 38 | const SpinlockTable = struct { |
| ... | @@ -94,40 +119,18 @@ fn __atomic_compare_exchange( | ... | @@ -94,40 +119,18 @@ fn __atomic_compare_exchange( |
| 94 | } | 119 | } |
| 95 | | 120 | |
| 96 | comptime { | 121 | comptime { |
| 97 | @export(__atomic_load, .{ .name = "__atomic_load", .linkage = linkage }); | 122 | if (supports_atomic_ops) { |
| 98 | @export(__atomic_store, .{ .name = "__atomic_store", .linkage = linkage }); | 123 | @export(__atomic_load, .{ .name = "__atomic_load", .linkage = linkage }); |
| 99 | @export(__atomic_exchange, .{ .name = "__atomic_exchange", .linkage = linkage }); | 124 | @export(__atomic_store, .{ .name = "__atomic_store", .linkage = linkage }); |
| 100 | @export(__atomic_compare_exchange, .{ .name = "__atomic_compare_exchange", .linkage = linkage }); | 125 | @export(__atomic_exchange, .{ .name = "__atomic_exchange", .linkage = linkage }); |
| | 126 | @export(__atomic_compare_exchange, .{ .name = "__atomic_compare_exchange", .linkage = linkage }); |
| | 127 | } |
| 101 | } | 128 | } |
| 102 | | 129 | |
| 103 | // Specialized versions of the GCC atomic builtin functions. | 130 | // Specialized versions of the GCC atomic builtin functions. |
| 104 | // LLVM emits those iff the object size is known and the pointers are correctly | 131 | // LLVM emits those iff the object size is known and the pointers are correctly |
| 105 | // aligned. | 132 | // aligned. |
| 106 | | 133 | |
| 107 | // The size (in bytes) of the biggest object that the architecture can | | |
| 108 | // load/store atomically. | | |
| 109 | // Objects bigger than this threshold require the use of a lock. | | |
| 110 | const largest_atomic_size = switch (builtin.arch) { | | |
| 111 | .x86_64 => 16, | | |
| 112 | else => @sizeOf(usize), | | |
| 113 | }; | | |
| 114 | | | |
| 115 | // The size (in bytes) of the biggest object that the architecture can perform | | |
| 116 | // an atomic CAS operation with. | | |
| 117 | // Objects bigger than this threshold require the use of a lock. | | |
| 118 | const largest_atomic_cas_size = switch (builtin.arch) { | | |
| 119 | .arm, .armeb, .thumb, .thumbeb => | | |
| 120 | // The ARM v6m ISA has no ldrex/strex and so it's impossible to do CAS | | |
| 121 | // operations unless we're targeting Linux or the user provides the missing | | |
| 122 | // builtin functions. | | |
| 123 | if (std.Target.arm.featureSetHas(std.Target.current.cpu.features, .has_v6m) and | | |
| 124 | std.Target.current.os.tag != .linux) | | |
| 125 | 0 | | |
| 126 | else | | |
| 127 | @sizeOf(usize), | | |
| 128 | else => @sizeOf(usize), | | |
| 129 | }; | | |
| 130 | | | |
| 131 | fn atomicLoadFn(comptime T: type) fn (*T, i32) callconv(.C) T { | 134 | fn atomicLoadFn(comptime T: type) fn (*T, i32) callconv(.C) T { |
| 132 | return struct { | 135 | return struct { |
| 133 | fn atomic_load_N(src: *T, model: i32) callconv(.C) T { | 136 | fn atomic_load_N(src: *T, model: i32) callconv(.C) T { |
| ... | @@ -143,10 +146,12 @@ fn atomicLoadFn(comptime T: type) fn (*T, i32) callconv(.C) T { | ... | @@ -143,10 +146,12 @@ fn atomicLoadFn(comptime T: type) fn (*T, i32) callconv(.C) T { |
| 143 | } | 146 | } |
| 144 | | 147 | |
| 145 | comptime { | 148 | comptime { |
| 146 | @export(atomicLoadFn(u8), .{ .name = "__atomic_load_1", .linkage = linkage }); | 149 | if (supports_atomic_ops) { |
| 147 | @export(atomicLoadFn(u16), .{ .name = "__atomic_load_2", .linkage = linkage }); | 150 | @export(atomicLoadFn(u8), .{ .name = "__atomic_load_1", .linkage = linkage }); |
| 148 | @export(atomicLoadFn(u32), .{ .name = "__atomic_load_4", .linkage = linkage }); | 151 | @export(atomicLoadFn(u16), .{ .name = "__atomic_load_2", .linkage = linkage }); |
| 149 | @export(atomicLoadFn(u64), .{ .name = "__atomic_load_8", .linkage = linkage }); | 152 | @export(atomicLoadFn(u32), .{ .name = "__atomic_load_4", .linkage = linkage }); |
| | 153 | @export(atomicLoadFn(u64), .{ .name = "__atomic_load_8", .linkage = linkage }); |
| | 154 | } |
| 150 | } | 155 | } |
| 151 | | 156 | |
| 152 | fn atomicStoreFn(comptime T: type) fn (*T, T, i32) callconv(.C) void { | 157 | fn atomicStoreFn(comptime T: type) fn (*T, T, i32) callconv(.C) void { |
| ... | @@ -164,16 +169,18 @@ fn atomicStoreFn(comptime T: type) fn (*T, T, i32) callconv(.C) void { | ... | @@ -164,16 +169,18 @@ fn atomicStoreFn(comptime T: type) fn (*T, T, i32) callconv(.C) void { |
| 164 | } | 169 | } |
| 165 | | 170 | |
| 166 | comptime { | 171 | comptime { |
| 167 | @export(atomicStoreFn(u8), .{ .name = "__atomic_store_1", .linkage = linkage }); | 172 | if (supports_atomic_ops) { |
| 168 | @export(atomicStoreFn(u16), .{ .name = "__atomic_store_2", .linkage = linkage }); | 173 | @export(atomicStoreFn(u8), .{ .name = "__atomic_store_1", .linkage = linkage }); |
| 169 | @export(atomicStoreFn(u32), .{ .name = "__atomic_store_4", .linkage = linkage }); | 174 | @export(atomicStoreFn(u16), .{ .name = "__atomic_store_2", .linkage = linkage }); |
| 170 | @export(atomicStoreFn(u64), .{ .name = "__atomic_store_8", .linkage = linkage }); | 175 | @export(atomicStoreFn(u32), .{ .name = "__atomic_store_4", .linkage = linkage }); |
| | 176 | @export(atomicStoreFn(u64), .{ .name = "__atomic_store_8", .linkage = linkage }); |
| | 177 | } |
| 171 | } | 178 | } |
| 172 | | 179 | |
| 173 | fn atomicExchangeFn(comptime T: type) fn (*T, T, i32) callconv(.C) T { | 180 | fn atomicExchangeFn(comptime T: type) fn (*T, T, i32) callconv(.C) T { |
| 174 | return struct { | 181 | return struct { |
| 175 | fn atomic_exchange_N(ptr: *T, val: T, model: i32) callconv(.C) T { | 182 | fn atomic_exchange_N(ptr: *T, val: T, model: i32) callconv(.C) T { |
| 176 | if (@sizeOf(T) > largest_atomic_cas_size) { | 183 | if (@sizeOf(T) > largest_atomic_size) { |
| 177 | var sl = spinlocks.get(@ptrToInt(ptr)); | 184 | var sl = spinlocks.get(@ptrToInt(ptr)); |
| 178 | defer sl.release(); | 185 | defer sl.release(); |
| 179 | const value = ptr.*; | 186 | const value = ptr.*; |
| ... | @@ -187,16 +194,18 @@ fn atomicExchangeFn(comptime T: type) fn (*T, T, i32) callconv(.C) T { | ... | @@ -187,16 +194,18 @@ fn atomicExchangeFn(comptime T: type) fn (*T, T, i32) callconv(.C) T { |
| 187 | } | 194 | } |
| 188 | | 195 | |
| 189 | comptime { | 196 | comptime { |
| 190 | @export(atomicExchangeFn(u8), .{ .name = "__atomic_exchange_1", .linkage = linkage }); | 197 | if (supports_atomic_ops) { |
| 191 | @export(atomicExchangeFn(u16), .{ .name = "__atomic_exchange_2", .linkage = linkage }); | 198 | @export(atomicExchangeFn(u8), .{ .name = "__atomic_exchange_1", .linkage = linkage }); |
| 192 | @export(atomicExchangeFn(u32), .{ .name = "__atomic_exchange_4", .linkage = linkage }); | 199 | @export(atomicExchangeFn(u16), .{ .name = "__atomic_exchange_2", .linkage = linkage }); |
| 193 | @export(atomicExchangeFn(u64), .{ .name = "__atomic_exchange_8", .linkage = linkage }); | 200 | @export(atomicExchangeFn(u32), .{ .name = "__atomic_exchange_4", .linkage = linkage }); |
| | 201 | @export(atomicExchangeFn(u64), .{ .name = "__atomic_exchange_8", .linkage = linkage }); |
| | 202 | } |
| 194 | } | 203 | } |
| 195 | | 204 | |
| 196 | fn atomicCompareExchangeFn(comptime T: type) fn (*T, *T, T, i32, i32) callconv(.C) i32 { | 205 | fn atomicCompareExchangeFn(comptime T: type) fn (*T, *T, T, i32, i32) callconv(.C) i32 { |
| 197 | return struct { | 206 | return struct { |
| 198 | fn atomic_compare_exchange_N(ptr: *T, expected: *T, desired: T, success: i32, failure: i32) callconv(.C) i32 { | 207 | fn atomic_compare_exchange_N(ptr: *T, expected: *T, desired: T, success: i32, failure: i32) callconv(.C) i32 { |
| 199 | if (@sizeOf(T) > largest_atomic_cas_size) { | 208 | if (@sizeOf(T) > largest_atomic_size) { |
| 200 | var sl = spinlocks.get(@ptrToInt(ptr)); | 209 | var sl = spinlocks.get(@ptrToInt(ptr)); |
| 201 | defer sl.release(); | 210 | defer sl.release(); |
| 202 | const value = ptr.*; | 211 | const value = ptr.*; |
| ... | @@ -218,16 +227,18 @@ fn atomicCompareExchangeFn(comptime T: type) fn (*T, *T, T, i32, i32) callconv(. | ... | @@ -218,16 +227,18 @@ fn atomicCompareExchangeFn(comptime T: type) fn (*T, *T, T, i32, i32) callconv(. |
| 218 | } | 227 | } |
| 219 | | 228 | |
| 220 | comptime { | 229 | comptime { |
| 221 | @export(atomicCompareExchangeFn(u8), .{ .name = "__atomic_compare_exchange_1", .linkage = linkage }); | 230 | if (supports_atomic_ops) { |
| 222 | @export(atomicCompareExchangeFn(u16), .{ .name = "__atomic_compare_exchange_2", .linkage = linkage }); | 231 | @export(atomicCompareExchangeFn(u8), .{ .name = "__atomic_compare_exchange_1", .linkage = linkage }); |
| 223 | @export(atomicCompareExchangeFn(u32), .{ .name = "__atomic_compare_exchange_4", .linkage = linkage }); | 232 | @export(atomicCompareExchangeFn(u16), .{ .name = "__atomic_compare_exchange_2", .linkage = linkage }); |
| 224 | @export(atomicCompareExchangeFn(u64), .{ .name = "__atomic_compare_exchange_8", .linkage = linkage }); | 233 | @export(atomicCompareExchangeFn(u32), .{ .name = "__atomic_compare_exchange_4", .linkage = linkage }); |
| | 234 | @export(atomicCompareExchangeFn(u64), .{ .name = "__atomic_compare_exchange_8", .linkage = linkage }); |
| | 235 | } |
| 225 | } | 236 | } |
| 226 | | 237 | |
| 227 | fn fetchFn(comptime T: type, comptime op: builtin.AtomicRmwOp) fn (*T, T, i32) callconv(.C) T { | 238 | fn fetchFn(comptime T: type, comptime op: builtin.AtomicRmwOp) fn (*T, T, i32) callconv(.C) T { |
| 228 | return struct { | 239 | return struct { |
| 229 | pub fn fetch_op_N(ptr: *T, val: T, model: i32) callconv(.C) T { | 240 | pub fn fetch_op_N(ptr: *T, val: T, model: i32) callconv(.C) T { |
| 230 | if (@sizeOf(T) > largest_atomic_cas_size) { | 241 | if (@sizeOf(T) > largest_atomic_size) { |
| 231 | var sl = spinlocks.get(@ptrToInt(ptr)); | 242 | var sl = spinlocks.get(@ptrToInt(ptr)); |
| 232 | defer sl.release(); | 243 | defer sl.release(); |
| 233 | | 244 | |
| ... | @@ -251,33 +262,35 @@ fn fetchFn(comptime T: type, comptime op: builtin.AtomicRmwOp) fn (*T, T, i32) c | ... | @@ -251,33 +262,35 @@ fn fetchFn(comptime T: type, comptime op: builtin.AtomicRmwOp) fn (*T, T, i32) c |
| 251 | } | 262 | } |
| 252 | | 263 | |
| 253 | comptime { | 264 | comptime { |
| 254 | @export(fetchFn(u8, .Add), .{ .name = "__atomic_fetch_add_1", .linkage = linkage }); | 265 | if (supports_atomic_ops) { |
| 255 | @export(fetchFn(u16, .Add), .{ .name = "__atomic_fetch_add_2", .linkage = linkage }); | 266 | @export(fetchFn(u8, .Add), .{ .name = "__atomic_fetch_add_1", .linkage = linkage }); |
| 256 | @export(fetchFn(u32, .Add), .{ .name = "__atomic_fetch_add_4", .linkage = linkage }); | 267 | @export(fetchFn(u16, .Add), .{ .name = "__atomic_fetch_add_2", .linkage = linkage }); |
| 257 | @export(fetchFn(u64, .Add), .{ .name = "__atomic_fetch_add_8", .linkage = linkage }); | 268 | @export(fetchFn(u32, .Add), .{ .name = "__atomic_fetch_add_4", .linkage = linkage }); |
| 258 | | 269 | @export(fetchFn(u64, .Add), .{ .name = "__atomic_fetch_add_8", .linkage = linkage }); |
| 259 | @export(fetchFn(u8, .Sub), .{ .name = "__atomic_fetch_sub_1", .linkage = linkage }); | 270 | |
| 260 | @export(fetchFn(u16, .Sub), .{ .name = "__atomic_fetch_sub_2", .linkage = linkage }); | 271 | @export(fetchFn(u8, .Sub), .{ .name = "__atomic_fetch_sub_1", .linkage = linkage }); |
| 261 | @export(fetchFn(u32, .Sub), .{ .name = "__atomic_fetch_sub_4", .linkage = linkage }); | 272 | @export(fetchFn(u16, .Sub), .{ .name = "__atomic_fetch_sub_2", .linkage = linkage }); |
| 262 | @export(fetchFn(u64, .Sub), .{ .name = "__atomic_fetch_sub_8", .linkage = linkage }); | 273 | @export(fetchFn(u32, .Sub), .{ .name = "__atomic_fetch_sub_4", .linkage = linkage }); |
| 263 | | 274 | @export(fetchFn(u64, .Sub), .{ .name = "__atomic_fetch_sub_8", .linkage = linkage }); |
| 264 | @export(fetchFn(u8, .And), .{ .name = "__atomic_fetch_and_1", .linkage = linkage }); | 275 | |
| 265 | @export(fetchFn(u16, .And), .{ .name = "__atomic_fetch_and_2", .linkage = linkage }); | 276 | @export(fetchFn(u8, .And), .{ .name = "__atomic_fetch_and_1", .linkage = linkage }); |
| 266 | @export(fetchFn(u32, .And), .{ .name = "__atomic_fetch_and_4", .linkage = linkage }); | 277 | @export(fetchFn(u16, .And), .{ .name = "__atomic_fetch_and_2", .linkage = linkage }); |
| 267 | @export(fetchFn(u64, .And), .{ .name = "__atomic_fetch_and_8", .linkage = linkage }); | 278 | @export(fetchFn(u32, .And), .{ .name = "__atomic_fetch_and_4", .linkage = linkage }); |
| 268 | | 279 | @export(fetchFn(u64, .And), .{ .name = "__atomic_fetch_and_8", .linkage = linkage }); |
| 269 | @export(fetchFn(u8, .Or), .{ .name = "__atomic_fetch_or_1", .linkage = linkage }); | 280 | |
| 270 | @export(fetchFn(u16, .Or), .{ .name = "__atomic_fetch_or_2", .linkage = linkage }); | 281 | @export(fetchFn(u8, .Or), .{ .name = "__atomic_fetch_or_1", .linkage = linkage }); |
| 271 | @export(fetchFn(u32, .Or), .{ .name = "__atomic_fetch_or_4", .linkage = linkage }); | 282 | @export(fetchFn(u16, .Or), .{ .name = "__atomic_fetch_or_2", .linkage = linkage }); |
| 272 | @export(fetchFn(u64, .Or), .{ .name = "__atomic_fetch_or_8", .linkage = linkage }); | 283 | @export(fetchFn(u32, .Or), .{ .name = "__atomic_fetch_or_4", .linkage = linkage }); |
| 273 | | 284 | @export(fetchFn(u64, .Or), .{ .name = "__atomic_fetch_or_8", .linkage = linkage }); |
| 274 | @export(fetchFn(u8, .Xor), .{ .name = "__atomic_fetch_xor_1", .linkage = linkage }); | 285 | |
| 275 | @export(fetchFn(u16, .Xor), .{ .name = "__atomic_fetch_xor_2", .linkage = linkage }); | 286 | @export(fetchFn(u8, .Xor), .{ .name = "__atomic_fetch_xor_1", .linkage = linkage }); |
| 276 | @export(fetchFn(u32, .Xor), .{ .name = "__atomic_fetch_xor_4", .linkage = linkage }); | 287 | @export(fetchFn(u16, .Xor), .{ .name = "__atomic_fetch_xor_2", .linkage = linkage }); |
| 277 | @export(fetchFn(u64, .Xor), .{ .name = "__atomic_fetch_xor_8", .linkage = linkage }); | 288 | @export(fetchFn(u32, .Xor), .{ .name = "__atomic_fetch_xor_4", .linkage = linkage }); |
| 278 | | 289 | @export(fetchFn(u64, .Xor), .{ .name = "__atomic_fetch_xor_8", .linkage = linkage }); |
| 279 | @export(fetchFn(u8, .Nand), .{ .name = "__atomic_fetch_nand_1", .linkage = linkage }); | 290 | |
| 280 | @export(fetchFn(u16, .Nand), .{ .name = "__atomic_fetch_nand_2", .linkage = linkage }); | 291 | @export(fetchFn(u8, .Nand), .{ .name = "__atomic_fetch_nand_1", .linkage = linkage }); |
| 281 | @export(fetchFn(u32, .Nand), .{ .name = "__atomic_fetch_nand_4", .linkage = linkage }); | 292 | @export(fetchFn(u16, .Nand), .{ .name = "__atomic_fetch_nand_2", .linkage = linkage }); |
| 282 | @export(fetchFn(u64, .Nand), .{ .name = "__atomic_fetch_nand_8", .linkage = linkage }); | 293 | @export(fetchFn(u32, .Nand), .{ .name = "__atomic_fetch_nand_4", .linkage = linkage }); |
| | 294 | @export(fetchFn(u64, .Nand), .{ .name = "__atomic_fetch_nand_8", .linkage = linkage }); |
| | 295 | } |
| 283 | } | 296 | } |