authorgravatar for drsingh2518@icloud.comDevin Singh <drsingh2518@icloud.com> 2022-06-08 19:31:55-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-21 17:17:02-07:00
loga8f2d00ec461522ce090fcfc09c80f013a70c51d
tree0fa670292d1f3d9d63429da6de7e702d7149c26d
parent7c527c6dfe590a1251b51f6e1cfe8a4ba9bb0d67

compiler_rt: add outlined lse atomics for aarch64

3 files changed, 11030 insertions(+), 0 deletions(-)

lib/compiler_rt.zig+1
...@@ -2,6 +2,7 @@ pub const panic = @import("compiler_rt/common.zig").panic;...@@ -2,6 +2,7 @@ pub const panic = @import("compiler_rt/common.zig").panic;
22
3comptime {3comptime {
4 _ = @import("compiler_rt/atomics.zig");4 _ = @import("compiler_rt/atomics.zig");
5 _ = @import("compiler_rt/lse_atomics.zig");
56
6 _ = @import("compiler_rt/addf3.zig");7 _ = @import("compiler_rt/addf3.zig");
7 _ = @import("compiler_rt/addhf3.zig");8 _ = @import("compiler_rt/addhf3.zig");
lib/compiler_rt/lse_atomics.zig created+10435
...@@ -0,0 +1,10435 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const arch = builtin.cpu.arch;
4const is_test = builtin.is_test;
5const target = std.Target;
6const os_tag = builtin.os.tag;
7const is_darwin = target.Os.Tag.isDarwin(os_tag);
8const has_lse = target.aarch64.featureSetHas(builtin.target.cpu.features, .lse);
9const linkage = if (is_test)
10 std.builtin.GlobalLinkage.Internal
11else
12 std.builtin.GlobalLinkage.Strong;
13
14fn cas1RelaxDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
15 @setRuntimeSafety(false);
16 __init_aarch64_have_lse_atomics();
17
18 return asm volatile (
19 \\ cbz w16, 8f
20 \\ casb w0, w1, [x2]
21 \\
22 \\ cbz wzr, 1f
23 \\8:
24 \\ uxtb w16, w0
25 \\0:
26 \\ ldxrb w0, [x2]
27 \\ cmp w0, w16
28 \\ bne 1f
29 \\ stxrb w17, w1, [x2]
30 \\ cbnz w17, 0b
31 \\1:
32 : [ret] "={w0}" (-> u8),
33 : [expected] "{w0}" (expected),
34 [desired] "{w1}" (desired),
35 [ptr] "{x2}" (ptr),
36 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
37 : "w15", "w16", "w17", "memory"
38 );
39}
40fn cas1RelaxDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
41 @setRuntimeSafety(false);
42 __init_aarch64_have_lse_atomics();
43
44 return asm volatile (
45 \\ cbz w16, 8f
46 \\ .inst 0x08a07c41 + 0x00000000 + 0x000000
47 \\
48 \\ cbz wzr, 1f
49 \\8:
50 \\ uxtb w16, w0
51 \\0:
52 \\ ldxrb w0, [x2]
53 \\ cmp w0, w16
54 \\ bne 1f
55 \\ stxrb w17, w1, [x2]
56 \\ cbnz w17, 0b
57 \\1:
58 : [ret] "={w0}" (-> u8),
59 : [expected] "{w0}" (expected),
60 [desired] "{w1}" (desired),
61 [ptr] "{x2}" (ptr),
62 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
63 : "w15", "w16", "w17", "memory"
64 );
65}
66fn cas1RelaxNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
67 @setRuntimeSafety(false);
68 __init_aarch64_have_lse_atomics();
69
70 return asm volatile (
71 \\ cbz w16, 8f
72 \\ casb w0, w1, [x2]
73 \\
74 \\ cbz wzr, 1f
75 \\8:
76 \\ uxtb w16, w0
77 \\0:
78 \\ ldxrb w0, [x2]
79 \\ cmp w0, w16
80 \\ bne 1f
81 \\ stxrb w17, w1, [x2]
82 \\ cbnz w17, 0b
83 \\1:
84 : [ret] "={w0}" (-> u8),
85 : [expected] "{w0}" (expected),
86 [desired] "{w1}" (desired),
87 [ptr] "{x2}" (ptr),
88 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
89 : "w15", "w16", "w17", "memory"
90 );
91}
92fn cas1RelaxNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
93 @setRuntimeSafety(false);
94 __init_aarch64_have_lse_atomics();
95
96 return asm volatile (
97 \\ cbz w16, 8f
98 \\ .inst 0x08a07c41 + 0x00000000 + 0x000000
99 \\
100 \\ cbz wzr, 1f
101 \\8:
102 \\ uxtb w16, w0
103 \\0:
104 \\ ldxrb w0, [x2]
105 \\ cmp w0, w16
106 \\ bne 1f
107 \\ stxrb w17, w1, [x2]
108 \\ cbnz w17, 0b
109 \\1:
110 : [ret] "={w0}" (-> u8),
111 : [expected] "{w0}" (expected),
112 [desired] "{w1}" (desired),
113 [ptr] "{x2}" (ptr),
114 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
115 : "w15", "w16", "w17", "memory"
116 );
117}
118const __aarch64_cas1_relax = if (is_darwin)
119 if (has_lse)
120 cas1RelaxDarwinLse
121 else
122 cas1RelaxDarwinNolse
123else if (has_lse)
124 cas1RelaxNondarwinLse
125else
126 cas1RelaxNondarwinNolse;
127fn swp1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
128 @setRuntimeSafety(false);
129 __init_aarch64_have_lse_atomics();
130
131 return asm volatile (
132 \\ cbz w16, 8f
133 \\ swpb w0, w0, [x1]
134 \\ cbz wzr, 1f
135 \\8:
136 \\ mov w16, w0
137 \\0:
138 \\ ldxrb w0, [x1]
139 \\ stxrb w17, w16, [x1]
140 \\ cbnz w17, 0b
141 \\1:
142 : [ret] "={w0}" (-> u8),
143 : [val] "{w0}" (val),
144 [ptr] "{x1}" (ptr),
145 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
146 : "w15", "w16", "w17", "memory"
147 );
148}
149fn swp1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
150 @setRuntimeSafety(false);
151 __init_aarch64_have_lse_atomics();
152
153 return asm volatile (
154 \\ cbz w16, 8f
155 \\ .inst 0x38208020 + 0x00000000 + 0x000000
156 \\ cbz wzr, 1f
157 \\8:
158 \\ mov w16, w0
159 \\0:
160 \\ ldxrb w0, [x1]
161 \\ stxrb w17, w16, [x1]
162 \\ cbnz w17, 0b
163 \\1:
164 : [ret] "={w0}" (-> u8),
165 : [val] "{w0}" (val),
166 [ptr] "{x1}" (ptr),
167 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
168 : "w15", "w16", "w17", "memory"
169 );
170}
171fn swp1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
172 @setRuntimeSafety(false);
173 __init_aarch64_have_lse_atomics();
174
175 return asm volatile (
176 \\ cbz w16, 8f
177 \\ swpb w0, w0, [x1]
178 \\ cbz wzr, 1f
179 \\8:
180 \\ mov w16, w0
181 \\0:
182 \\ ldxrb w0, [x1]
183 \\ stxrb w17, w16, [x1]
184 \\ cbnz w17, 0b
185 \\1:
186 : [ret] "={w0}" (-> u8),
187 : [val] "{w0}" (val),
188 [ptr] "{x1}" (ptr),
189 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
190 : "w15", "w16", "w17", "memory"
191 );
192}
193fn swp1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
194 @setRuntimeSafety(false);
195 __init_aarch64_have_lse_atomics();
196
197 return asm volatile (
198 \\ cbz w16, 8f
199 \\ .inst 0x38208020 + 0x00000000 + 0x000000
200 \\ cbz wzr, 1f
201 \\8:
202 \\ mov w16, w0
203 \\0:
204 \\ ldxrb w0, [x1]
205 \\ stxrb w17, w16, [x1]
206 \\ cbnz w17, 0b
207 \\1:
208 : [ret] "={w0}" (-> u8),
209 : [val] "{w0}" (val),
210 [ptr] "{x1}" (ptr),
211 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
212 : "w15", "w16", "w17", "memory"
213 );
214}
215const __aarch64_swp1_relax = if (is_darwin)
216 if (has_lse)
217 swp1RelaxDarwinLse
218 else
219 swp1RelaxDarwinNolse
220else if (has_lse)
221 swp1RelaxNondarwinLse
222else
223 swp1RelaxNondarwinNolse;
224fn ldadd1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
225 @setRuntimeSafety(false);
226 __init_aarch64_have_lse_atomics();
227
228 return asm volatile (
229 \\ cbz w16, 8f
230 \\ ldaddb w0, w0, [x1]
231 \\ cbz wzr, 1f
232 \\8:
233 \\ mov w16, w0
234 \\0:
235 \\ ldxrb w0, [x1]
236 \\ add w17, w0, w16
237 \\ stxrb w15, w17, [x1]
238 \\ cbnz w15, 0b
239 \\1:
240 : [ret] "={w0}" (-> u8),
241 : [val] "{w0}" (val),
242 [ptr] "{x1}" (ptr),
243 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
244 : "w15", "w16", "w17", "memory"
245 );
246}
247fn ldadd1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
248 @setRuntimeSafety(false);
249 __init_aarch64_have_lse_atomics();
250
251 return asm volatile (
252 \\ cbz w16, 8f
253 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
254 \\ cbz wzr, 1f
255 \\8:
256 \\ mov w16, w0
257 \\0:
258 \\ ldxrb w0, [x1]
259 \\ add w17, w0, w16
260 \\ stxrb w15, w17, [x1]
261 \\ cbnz w15, 0b
262 \\1:
263 : [ret] "={w0}" (-> u8),
264 : [val] "{w0}" (val),
265 [ptr] "{x1}" (ptr),
266 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
267 : "w15", "w16", "w17", "memory"
268 );
269}
270fn ldadd1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
271 @setRuntimeSafety(false);
272 __init_aarch64_have_lse_atomics();
273
274 return asm volatile (
275 \\ cbz w16, 8f
276 \\ ldaddb w0, w0, [x1]
277 \\ cbz wzr, 1f
278 \\8:
279 \\ mov w16, w0
280 \\0:
281 \\ ldxrb w0, [x1]
282 \\ add w17, w0, w16
283 \\ stxrb w15, w17, [x1]
284 \\ cbnz w15, 0b
285 \\1:
286 : [ret] "={w0}" (-> u8),
287 : [val] "{w0}" (val),
288 [ptr] "{x1}" (ptr),
289 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
290 : "w15", "w16", "w17", "memory"
291 );
292}
293fn ldadd1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
294 @setRuntimeSafety(false);
295 __init_aarch64_have_lse_atomics();
296
297 return asm volatile (
298 \\ cbz w16, 8f
299 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
300 \\ cbz wzr, 1f
301 \\8:
302 \\ mov w16, w0
303 \\0:
304 \\ ldxrb w0, [x1]
305 \\ add w17, w0, w16
306 \\ stxrb w15, w17, [x1]
307 \\ cbnz w15, 0b
308 \\1:
309 : [ret] "={w0}" (-> u8),
310 : [val] "{w0}" (val),
311 [ptr] "{x1}" (ptr),
312 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
313 : "w15", "w16", "w17", "memory"
314 );
315}
316const __aarch64_ldadd1_relax = if (is_darwin)
317 if (has_lse)
318 ldadd1RelaxDarwinLse
319 else
320 ldadd1RelaxDarwinNolse
321else if (has_lse)
322 ldadd1RelaxNondarwinLse
323else
324 ldadd1RelaxNondarwinNolse;
325fn ldclr1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
326 @setRuntimeSafety(false);
327 __init_aarch64_have_lse_atomics();
328
329 return asm volatile (
330 \\ cbz w16, 8f
331 \\ ldclrb w0, w0, [x1]
332 \\ cbz wzr, 1f
333 \\8:
334 \\ mov w16, w0
335 \\0:
336 \\ ldxrb w0, [x1]
337 \\ bic w17, w0, w16
338 \\ stxrb w15, w17, [x1]
339 \\ cbnz w15, 0b
340 \\1:
341 : [ret] "={w0}" (-> u8),
342 : [val] "{w0}" (val),
343 [ptr] "{x1}" (ptr),
344 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
345 : "w15", "w16", "w17", "memory"
346 );
347}
348fn ldclr1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
349 @setRuntimeSafety(false);
350 __init_aarch64_have_lse_atomics();
351
352 return asm volatile (
353 \\ cbz w16, 8f
354 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
355 \\ cbz wzr, 1f
356 \\8:
357 \\ mov w16, w0
358 \\0:
359 \\ ldxrb w0, [x1]
360 \\ bic w17, w0, w16
361 \\ stxrb w15, w17, [x1]
362 \\ cbnz w15, 0b
363 \\1:
364 : [ret] "={w0}" (-> u8),
365 : [val] "{w0}" (val),
366 [ptr] "{x1}" (ptr),
367 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
368 : "w15", "w16", "w17", "memory"
369 );
370}
371fn ldclr1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
372 @setRuntimeSafety(false);
373 __init_aarch64_have_lse_atomics();
374
375 return asm volatile (
376 \\ cbz w16, 8f
377 \\ ldclrb w0, w0, [x1]
378 \\ cbz wzr, 1f
379 \\8:
380 \\ mov w16, w0
381 \\0:
382 \\ ldxrb w0, [x1]
383 \\ bic w17, w0, w16
384 \\ stxrb w15, w17, [x1]
385 \\ cbnz w15, 0b
386 \\1:
387 : [ret] "={w0}" (-> u8),
388 : [val] "{w0}" (val),
389 [ptr] "{x1}" (ptr),
390 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
391 : "w15", "w16", "w17", "memory"
392 );
393}
394fn ldclr1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
395 @setRuntimeSafety(false);
396 __init_aarch64_have_lse_atomics();
397
398 return asm volatile (
399 \\ cbz w16, 8f
400 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
401 \\ cbz wzr, 1f
402 \\8:
403 \\ mov w16, w0
404 \\0:
405 \\ ldxrb w0, [x1]
406 \\ bic w17, w0, w16
407 \\ stxrb w15, w17, [x1]
408 \\ cbnz w15, 0b
409 \\1:
410 : [ret] "={w0}" (-> u8),
411 : [val] "{w0}" (val),
412 [ptr] "{x1}" (ptr),
413 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
414 : "w15", "w16", "w17", "memory"
415 );
416}
417const __aarch64_ldclr1_relax = if (is_darwin)
418 if (has_lse)
419 ldclr1RelaxDarwinLse
420 else
421 ldclr1RelaxDarwinNolse
422else if (has_lse)
423 ldclr1RelaxNondarwinLse
424else
425 ldclr1RelaxNondarwinNolse;
426fn ldeor1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
427 @setRuntimeSafety(false);
428 __init_aarch64_have_lse_atomics();
429
430 return asm volatile (
431 \\ cbz w16, 8f
432 \\ ldeorb w0, w0, [x1]
433 \\ cbz wzr, 1f
434 \\8:
435 \\ mov w16, w0
436 \\0:
437 \\ ldxrb w0, [x1]
438 \\ eor w17, w0, w16
439 \\ stxrb w15, w17, [x1]
440 \\ cbnz w15, 0b
441 \\1:
442 : [ret] "={w0}" (-> u8),
443 : [val] "{w0}" (val),
444 [ptr] "{x1}" (ptr),
445 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
446 : "w15", "w16", "w17", "memory"
447 );
448}
449fn ldeor1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
450 @setRuntimeSafety(false);
451 __init_aarch64_have_lse_atomics();
452
453 return asm volatile (
454 \\ cbz w16, 8f
455 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
456 \\ cbz wzr, 1f
457 \\8:
458 \\ mov w16, w0
459 \\0:
460 \\ ldxrb w0, [x1]
461 \\ eor w17, w0, w16
462 \\ stxrb w15, w17, [x1]
463 \\ cbnz w15, 0b
464 \\1:
465 : [ret] "={w0}" (-> u8),
466 : [val] "{w0}" (val),
467 [ptr] "{x1}" (ptr),
468 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
469 : "w15", "w16", "w17", "memory"
470 );
471}
472fn ldeor1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
473 @setRuntimeSafety(false);
474 __init_aarch64_have_lse_atomics();
475
476 return asm volatile (
477 \\ cbz w16, 8f
478 \\ ldeorb w0, w0, [x1]
479 \\ cbz wzr, 1f
480 \\8:
481 \\ mov w16, w0
482 \\0:
483 \\ ldxrb w0, [x1]
484 \\ eor w17, w0, w16
485 \\ stxrb w15, w17, [x1]
486 \\ cbnz w15, 0b
487 \\1:
488 : [ret] "={w0}" (-> u8),
489 : [val] "{w0}" (val),
490 [ptr] "{x1}" (ptr),
491 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
492 : "w15", "w16", "w17", "memory"
493 );
494}
495fn ldeor1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
496 @setRuntimeSafety(false);
497 __init_aarch64_have_lse_atomics();
498
499 return asm volatile (
500 \\ cbz w16, 8f
501 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
502 \\ cbz wzr, 1f
503 \\8:
504 \\ mov w16, w0
505 \\0:
506 \\ ldxrb w0, [x1]
507 \\ eor w17, w0, w16
508 \\ stxrb w15, w17, [x1]
509 \\ cbnz w15, 0b
510 \\1:
511 : [ret] "={w0}" (-> u8),
512 : [val] "{w0}" (val),
513 [ptr] "{x1}" (ptr),
514 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
515 : "w15", "w16", "w17", "memory"
516 );
517}
518const __aarch64_ldeor1_relax = if (is_darwin)
519 if (has_lse)
520 ldeor1RelaxDarwinLse
521 else
522 ldeor1RelaxDarwinNolse
523else if (has_lse)
524 ldeor1RelaxNondarwinLse
525else
526 ldeor1RelaxNondarwinNolse;
527fn ldset1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
528 @setRuntimeSafety(false);
529 __init_aarch64_have_lse_atomics();
530
531 return asm volatile (
532 \\ cbz w16, 8f
533 \\ ldsetb w0, w0, [x1]
534 \\ cbz wzr, 1f
535 \\8:
536 \\ mov w16, w0
537 \\0:
538 \\ ldxrb w0, [x1]
539 \\ orr w17, w0, w16
540 \\ stxrb w15, w17, [x1]
541 \\ cbnz w15, 0b
542 \\1:
543 : [ret] "={w0}" (-> u8),
544 : [val] "{w0}" (val),
545 [ptr] "{x1}" (ptr),
546 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
547 : "w15", "w16", "w17", "memory"
548 );
549}
550fn ldset1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
551 @setRuntimeSafety(false);
552 __init_aarch64_have_lse_atomics();
553
554 return asm volatile (
555 \\ cbz w16, 8f
556 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
557 \\ cbz wzr, 1f
558 \\8:
559 \\ mov w16, w0
560 \\0:
561 \\ ldxrb w0, [x1]
562 \\ orr w17, w0, w16
563 \\ stxrb w15, w17, [x1]
564 \\ cbnz w15, 0b
565 \\1:
566 : [ret] "={w0}" (-> u8),
567 : [val] "{w0}" (val),
568 [ptr] "{x1}" (ptr),
569 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
570 : "w15", "w16", "w17", "memory"
571 );
572}
573fn ldset1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
574 @setRuntimeSafety(false);
575 __init_aarch64_have_lse_atomics();
576
577 return asm volatile (
578 \\ cbz w16, 8f
579 \\ ldsetb w0, w0, [x1]
580 \\ cbz wzr, 1f
581 \\8:
582 \\ mov w16, w0
583 \\0:
584 \\ ldxrb w0, [x1]
585 \\ orr w17, w0, w16
586 \\ stxrb w15, w17, [x1]
587 \\ cbnz w15, 0b
588 \\1:
589 : [ret] "={w0}" (-> u8),
590 : [val] "{w0}" (val),
591 [ptr] "{x1}" (ptr),
592 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
593 : "w15", "w16", "w17", "memory"
594 );
595}
596fn ldset1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
597 @setRuntimeSafety(false);
598 __init_aarch64_have_lse_atomics();
599
600 return asm volatile (
601 \\ cbz w16, 8f
602 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
603 \\ cbz wzr, 1f
604 \\8:
605 \\ mov w16, w0
606 \\0:
607 \\ ldxrb w0, [x1]
608 \\ orr w17, w0, w16
609 \\ stxrb w15, w17, [x1]
610 \\ cbnz w15, 0b
611 \\1:
612 : [ret] "={w0}" (-> u8),
613 : [val] "{w0}" (val),
614 [ptr] "{x1}" (ptr),
615 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
616 : "w15", "w16", "w17", "memory"
617 );
618}
619const __aarch64_ldset1_relax = if (is_darwin)
620 if (has_lse)
621 ldset1RelaxDarwinLse
622 else
623 ldset1RelaxDarwinNolse
624else if (has_lse)
625 ldset1RelaxNondarwinLse
626else
627 ldset1RelaxNondarwinNolse;
628fn cas1AcqDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
629 @setRuntimeSafety(false);
630 __init_aarch64_have_lse_atomics();
631
632 return asm volatile (
633 \\ cbz w16, 8f
634 \\ casab w0, w1, [x2]
635 \\
636 \\ cbz wzr, 1f
637 \\8:
638 \\ uxtb w16, w0
639 \\0:
640 \\ ldaxrb w0, [x2]
641 \\ cmp w0, w16
642 \\ bne 1f
643 \\ stxrb w17, w1, [x2]
644 \\ cbnz w17, 0b
645 \\1:
646 : [ret] "={w0}" (-> u8),
647 : [expected] "{w0}" (expected),
648 [desired] "{w1}" (desired),
649 [ptr] "{x2}" (ptr),
650 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
651 : "w15", "w16", "w17", "memory"
652 );
653}
654fn cas1AcqDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
655 @setRuntimeSafety(false);
656 __init_aarch64_have_lse_atomics();
657
658 return asm volatile (
659 \\ cbz w16, 8f
660 \\ .inst 0x08a07c41 + 0x00000000 + 0x400000
661 \\
662 \\ cbz wzr, 1f
663 \\8:
664 \\ uxtb w16, w0
665 \\0:
666 \\ ldaxrb w0, [x2]
667 \\ cmp w0, w16
668 \\ bne 1f
669 \\ stxrb w17, w1, [x2]
670 \\ cbnz w17, 0b
671 \\1:
672 : [ret] "={w0}" (-> u8),
673 : [expected] "{w0}" (expected),
674 [desired] "{w1}" (desired),
675 [ptr] "{x2}" (ptr),
676 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
677 : "w15", "w16", "w17", "memory"
678 );
679}
680fn cas1AcqNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
681 @setRuntimeSafety(false);
682 __init_aarch64_have_lse_atomics();
683
684 return asm volatile (
685 \\ cbz w16, 8f
686 \\ casab w0, w1, [x2]
687 \\
688 \\ cbz wzr, 1f
689 \\8:
690 \\ uxtb w16, w0
691 \\0:
692 \\ ldaxrb w0, [x2]
693 \\ cmp w0, w16
694 \\ bne 1f
695 \\ stxrb w17, w1, [x2]
696 \\ cbnz w17, 0b
697 \\1:
698 : [ret] "={w0}" (-> u8),
699 : [expected] "{w0}" (expected),
700 [desired] "{w1}" (desired),
701 [ptr] "{x2}" (ptr),
702 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
703 : "w15", "w16", "w17", "memory"
704 );
705}
706fn cas1AcqNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
707 @setRuntimeSafety(false);
708 __init_aarch64_have_lse_atomics();
709
710 return asm volatile (
711 \\ cbz w16, 8f
712 \\ .inst 0x08a07c41 + 0x00000000 + 0x400000
713 \\
714 \\ cbz wzr, 1f
715 \\8:
716 \\ uxtb w16, w0
717 \\0:
718 \\ ldaxrb w0, [x2]
719 \\ cmp w0, w16
720 \\ bne 1f
721 \\ stxrb w17, w1, [x2]
722 \\ cbnz w17, 0b
723 \\1:
724 : [ret] "={w0}" (-> u8),
725 : [expected] "{w0}" (expected),
726 [desired] "{w1}" (desired),
727 [ptr] "{x2}" (ptr),
728 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
729 : "w15", "w16", "w17", "memory"
730 );
731}
732const __aarch64_cas1_acq = if (is_darwin)
733 if (has_lse)
734 cas1AcqDarwinLse
735 else
736 cas1AcqDarwinNolse
737else if (has_lse)
738 cas1AcqNondarwinLse
739else
740 cas1AcqNondarwinNolse;
741fn swp1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
742 @setRuntimeSafety(false);
743 __init_aarch64_have_lse_atomics();
744
745 return asm volatile (
746 \\ cbz w16, 8f
747 \\ swpab w0, w0, [x1]
748 \\ cbz wzr, 1f
749 \\8:
750 \\ mov w16, w0
751 \\0:
752 \\ ldaxrb w0, [x1]
753 \\ stxrb w17, w16, [x1]
754 \\ cbnz w17, 0b
755 \\1:
756 : [ret] "={w0}" (-> u8),
757 : [val] "{w0}" (val),
758 [ptr] "{x1}" (ptr),
759 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
760 : "w15", "w16", "w17", "memory"
761 );
762}
763fn swp1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
764 @setRuntimeSafety(false);
765 __init_aarch64_have_lse_atomics();
766
767 return asm volatile (
768 \\ cbz w16, 8f
769 \\ .inst 0x38208020 + 0x00000000 + 0x800000
770 \\ cbz wzr, 1f
771 \\8:
772 \\ mov w16, w0
773 \\0:
774 \\ ldaxrb w0, [x1]
775 \\ stxrb w17, w16, [x1]
776 \\ cbnz w17, 0b
777 \\1:
778 : [ret] "={w0}" (-> u8),
779 : [val] "{w0}" (val),
780 [ptr] "{x1}" (ptr),
781 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
782 : "w15", "w16", "w17", "memory"
783 );
784}
785fn swp1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
786 @setRuntimeSafety(false);
787 __init_aarch64_have_lse_atomics();
788
789 return asm volatile (
790 \\ cbz w16, 8f
791 \\ swpab w0, w0, [x1]
792 \\ cbz wzr, 1f
793 \\8:
794 \\ mov w16, w0
795 \\0:
796 \\ ldaxrb w0, [x1]
797 \\ stxrb w17, w16, [x1]
798 \\ cbnz w17, 0b
799 \\1:
800 : [ret] "={w0}" (-> u8),
801 : [val] "{w0}" (val),
802 [ptr] "{x1}" (ptr),
803 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
804 : "w15", "w16", "w17", "memory"
805 );
806}
807fn swp1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
808 @setRuntimeSafety(false);
809 __init_aarch64_have_lse_atomics();
810
811 return asm volatile (
812 \\ cbz w16, 8f
813 \\ .inst 0x38208020 + 0x00000000 + 0x800000
814 \\ cbz wzr, 1f
815 \\8:
816 \\ mov w16, w0
817 \\0:
818 \\ ldaxrb w0, [x1]
819 \\ stxrb w17, w16, [x1]
820 \\ cbnz w17, 0b
821 \\1:
822 : [ret] "={w0}" (-> u8),
823 : [val] "{w0}" (val),
824 [ptr] "{x1}" (ptr),
825 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
826 : "w15", "w16", "w17", "memory"
827 );
828}
829const __aarch64_swp1_acq = if (is_darwin)
830 if (has_lse)
831 swp1AcqDarwinLse
832 else
833 swp1AcqDarwinNolse
834else if (has_lse)
835 swp1AcqNondarwinLse
836else
837 swp1AcqNondarwinNolse;
838fn ldadd1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
839 @setRuntimeSafety(false);
840 __init_aarch64_have_lse_atomics();
841
842 return asm volatile (
843 \\ cbz w16, 8f
844 \\ ldaddab w0, w0, [x1]
845 \\ cbz wzr, 1f
846 \\8:
847 \\ mov w16, w0
848 \\0:
849 \\ ldaxrb w0, [x1]
850 \\ add w17, w0, w16
851 \\ stxrb w15, w17, [x1]
852 \\ cbnz w15, 0b
853 \\1:
854 : [ret] "={w0}" (-> u8),
855 : [val] "{w0}" (val),
856 [ptr] "{x1}" (ptr),
857 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
858 : "w15", "w16", "w17", "memory"
859 );
860}
861fn ldadd1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
862 @setRuntimeSafety(false);
863 __init_aarch64_have_lse_atomics();
864
865 return asm volatile (
866 \\ cbz w16, 8f
867 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
868 \\ cbz wzr, 1f
869 \\8:
870 \\ mov w16, w0
871 \\0:
872 \\ ldaxrb w0, [x1]
873 \\ add w17, w0, w16
874 \\ stxrb w15, w17, [x1]
875 \\ cbnz w15, 0b
876 \\1:
877 : [ret] "={w0}" (-> u8),
878 : [val] "{w0}" (val),
879 [ptr] "{x1}" (ptr),
880 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
881 : "w15", "w16", "w17", "memory"
882 );
883}
884fn ldadd1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
885 @setRuntimeSafety(false);
886 __init_aarch64_have_lse_atomics();
887
888 return asm volatile (
889 \\ cbz w16, 8f
890 \\ ldaddab w0, w0, [x1]
891 \\ cbz wzr, 1f
892 \\8:
893 \\ mov w16, w0
894 \\0:
895 \\ ldaxrb w0, [x1]
896 \\ add w17, w0, w16
897 \\ stxrb w15, w17, [x1]
898 \\ cbnz w15, 0b
899 \\1:
900 : [ret] "={w0}" (-> u8),
901 : [val] "{w0}" (val),
902 [ptr] "{x1}" (ptr),
903 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
904 : "w15", "w16", "w17", "memory"
905 );
906}
907fn ldadd1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
908 @setRuntimeSafety(false);
909 __init_aarch64_have_lse_atomics();
910
911 return asm volatile (
912 \\ cbz w16, 8f
913 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
914 \\ cbz wzr, 1f
915 \\8:
916 \\ mov w16, w0
917 \\0:
918 \\ ldaxrb w0, [x1]
919 \\ add w17, w0, w16
920 \\ stxrb w15, w17, [x1]
921 \\ cbnz w15, 0b
922 \\1:
923 : [ret] "={w0}" (-> u8),
924 : [val] "{w0}" (val),
925 [ptr] "{x1}" (ptr),
926 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
927 : "w15", "w16", "w17", "memory"
928 );
929}
930const __aarch64_ldadd1_acq = if (is_darwin)
931 if (has_lse)
932 ldadd1AcqDarwinLse
933 else
934 ldadd1AcqDarwinNolse
935else if (has_lse)
936 ldadd1AcqNondarwinLse
937else
938 ldadd1AcqNondarwinNolse;
939fn ldclr1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
940 @setRuntimeSafety(false);
941 __init_aarch64_have_lse_atomics();
942
943 return asm volatile (
944 \\ cbz w16, 8f
945 \\ ldclrab w0, w0, [x1]
946 \\ cbz wzr, 1f
947 \\8:
948 \\ mov w16, w0
949 \\0:
950 \\ ldaxrb w0, [x1]
951 \\ bic w17, w0, w16
952 \\ stxrb w15, w17, [x1]
953 \\ cbnz w15, 0b
954 \\1:
955 : [ret] "={w0}" (-> u8),
956 : [val] "{w0}" (val),
957 [ptr] "{x1}" (ptr),
958 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
959 : "w15", "w16", "w17", "memory"
960 );
961}
962fn ldclr1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
963 @setRuntimeSafety(false);
964 __init_aarch64_have_lse_atomics();
965
966 return asm volatile (
967 \\ cbz w16, 8f
968 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
969 \\ cbz wzr, 1f
970 \\8:
971 \\ mov w16, w0
972 \\0:
973 \\ ldaxrb w0, [x1]
974 \\ bic w17, w0, w16
975 \\ stxrb w15, w17, [x1]
976 \\ cbnz w15, 0b
977 \\1:
978 : [ret] "={w0}" (-> u8),
979 : [val] "{w0}" (val),
980 [ptr] "{x1}" (ptr),
981 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
982 : "w15", "w16", "w17", "memory"
983 );
984}
985fn ldclr1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
986 @setRuntimeSafety(false);
987 __init_aarch64_have_lse_atomics();
988
989 return asm volatile (
990 \\ cbz w16, 8f
991 \\ ldclrab w0, w0, [x1]
992 \\ cbz wzr, 1f
993 \\8:
994 \\ mov w16, w0
995 \\0:
996 \\ ldaxrb w0, [x1]
997 \\ bic w17, w0, w16
998 \\ stxrb w15, w17, [x1]
999 \\ cbnz w15, 0b
1000 \\1:
1001 : [ret] "={w0}" (-> u8),
1002 : [val] "{w0}" (val),
1003 [ptr] "{x1}" (ptr),
1004 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1005 : "w15", "w16", "w17", "memory"
1006 );
1007}
1008fn ldclr1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1009 @setRuntimeSafety(false);
1010 __init_aarch64_have_lse_atomics();
1011
1012 return asm volatile (
1013 \\ cbz w16, 8f
1014 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
1015 \\ cbz wzr, 1f
1016 \\8:
1017 \\ mov w16, w0
1018 \\0:
1019 \\ ldaxrb w0, [x1]
1020 \\ bic w17, w0, w16
1021 \\ stxrb w15, w17, [x1]
1022 \\ cbnz w15, 0b
1023 \\1:
1024 : [ret] "={w0}" (-> u8),
1025 : [val] "{w0}" (val),
1026 [ptr] "{x1}" (ptr),
1027 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1028 : "w15", "w16", "w17", "memory"
1029 );
1030}
1031const __aarch64_ldclr1_acq = if (is_darwin)
1032 if (has_lse)
1033 ldclr1AcqDarwinLse
1034 else
1035 ldclr1AcqDarwinNolse
1036else if (has_lse)
1037 ldclr1AcqNondarwinLse
1038else
1039 ldclr1AcqNondarwinNolse;
1040fn ldeor1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1041 @setRuntimeSafety(false);
1042 __init_aarch64_have_lse_atomics();
1043
1044 return asm volatile (
1045 \\ cbz w16, 8f
1046 \\ ldeorab w0, w0, [x1]
1047 \\ cbz wzr, 1f
1048 \\8:
1049 \\ mov w16, w0
1050 \\0:
1051 \\ ldaxrb w0, [x1]
1052 \\ eor w17, w0, w16
1053 \\ stxrb w15, w17, [x1]
1054 \\ cbnz w15, 0b
1055 \\1:
1056 : [ret] "={w0}" (-> u8),
1057 : [val] "{w0}" (val),
1058 [ptr] "{x1}" (ptr),
1059 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1060 : "w15", "w16", "w17", "memory"
1061 );
1062}
1063fn ldeor1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1064 @setRuntimeSafety(false);
1065 __init_aarch64_have_lse_atomics();
1066
1067 return asm volatile (
1068 \\ cbz w16, 8f
1069 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
1070 \\ cbz wzr, 1f
1071 \\8:
1072 \\ mov w16, w0
1073 \\0:
1074 \\ ldaxrb w0, [x1]
1075 \\ eor w17, w0, w16
1076 \\ stxrb w15, w17, [x1]
1077 \\ cbnz w15, 0b
1078 \\1:
1079 : [ret] "={w0}" (-> u8),
1080 : [val] "{w0}" (val),
1081 [ptr] "{x1}" (ptr),
1082 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1083 : "w15", "w16", "w17", "memory"
1084 );
1085}
1086fn ldeor1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1087 @setRuntimeSafety(false);
1088 __init_aarch64_have_lse_atomics();
1089
1090 return asm volatile (
1091 \\ cbz w16, 8f
1092 \\ ldeorab w0, w0, [x1]
1093 \\ cbz wzr, 1f
1094 \\8:
1095 \\ mov w16, w0
1096 \\0:
1097 \\ ldaxrb w0, [x1]
1098 \\ eor w17, w0, w16
1099 \\ stxrb w15, w17, [x1]
1100 \\ cbnz w15, 0b
1101 \\1:
1102 : [ret] "={w0}" (-> u8),
1103 : [val] "{w0}" (val),
1104 [ptr] "{x1}" (ptr),
1105 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1106 : "w15", "w16", "w17", "memory"
1107 );
1108}
1109fn ldeor1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1110 @setRuntimeSafety(false);
1111 __init_aarch64_have_lse_atomics();
1112
1113 return asm volatile (
1114 \\ cbz w16, 8f
1115 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
1116 \\ cbz wzr, 1f
1117 \\8:
1118 \\ mov w16, w0
1119 \\0:
1120 \\ ldaxrb w0, [x1]
1121 \\ eor w17, w0, w16
1122 \\ stxrb w15, w17, [x1]
1123 \\ cbnz w15, 0b
1124 \\1:
1125 : [ret] "={w0}" (-> u8),
1126 : [val] "{w0}" (val),
1127 [ptr] "{x1}" (ptr),
1128 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1129 : "w15", "w16", "w17", "memory"
1130 );
1131}
1132const __aarch64_ldeor1_acq = if (is_darwin)
1133 if (has_lse)
1134 ldeor1AcqDarwinLse
1135 else
1136 ldeor1AcqDarwinNolse
1137else if (has_lse)
1138 ldeor1AcqNondarwinLse
1139else
1140 ldeor1AcqNondarwinNolse;
1141fn ldset1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1142 @setRuntimeSafety(false);
1143 __init_aarch64_have_lse_atomics();
1144
1145 return asm volatile (
1146 \\ cbz w16, 8f
1147 \\ ldsetab w0, w0, [x1]
1148 \\ cbz wzr, 1f
1149 \\8:
1150 \\ mov w16, w0
1151 \\0:
1152 \\ ldaxrb w0, [x1]
1153 \\ orr w17, w0, w16
1154 \\ stxrb w15, w17, [x1]
1155 \\ cbnz w15, 0b
1156 \\1:
1157 : [ret] "={w0}" (-> u8),
1158 : [val] "{w0}" (val),
1159 [ptr] "{x1}" (ptr),
1160 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1161 : "w15", "w16", "w17", "memory"
1162 );
1163}
1164fn ldset1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1165 @setRuntimeSafety(false);
1166 __init_aarch64_have_lse_atomics();
1167
1168 return asm volatile (
1169 \\ cbz w16, 8f
1170 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
1171 \\ cbz wzr, 1f
1172 \\8:
1173 \\ mov w16, w0
1174 \\0:
1175 \\ ldaxrb w0, [x1]
1176 \\ orr w17, w0, w16
1177 \\ stxrb w15, w17, [x1]
1178 \\ cbnz w15, 0b
1179 \\1:
1180 : [ret] "={w0}" (-> u8),
1181 : [val] "{w0}" (val),
1182 [ptr] "{x1}" (ptr),
1183 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1184 : "w15", "w16", "w17", "memory"
1185 );
1186}
1187fn ldset1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1188 @setRuntimeSafety(false);
1189 __init_aarch64_have_lse_atomics();
1190
1191 return asm volatile (
1192 \\ cbz w16, 8f
1193 \\ ldsetab w0, w0, [x1]
1194 \\ cbz wzr, 1f
1195 \\8:
1196 \\ mov w16, w0
1197 \\0:
1198 \\ ldaxrb w0, [x1]
1199 \\ orr w17, w0, w16
1200 \\ stxrb w15, w17, [x1]
1201 \\ cbnz w15, 0b
1202 \\1:
1203 : [ret] "={w0}" (-> u8),
1204 : [val] "{w0}" (val),
1205 [ptr] "{x1}" (ptr),
1206 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1207 : "w15", "w16", "w17", "memory"
1208 );
1209}
1210fn ldset1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1211 @setRuntimeSafety(false);
1212 __init_aarch64_have_lse_atomics();
1213
1214 return asm volatile (
1215 \\ cbz w16, 8f
1216 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
1217 \\ cbz wzr, 1f
1218 \\8:
1219 \\ mov w16, w0
1220 \\0:
1221 \\ ldaxrb w0, [x1]
1222 \\ orr w17, w0, w16
1223 \\ stxrb w15, w17, [x1]
1224 \\ cbnz w15, 0b
1225 \\1:
1226 : [ret] "={w0}" (-> u8),
1227 : [val] "{w0}" (val),
1228 [ptr] "{x1}" (ptr),
1229 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1230 : "w15", "w16", "w17", "memory"
1231 );
1232}
1233const __aarch64_ldset1_acq = if (is_darwin)
1234 if (has_lse)
1235 ldset1AcqDarwinLse
1236 else
1237 ldset1AcqDarwinNolse
1238else if (has_lse)
1239 ldset1AcqNondarwinLse
1240else
1241 ldset1AcqNondarwinNolse;
1242fn cas1RelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1243 @setRuntimeSafety(false);
1244 __init_aarch64_have_lse_atomics();
1245
1246 return asm volatile (
1247 \\ cbz w16, 8f
1248 \\ caslb w0, w1, [x2]
1249 \\
1250 \\ cbz wzr, 1f
1251 \\8:
1252 \\ uxtb w16, w0
1253 \\0:
1254 \\ ldxrb w0, [x2]
1255 \\ cmp w0, w16
1256 \\ bne 1f
1257 \\ stlxrb w17, w1, [x2]
1258 \\ cbnz w17, 0b
1259 \\1:
1260 : [ret] "={w0}" (-> u8),
1261 : [expected] "{w0}" (expected),
1262 [desired] "{w1}" (desired),
1263 [ptr] "{x2}" (ptr),
1264 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1265 : "w15", "w16", "w17", "memory"
1266 );
1267}
1268fn cas1RelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1269 @setRuntimeSafety(false);
1270 __init_aarch64_have_lse_atomics();
1271
1272 return asm volatile (
1273 \\ cbz w16, 8f
1274 \\ .inst 0x08a07c41 + 0x00000000 + 0x008000
1275 \\
1276 \\ cbz wzr, 1f
1277 \\8:
1278 \\ uxtb w16, w0
1279 \\0:
1280 \\ ldxrb w0, [x2]
1281 \\ cmp w0, w16
1282 \\ bne 1f
1283 \\ stlxrb w17, w1, [x2]
1284 \\ cbnz w17, 0b
1285 \\1:
1286 : [ret] "={w0}" (-> u8),
1287 : [expected] "{w0}" (expected),
1288 [desired] "{w1}" (desired),
1289 [ptr] "{x2}" (ptr),
1290 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1291 : "w15", "w16", "w17", "memory"
1292 );
1293}
1294fn cas1RelNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1295 @setRuntimeSafety(false);
1296 __init_aarch64_have_lse_atomics();
1297
1298 return asm volatile (
1299 \\ cbz w16, 8f
1300 \\ caslb w0, w1, [x2]
1301 \\
1302 \\ cbz wzr, 1f
1303 \\8:
1304 \\ uxtb w16, w0
1305 \\0:
1306 \\ ldxrb w0, [x2]
1307 \\ cmp w0, w16
1308 \\ bne 1f
1309 \\ stlxrb w17, w1, [x2]
1310 \\ cbnz w17, 0b
1311 \\1:
1312 : [ret] "={w0}" (-> u8),
1313 : [expected] "{w0}" (expected),
1314 [desired] "{w1}" (desired),
1315 [ptr] "{x2}" (ptr),
1316 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1317 : "w15", "w16", "w17", "memory"
1318 );
1319}
1320fn cas1RelNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1321 @setRuntimeSafety(false);
1322 __init_aarch64_have_lse_atomics();
1323
1324 return asm volatile (
1325 \\ cbz w16, 8f
1326 \\ .inst 0x08a07c41 + 0x00000000 + 0x008000
1327 \\
1328 \\ cbz wzr, 1f
1329 \\8:
1330 \\ uxtb w16, w0
1331 \\0:
1332 \\ ldxrb w0, [x2]
1333 \\ cmp w0, w16
1334 \\ bne 1f
1335 \\ stlxrb w17, w1, [x2]
1336 \\ cbnz w17, 0b
1337 \\1:
1338 : [ret] "={w0}" (-> u8),
1339 : [expected] "{w0}" (expected),
1340 [desired] "{w1}" (desired),
1341 [ptr] "{x2}" (ptr),
1342 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1343 : "w15", "w16", "w17", "memory"
1344 );
1345}
1346const __aarch64_cas1_rel = if (is_darwin)
1347 if (has_lse)
1348 cas1RelDarwinLse
1349 else
1350 cas1RelDarwinNolse
1351else if (has_lse)
1352 cas1RelNondarwinLse
1353else
1354 cas1RelNondarwinNolse;
1355fn swp1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1356 @setRuntimeSafety(false);
1357 __init_aarch64_have_lse_atomics();
1358
1359 return asm volatile (
1360 \\ cbz w16, 8f
1361 \\ swplb w0, w0, [x1]
1362 \\ cbz wzr, 1f
1363 \\8:
1364 \\ mov w16, w0
1365 \\0:
1366 \\ ldxrb w0, [x1]
1367 \\ stlxrb w17, w16, [x1]
1368 \\ cbnz w17, 0b
1369 \\1:
1370 : [ret] "={w0}" (-> u8),
1371 : [val] "{w0}" (val),
1372 [ptr] "{x1}" (ptr),
1373 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1374 : "w15", "w16", "w17", "memory"
1375 );
1376}
1377fn swp1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1378 @setRuntimeSafety(false);
1379 __init_aarch64_have_lse_atomics();
1380
1381 return asm volatile (
1382 \\ cbz w16, 8f
1383 \\ .inst 0x38208020 + 0x00000000 + 0x400000
1384 \\ cbz wzr, 1f
1385 \\8:
1386 \\ mov w16, w0
1387 \\0:
1388 \\ ldxrb w0, [x1]
1389 \\ stlxrb w17, w16, [x1]
1390 \\ cbnz w17, 0b
1391 \\1:
1392 : [ret] "={w0}" (-> u8),
1393 : [val] "{w0}" (val),
1394 [ptr] "{x1}" (ptr),
1395 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1396 : "w15", "w16", "w17", "memory"
1397 );
1398}
1399fn swp1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1400 @setRuntimeSafety(false);
1401 __init_aarch64_have_lse_atomics();
1402
1403 return asm volatile (
1404 \\ cbz w16, 8f
1405 \\ swplb w0, w0, [x1]
1406 \\ cbz wzr, 1f
1407 \\8:
1408 \\ mov w16, w0
1409 \\0:
1410 \\ ldxrb w0, [x1]
1411 \\ stlxrb w17, w16, [x1]
1412 \\ cbnz w17, 0b
1413 \\1:
1414 : [ret] "={w0}" (-> u8),
1415 : [val] "{w0}" (val),
1416 [ptr] "{x1}" (ptr),
1417 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1418 : "w15", "w16", "w17", "memory"
1419 );
1420}
1421fn swp1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1422 @setRuntimeSafety(false);
1423 __init_aarch64_have_lse_atomics();
1424
1425 return asm volatile (
1426 \\ cbz w16, 8f
1427 \\ .inst 0x38208020 + 0x00000000 + 0x400000
1428 \\ cbz wzr, 1f
1429 \\8:
1430 \\ mov w16, w0
1431 \\0:
1432 \\ ldxrb w0, [x1]
1433 \\ stlxrb w17, w16, [x1]
1434 \\ cbnz w17, 0b
1435 \\1:
1436 : [ret] "={w0}" (-> u8),
1437 : [val] "{w0}" (val),
1438 [ptr] "{x1}" (ptr),
1439 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1440 : "w15", "w16", "w17", "memory"
1441 );
1442}
1443const __aarch64_swp1_rel = if (is_darwin)
1444 if (has_lse)
1445 swp1RelDarwinLse
1446 else
1447 swp1RelDarwinNolse
1448else if (has_lse)
1449 swp1RelNondarwinLse
1450else
1451 swp1RelNondarwinNolse;
1452fn ldadd1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1453 @setRuntimeSafety(false);
1454 __init_aarch64_have_lse_atomics();
1455
1456 return asm volatile (
1457 \\ cbz w16, 8f
1458 \\ ldaddlb w0, w0, [x1]
1459 \\ cbz wzr, 1f
1460 \\8:
1461 \\ mov w16, w0
1462 \\0:
1463 \\ ldxrb w0, [x1]
1464 \\ add w17, w0, w16
1465 \\ stlxrb w15, w17, [x1]
1466 \\ cbnz w15, 0b
1467 \\1:
1468 : [ret] "={w0}" (-> u8),
1469 : [val] "{w0}" (val),
1470 [ptr] "{x1}" (ptr),
1471 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1472 : "w15", "w16", "w17", "memory"
1473 );
1474}
1475fn ldadd1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1476 @setRuntimeSafety(false);
1477 __init_aarch64_have_lse_atomics();
1478
1479 return asm volatile (
1480 \\ cbz w16, 8f
1481 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
1482 \\ cbz wzr, 1f
1483 \\8:
1484 \\ mov w16, w0
1485 \\0:
1486 \\ ldxrb w0, [x1]
1487 \\ add w17, w0, w16
1488 \\ stlxrb w15, w17, [x1]
1489 \\ cbnz w15, 0b
1490 \\1:
1491 : [ret] "={w0}" (-> u8),
1492 : [val] "{w0}" (val),
1493 [ptr] "{x1}" (ptr),
1494 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1495 : "w15", "w16", "w17", "memory"
1496 );
1497}
1498fn ldadd1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1499 @setRuntimeSafety(false);
1500 __init_aarch64_have_lse_atomics();
1501
1502 return asm volatile (
1503 \\ cbz w16, 8f
1504 \\ ldaddlb w0, w0, [x1]
1505 \\ cbz wzr, 1f
1506 \\8:
1507 \\ mov w16, w0
1508 \\0:
1509 \\ ldxrb w0, [x1]
1510 \\ add w17, w0, w16
1511 \\ stlxrb w15, w17, [x1]
1512 \\ cbnz w15, 0b
1513 \\1:
1514 : [ret] "={w0}" (-> u8),
1515 : [val] "{w0}" (val),
1516 [ptr] "{x1}" (ptr),
1517 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1518 : "w15", "w16", "w17", "memory"
1519 );
1520}
1521fn ldadd1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1522 @setRuntimeSafety(false);
1523 __init_aarch64_have_lse_atomics();
1524
1525 return asm volatile (
1526 \\ cbz w16, 8f
1527 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
1528 \\ cbz wzr, 1f
1529 \\8:
1530 \\ mov w16, w0
1531 \\0:
1532 \\ ldxrb w0, [x1]
1533 \\ add w17, w0, w16
1534 \\ stlxrb w15, w17, [x1]
1535 \\ cbnz w15, 0b
1536 \\1:
1537 : [ret] "={w0}" (-> u8),
1538 : [val] "{w0}" (val),
1539 [ptr] "{x1}" (ptr),
1540 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1541 : "w15", "w16", "w17", "memory"
1542 );
1543}
1544const __aarch64_ldadd1_rel = if (is_darwin)
1545 if (has_lse)
1546 ldadd1RelDarwinLse
1547 else
1548 ldadd1RelDarwinNolse
1549else if (has_lse)
1550 ldadd1RelNondarwinLse
1551else
1552 ldadd1RelNondarwinNolse;
1553fn ldclr1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1554 @setRuntimeSafety(false);
1555 __init_aarch64_have_lse_atomics();
1556
1557 return asm volatile (
1558 \\ cbz w16, 8f
1559 \\ ldclrlb w0, w0, [x1]
1560 \\ cbz wzr, 1f
1561 \\8:
1562 \\ mov w16, w0
1563 \\0:
1564 \\ ldxrb w0, [x1]
1565 \\ bic w17, w0, w16
1566 \\ stlxrb w15, w17, [x1]
1567 \\ cbnz w15, 0b
1568 \\1:
1569 : [ret] "={w0}" (-> u8),
1570 : [val] "{w0}" (val),
1571 [ptr] "{x1}" (ptr),
1572 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1573 : "w15", "w16", "w17", "memory"
1574 );
1575}
1576fn ldclr1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1577 @setRuntimeSafety(false);
1578 __init_aarch64_have_lse_atomics();
1579
1580 return asm volatile (
1581 \\ cbz w16, 8f
1582 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
1583 \\ cbz wzr, 1f
1584 \\8:
1585 \\ mov w16, w0
1586 \\0:
1587 \\ ldxrb w0, [x1]
1588 \\ bic w17, w0, w16
1589 \\ stlxrb w15, w17, [x1]
1590 \\ cbnz w15, 0b
1591 \\1:
1592 : [ret] "={w0}" (-> u8),
1593 : [val] "{w0}" (val),
1594 [ptr] "{x1}" (ptr),
1595 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1596 : "w15", "w16", "w17", "memory"
1597 );
1598}
1599fn ldclr1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1600 @setRuntimeSafety(false);
1601 __init_aarch64_have_lse_atomics();
1602
1603 return asm volatile (
1604 \\ cbz w16, 8f
1605 \\ ldclrlb w0, w0, [x1]
1606 \\ cbz wzr, 1f
1607 \\8:
1608 \\ mov w16, w0
1609 \\0:
1610 \\ ldxrb w0, [x1]
1611 \\ bic w17, w0, w16
1612 \\ stlxrb w15, w17, [x1]
1613 \\ cbnz w15, 0b
1614 \\1:
1615 : [ret] "={w0}" (-> u8),
1616 : [val] "{w0}" (val),
1617 [ptr] "{x1}" (ptr),
1618 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1619 : "w15", "w16", "w17", "memory"
1620 );
1621}
1622fn ldclr1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1623 @setRuntimeSafety(false);
1624 __init_aarch64_have_lse_atomics();
1625
1626 return asm volatile (
1627 \\ cbz w16, 8f
1628 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
1629 \\ cbz wzr, 1f
1630 \\8:
1631 \\ mov w16, w0
1632 \\0:
1633 \\ ldxrb w0, [x1]
1634 \\ bic w17, w0, w16
1635 \\ stlxrb w15, w17, [x1]
1636 \\ cbnz w15, 0b
1637 \\1:
1638 : [ret] "={w0}" (-> u8),
1639 : [val] "{w0}" (val),
1640 [ptr] "{x1}" (ptr),
1641 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1642 : "w15", "w16", "w17", "memory"
1643 );
1644}
1645const __aarch64_ldclr1_rel = if (is_darwin)
1646 if (has_lse)
1647 ldclr1RelDarwinLse
1648 else
1649 ldclr1RelDarwinNolse
1650else if (has_lse)
1651 ldclr1RelNondarwinLse
1652else
1653 ldclr1RelNondarwinNolse;
1654fn ldeor1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1655 @setRuntimeSafety(false);
1656 __init_aarch64_have_lse_atomics();
1657
1658 return asm volatile (
1659 \\ cbz w16, 8f
1660 \\ ldeorlb w0, w0, [x1]
1661 \\ cbz wzr, 1f
1662 \\8:
1663 \\ mov w16, w0
1664 \\0:
1665 \\ ldxrb w0, [x1]
1666 \\ eor w17, w0, w16
1667 \\ stlxrb w15, w17, [x1]
1668 \\ cbnz w15, 0b
1669 \\1:
1670 : [ret] "={w0}" (-> u8),
1671 : [val] "{w0}" (val),
1672 [ptr] "{x1}" (ptr),
1673 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1674 : "w15", "w16", "w17", "memory"
1675 );
1676}
1677fn ldeor1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1678 @setRuntimeSafety(false);
1679 __init_aarch64_have_lse_atomics();
1680
1681 return asm volatile (
1682 \\ cbz w16, 8f
1683 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
1684 \\ cbz wzr, 1f
1685 \\8:
1686 \\ mov w16, w0
1687 \\0:
1688 \\ ldxrb w0, [x1]
1689 \\ eor w17, w0, w16
1690 \\ stlxrb w15, w17, [x1]
1691 \\ cbnz w15, 0b
1692 \\1:
1693 : [ret] "={w0}" (-> u8),
1694 : [val] "{w0}" (val),
1695 [ptr] "{x1}" (ptr),
1696 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1697 : "w15", "w16", "w17", "memory"
1698 );
1699}
1700fn ldeor1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1701 @setRuntimeSafety(false);
1702 __init_aarch64_have_lse_atomics();
1703
1704 return asm volatile (
1705 \\ cbz w16, 8f
1706 \\ ldeorlb w0, w0, [x1]
1707 \\ cbz wzr, 1f
1708 \\8:
1709 \\ mov w16, w0
1710 \\0:
1711 \\ ldxrb w0, [x1]
1712 \\ eor w17, w0, w16
1713 \\ stlxrb w15, w17, [x1]
1714 \\ cbnz w15, 0b
1715 \\1:
1716 : [ret] "={w0}" (-> u8),
1717 : [val] "{w0}" (val),
1718 [ptr] "{x1}" (ptr),
1719 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1720 : "w15", "w16", "w17", "memory"
1721 );
1722}
1723fn ldeor1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1724 @setRuntimeSafety(false);
1725 __init_aarch64_have_lse_atomics();
1726
1727 return asm volatile (
1728 \\ cbz w16, 8f
1729 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
1730 \\ cbz wzr, 1f
1731 \\8:
1732 \\ mov w16, w0
1733 \\0:
1734 \\ ldxrb w0, [x1]
1735 \\ eor w17, w0, w16
1736 \\ stlxrb w15, w17, [x1]
1737 \\ cbnz w15, 0b
1738 \\1:
1739 : [ret] "={w0}" (-> u8),
1740 : [val] "{w0}" (val),
1741 [ptr] "{x1}" (ptr),
1742 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1743 : "w15", "w16", "w17", "memory"
1744 );
1745}
1746const __aarch64_ldeor1_rel = if (is_darwin)
1747 if (has_lse)
1748 ldeor1RelDarwinLse
1749 else
1750 ldeor1RelDarwinNolse
1751else if (has_lse)
1752 ldeor1RelNondarwinLse
1753else
1754 ldeor1RelNondarwinNolse;
1755fn ldset1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1756 @setRuntimeSafety(false);
1757 __init_aarch64_have_lse_atomics();
1758
1759 return asm volatile (
1760 \\ cbz w16, 8f
1761 \\ ldsetlb w0, w0, [x1]
1762 \\ cbz wzr, 1f
1763 \\8:
1764 \\ mov w16, w0
1765 \\0:
1766 \\ ldxrb w0, [x1]
1767 \\ orr w17, w0, w16
1768 \\ stlxrb w15, w17, [x1]
1769 \\ cbnz w15, 0b
1770 \\1:
1771 : [ret] "={w0}" (-> u8),
1772 : [val] "{w0}" (val),
1773 [ptr] "{x1}" (ptr),
1774 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1775 : "w15", "w16", "w17", "memory"
1776 );
1777}
1778fn ldset1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1779 @setRuntimeSafety(false);
1780 __init_aarch64_have_lse_atomics();
1781
1782 return asm volatile (
1783 \\ cbz w16, 8f
1784 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
1785 \\ cbz wzr, 1f
1786 \\8:
1787 \\ mov w16, w0
1788 \\0:
1789 \\ ldxrb w0, [x1]
1790 \\ orr w17, w0, w16
1791 \\ stlxrb w15, w17, [x1]
1792 \\ cbnz w15, 0b
1793 \\1:
1794 : [ret] "={w0}" (-> u8),
1795 : [val] "{w0}" (val),
1796 [ptr] "{x1}" (ptr),
1797 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1798 : "w15", "w16", "w17", "memory"
1799 );
1800}
1801fn ldset1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1802 @setRuntimeSafety(false);
1803 __init_aarch64_have_lse_atomics();
1804
1805 return asm volatile (
1806 \\ cbz w16, 8f
1807 \\ ldsetlb w0, w0, [x1]
1808 \\ cbz wzr, 1f
1809 \\8:
1810 \\ mov w16, w0
1811 \\0:
1812 \\ ldxrb w0, [x1]
1813 \\ orr w17, w0, w16
1814 \\ stlxrb w15, w17, [x1]
1815 \\ cbnz w15, 0b
1816 \\1:
1817 : [ret] "={w0}" (-> u8),
1818 : [val] "{w0}" (val),
1819 [ptr] "{x1}" (ptr),
1820 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1821 : "w15", "w16", "w17", "memory"
1822 );
1823}
1824fn ldset1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1825 @setRuntimeSafety(false);
1826 __init_aarch64_have_lse_atomics();
1827
1828 return asm volatile (
1829 \\ cbz w16, 8f
1830 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
1831 \\ cbz wzr, 1f
1832 \\8:
1833 \\ mov w16, w0
1834 \\0:
1835 \\ ldxrb w0, [x1]
1836 \\ orr w17, w0, w16
1837 \\ stlxrb w15, w17, [x1]
1838 \\ cbnz w15, 0b
1839 \\1:
1840 : [ret] "={w0}" (-> u8),
1841 : [val] "{w0}" (val),
1842 [ptr] "{x1}" (ptr),
1843 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1844 : "w15", "w16", "w17", "memory"
1845 );
1846}
1847const __aarch64_ldset1_rel = if (is_darwin)
1848 if (has_lse)
1849 ldset1RelDarwinLse
1850 else
1851 ldset1RelDarwinNolse
1852else if (has_lse)
1853 ldset1RelNondarwinLse
1854else
1855 ldset1RelNondarwinNolse;
1856fn cas1AcqRelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1857 @setRuntimeSafety(false);
1858 __init_aarch64_have_lse_atomics();
1859
1860 return asm volatile (
1861 \\ cbz w16, 8f
1862 \\ casalb w0, w1, [x2]
1863 \\
1864 \\ cbz wzr, 1f
1865 \\8:
1866 \\ uxtb w16, w0
1867 \\0:
1868 \\ ldaxrb w0, [x2]
1869 \\ cmp w0, w16
1870 \\ bne 1f
1871 \\ stlxrb w17, w1, [x2]
1872 \\ cbnz w17, 0b
1873 \\1:
1874 : [ret] "={w0}" (-> u8),
1875 : [expected] "{w0}" (expected),
1876 [desired] "{w1}" (desired),
1877 [ptr] "{x2}" (ptr),
1878 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1879 : "w15", "w16", "w17", "memory"
1880 );
1881}
1882fn cas1AcqRelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1883 @setRuntimeSafety(false);
1884 __init_aarch64_have_lse_atomics();
1885
1886 return asm volatile (
1887 \\ cbz w16, 8f
1888 \\ .inst 0x08a07c41 + 0x00000000 + 0x408000
1889 \\
1890 \\ cbz wzr, 1f
1891 \\8:
1892 \\ uxtb w16, w0
1893 \\0:
1894 \\ ldaxrb w0, [x2]
1895 \\ cmp w0, w16
1896 \\ bne 1f
1897 \\ stlxrb w17, w1, [x2]
1898 \\ cbnz w17, 0b
1899 \\1:
1900 : [ret] "={w0}" (-> u8),
1901 : [expected] "{w0}" (expected),
1902 [desired] "{w1}" (desired),
1903 [ptr] "{x2}" (ptr),
1904 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1905 : "w15", "w16", "w17", "memory"
1906 );
1907}
1908fn cas1AcqRelNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1909 @setRuntimeSafety(false);
1910 __init_aarch64_have_lse_atomics();
1911
1912 return asm volatile (
1913 \\ cbz w16, 8f
1914 \\ casalb w0, w1, [x2]
1915 \\
1916 \\ cbz wzr, 1f
1917 \\8:
1918 \\ uxtb w16, w0
1919 \\0:
1920 \\ ldaxrb w0, [x2]
1921 \\ cmp w0, w16
1922 \\ bne 1f
1923 \\ stlxrb w17, w1, [x2]
1924 \\ cbnz w17, 0b
1925 \\1:
1926 : [ret] "={w0}" (-> u8),
1927 : [expected] "{w0}" (expected),
1928 [desired] "{w1}" (desired),
1929 [ptr] "{x2}" (ptr),
1930 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1931 : "w15", "w16", "w17", "memory"
1932 );
1933}
1934fn cas1AcqRelNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1935 @setRuntimeSafety(false);
1936 __init_aarch64_have_lse_atomics();
1937
1938 return asm volatile (
1939 \\ cbz w16, 8f
1940 \\ .inst 0x08a07c41 + 0x00000000 + 0x408000
1941 \\
1942 \\ cbz wzr, 1f
1943 \\8:
1944 \\ uxtb w16, w0
1945 \\0:
1946 \\ ldaxrb w0, [x2]
1947 \\ cmp w0, w16
1948 \\ bne 1f
1949 \\ stlxrb w17, w1, [x2]
1950 \\ cbnz w17, 0b
1951 \\1:
1952 : [ret] "={w0}" (-> u8),
1953 : [expected] "{w0}" (expected),
1954 [desired] "{w1}" (desired),
1955 [ptr] "{x2}" (ptr),
1956 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1957 : "w15", "w16", "w17", "memory"
1958 );
1959}
1960const __aarch64_cas1_acq_rel = if (is_darwin)
1961 if (has_lse)
1962 cas1AcqRelDarwinLse
1963 else
1964 cas1AcqRelDarwinNolse
1965else if (has_lse)
1966 cas1AcqRelNondarwinLse
1967else
1968 cas1AcqRelNondarwinNolse;
1969fn swp1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1970 @setRuntimeSafety(false);
1971 __init_aarch64_have_lse_atomics();
1972
1973 return asm volatile (
1974 \\ cbz w16, 8f
1975 \\ swpalb w0, w0, [x1]
1976 \\ cbz wzr, 1f
1977 \\8:
1978 \\ mov w16, w0
1979 \\0:
1980 \\ ldaxrb w0, [x1]
1981 \\ stlxrb w17, w16, [x1]
1982 \\ cbnz w17, 0b
1983 \\1:
1984 : [ret] "={w0}" (-> u8),
1985 : [val] "{w0}" (val),
1986 [ptr] "{x1}" (ptr),
1987 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1988 : "w15", "w16", "w17", "memory"
1989 );
1990}
1991fn swp1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1992 @setRuntimeSafety(false);
1993 __init_aarch64_have_lse_atomics();
1994
1995 return asm volatile (
1996 \\ cbz w16, 8f
1997 \\ .inst 0x38208020 + 0x00000000 + 0xc00000
1998 \\ cbz wzr, 1f
1999 \\8:
2000 \\ mov w16, w0
2001 \\0:
2002 \\ ldaxrb w0, [x1]
2003 \\ stlxrb w17, w16, [x1]
2004 \\ cbnz w17, 0b
2005 \\1:
2006 : [ret] "={w0}" (-> u8),
2007 : [val] "{w0}" (val),
2008 [ptr] "{x1}" (ptr),
2009 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2010 : "w15", "w16", "w17", "memory"
2011 );
2012}
2013fn swp1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2014 @setRuntimeSafety(false);
2015 __init_aarch64_have_lse_atomics();
2016
2017 return asm volatile (
2018 \\ cbz w16, 8f
2019 \\ swpalb w0, w0, [x1]
2020 \\ cbz wzr, 1f
2021 \\8:
2022 \\ mov w16, w0
2023 \\0:
2024 \\ ldaxrb w0, [x1]
2025 \\ stlxrb w17, w16, [x1]
2026 \\ cbnz w17, 0b
2027 \\1:
2028 : [ret] "={w0}" (-> u8),
2029 : [val] "{w0}" (val),
2030 [ptr] "{x1}" (ptr),
2031 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2032 : "w15", "w16", "w17", "memory"
2033 );
2034}
2035fn swp1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2036 @setRuntimeSafety(false);
2037 __init_aarch64_have_lse_atomics();
2038
2039 return asm volatile (
2040 \\ cbz w16, 8f
2041 \\ .inst 0x38208020 + 0x00000000 + 0xc00000
2042 \\ cbz wzr, 1f
2043 \\8:
2044 \\ mov w16, w0
2045 \\0:
2046 \\ ldaxrb w0, [x1]
2047 \\ stlxrb w17, w16, [x1]
2048 \\ cbnz w17, 0b
2049 \\1:
2050 : [ret] "={w0}" (-> u8),
2051 : [val] "{w0}" (val),
2052 [ptr] "{x1}" (ptr),
2053 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2054 : "w15", "w16", "w17", "memory"
2055 );
2056}
2057const __aarch64_swp1_acq_rel = if (is_darwin)
2058 if (has_lse)
2059 swp1AcqRelDarwinLse
2060 else
2061 swp1AcqRelDarwinNolse
2062else if (has_lse)
2063 swp1AcqRelNondarwinLse
2064else
2065 swp1AcqRelNondarwinNolse;
2066fn ldadd1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2067 @setRuntimeSafety(false);
2068 __init_aarch64_have_lse_atomics();
2069
2070 return asm volatile (
2071 \\ cbz w16, 8f
2072 \\ ldaddalb w0, w0, [x1]
2073 \\ cbz wzr, 1f
2074 \\8:
2075 \\ mov w16, w0
2076 \\0:
2077 \\ ldaxrb w0, [x1]
2078 \\ add w17, w0, w16
2079 \\ stlxrb w15, w17, [x1]
2080 \\ cbnz w15, 0b
2081 \\1:
2082 : [ret] "={w0}" (-> u8),
2083 : [val] "{w0}" (val),
2084 [ptr] "{x1}" (ptr),
2085 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2086 : "w15", "w16", "w17", "memory"
2087 );
2088}
2089fn ldadd1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2090 @setRuntimeSafety(false);
2091 __init_aarch64_have_lse_atomics();
2092
2093 return asm volatile (
2094 \\ cbz w16, 8f
2095 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
2096 \\ cbz wzr, 1f
2097 \\8:
2098 \\ mov w16, w0
2099 \\0:
2100 \\ ldaxrb w0, [x1]
2101 \\ add w17, w0, w16
2102 \\ stlxrb w15, w17, [x1]
2103 \\ cbnz w15, 0b
2104 \\1:
2105 : [ret] "={w0}" (-> u8),
2106 : [val] "{w0}" (val),
2107 [ptr] "{x1}" (ptr),
2108 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2109 : "w15", "w16", "w17", "memory"
2110 );
2111}
2112fn ldadd1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2113 @setRuntimeSafety(false);
2114 __init_aarch64_have_lse_atomics();
2115
2116 return asm volatile (
2117 \\ cbz w16, 8f
2118 \\ ldaddalb w0, w0, [x1]
2119 \\ cbz wzr, 1f
2120 \\8:
2121 \\ mov w16, w0
2122 \\0:
2123 \\ ldaxrb w0, [x1]
2124 \\ add w17, w0, w16
2125 \\ stlxrb w15, w17, [x1]
2126 \\ cbnz w15, 0b
2127 \\1:
2128 : [ret] "={w0}" (-> u8),
2129 : [val] "{w0}" (val),
2130 [ptr] "{x1}" (ptr),
2131 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2132 : "w15", "w16", "w17", "memory"
2133 );
2134}
2135fn ldadd1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2136 @setRuntimeSafety(false);
2137 __init_aarch64_have_lse_atomics();
2138
2139 return asm volatile (
2140 \\ cbz w16, 8f
2141 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
2142 \\ cbz wzr, 1f
2143 \\8:
2144 \\ mov w16, w0
2145 \\0:
2146 \\ ldaxrb w0, [x1]
2147 \\ add w17, w0, w16
2148 \\ stlxrb w15, w17, [x1]
2149 \\ cbnz w15, 0b
2150 \\1:
2151 : [ret] "={w0}" (-> u8),
2152 : [val] "{w0}" (val),
2153 [ptr] "{x1}" (ptr),
2154 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2155 : "w15", "w16", "w17", "memory"
2156 );
2157}
2158const __aarch64_ldadd1_acq_rel = if (is_darwin)
2159 if (has_lse)
2160 ldadd1AcqRelDarwinLse
2161 else
2162 ldadd1AcqRelDarwinNolse
2163else if (has_lse)
2164 ldadd1AcqRelNondarwinLse
2165else
2166 ldadd1AcqRelNondarwinNolse;
2167fn ldclr1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2168 @setRuntimeSafety(false);
2169 __init_aarch64_have_lse_atomics();
2170
2171 return asm volatile (
2172 \\ cbz w16, 8f
2173 \\ ldclralb w0, w0, [x1]
2174 \\ cbz wzr, 1f
2175 \\8:
2176 \\ mov w16, w0
2177 \\0:
2178 \\ ldaxrb w0, [x1]
2179 \\ bic w17, w0, w16
2180 \\ stlxrb w15, w17, [x1]
2181 \\ cbnz w15, 0b
2182 \\1:
2183 : [ret] "={w0}" (-> u8),
2184 : [val] "{w0}" (val),
2185 [ptr] "{x1}" (ptr),
2186 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2187 : "w15", "w16", "w17", "memory"
2188 );
2189}
2190fn ldclr1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2191 @setRuntimeSafety(false);
2192 __init_aarch64_have_lse_atomics();
2193
2194 return asm volatile (
2195 \\ cbz w16, 8f
2196 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
2197 \\ cbz wzr, 1f
2198 \\8:
2199 \\ mov w16, w0
2200 \\0:
2201 \\ ldaxrb w0, [x1]
2202 \\ bic w17, w0, w16
2203 \\ stlxrb w15, w17, [x1]
2204 \\ cbnz w15, 0b
2205 \\1:
2206 : [ret] "={w0}" (-> u8),
2207 : [val] "{w0}" (val),
2208 [ptr] "{x1}" (ptr),
2209 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2210 : "w15", "w16", "w17", "memory"
2211 );
2212}
2213fn ldclr1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2214 @setRuntimeSafety(false);
2215 __init_aarch64_have_lse_atomics();
2216
2217 return asm volatile (
2218 \\ cbz w16, 8f
2219 \\ ldclralb w0, w0, [x1]
2220 \\ cbz wzr, 1f
2221 \\8:
2222 \\ mov w16, w0
2223 \\0:
2224 \\ ldaxrb w0, [x1]
2225 \\ bic w17, w0, w16
2226 \\ stlxrb w15, w17, [x1]
2227 \\ cbnz w15, 0b
2228 \\1:
2229 : [ret] "={w0}" (-> u8),
2230 : [val] "{w0}" (val),
2231 [ptr] "{x1}" (ptr),
2232 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2233 : "w15", "w16", "w17", "memory"
2234 );
2235}
2236fn ldclr1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2237 @setRuntimeSafety(false);
2238 __init_aarch64_have_lse_atomics();
2239
2240 return asm volatile (
2241 \\ cbz w16, 8f
2242 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
2243 \\ cbz wzr, 1f
2244 \\8:
2245 \\ mov w16, w0
2246 \\0:
2247 \\ ldaxrb w0, [x1]
2248 \\ bic w17, w0, w16
2249 \\ stlxrb w15, w17, [x1]
2250 \\ cbnz w15, 0b
2251 \\1:
2252 : [ret] "={w0}" (-> u8),
2253 : [val] "{w0}" (val),
2254 [ptr] "{x1}" (ptr),
2255 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2256 : "w15", "w16", "w17", "memory"
2257 );
2258}
2259const __aarch64_ldclr1_acq_rel = if (is_darwin)
2260 if (has_lse)
2261 ldclr1AcqRelDarwinLse
2262 else
2263 ldclr1AcqRelDarwinNolse
2264else if (has_lse)
2265 ldclr1AcqRelNondarwinLse
2266else
2267 ldclr1AcqRelNondarwinNolse;
2268fn ldeor1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2269 @setRuntimeSafety(false);
2270 __init_aarch64_have_lse_atomics();
2271
2272 return asm volatile (
2273 \\ cbz w16, 8f
2274 \\ ldeoralb w0, w0, [x1]
2275 \\ cbz wzr, 1f
2276 \\8:
2277 \\ mov w16, w0
2278 \\0:
2279 \\ ldaxrb w0, [x1]
2280 \\ eor w17, w0, w16
2281 \\ stlxrb w15, w17, [x1]
2282 \\ cbnz w15, 0b
2283 \\1:
2284 : [ret] "={w0}" (-> u8),
2285 : [val] "{w0}" (val),
2286 [ptr] "{x1}" (ptr),
2287 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2288 : "w15", "w16", "w17", "memory"
2289 );
2290}
2291fn ldeor1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2292 @setRuntimeSafety(false);
2293 __init_aarch64_have_lse_atomics();
2294
2295 return asm volatile (
2296 \\ cbz w16, 8f
2297 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
2298 \\ cbz wzr, 1f
2299 \\8:
2300 \\ mov w16, w0
2301 \\0:
2302 \\ ldaxrb w0, [x1]
2303 \\ eor w17, w0, w16
2304 \\ stlxrb w15, w17, [x1]
2305 \\ cbnz w15, 0b
2306 \\1:
2307 : [ret] "={w0}" (-> u8),
2308 : [val] "{w0}" (val),
2309 [ptr] "{x1}" (ptr),
2310 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2311 : "w15", "w16", "w17", "memory"
2312 );
2313}
2314fn ldeor1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2315 @setRuntimeSafety(false);
2316 __init_aarch64_have_lse_atomics();
2317
2318 return asm volatile (
2319 \\ cbz w16, 8f
2320 \\ ldeoralb w0, w0, [x1]
2321 \\ cbz wzr, 1f
2322 \\8:
2323 \\ mov w16, w0
2324 \\0:
2325 \\ ldaxrb w0, [x1]
2326 \\ eor w17, w0, w16
2327 \\ stlxrb w15, w17, [x1]
2328 \\ cbnz w15, 0b
2329 \\1:
2330 : [ret] "={w0}" (-> u8),
2331 : [val] "{w0}" (val),
2332 [ptr] "{x1}" (ptr),
2333 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2334 : "w15", "w16", "w17", "memory"
2335 );
2336}
2337fn ldeor1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2338 @setRuntimeSafety(false);
2339 __init_aarch64_have_lse_atomics();
2340
2341 return asm volatile (
2342 \\ cbz w16, 8f
2343 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
2344 \\ cbz wzr, 1f
2345 \\8:
2346 \\ mov w16, w0
2347 \\0:
2348 \\ ldaxrb w0, [x1]
2349 \\ eor w17, w0, w16
2350 \\ stlxrb w15, w17, [x1]
2351 \\ cbnz w15, 0b
2352 \\1:
2353 : [ret] "={w0}" (-> u8),
2354 : [val] "{w0}" (val),
2355 [ptr] "{x1}" (ptr),
2356 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2357 : "w15", "w16", "w17", "memory"
2358 );
2359}
2360const __aarch64_ldeor1_acq_rel = if (is_darwin)
2361 if (has_lse)
2362 ldeor1AcqRelDarwinLse
2363 else
2364 ldeor1AcqRelDarwinNolse
2365else if (has_lse)
2366 ldeor1AcqRelNondarwinLse
2367else
2368 ldeor1AcqRelNondarwinNolse;
2369fn ldset1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2370 @setRuntimeSafety(false);
2371 __init_aarch64_have_lse_atomics();
2372
2373 return asm volatile (
2374 \\ cbz w16, 8f
2375 \\ ldsetalb w0, w0, [x1]
2376 \\ cbz wzr, 1f
2377 \\8:
2378 \\ mov w16, w0
2379 \\0:
2380 \\ ldaxrb w0, [x1]
2381 \\ orr w17, w0, w16
2382 \\ stlxrb w15, w17, [x1]
2383 \\ cbnz w15, 0b
2384 \\1:
2385 : [ret] "={w0}" (-> u8),
2386 : [val] "{w0}" (val),
2387 [ptr] "{x1}" (ptr),
2388 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2389 : "w15", "w16", "w17", "memory"
2390 );
2391}
2392fn ldset1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2393 @setRuntimeSafety(false);
2394 __init_aarch64_have_lse_atomics();
2395
2396 return asm volatile (
2397 \\ cbz w16, 8f
2398 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
2399 \\ cbz wzr, 1f
2400 \\8:
2401 \\ mov w16, w0
2402 \\0:
2403 \\ ldaxrb w0, [x1]
2404 \\ orr w17, w0, w16
2405 \\ stlxrb w15, w17, [x1]
2406 \\ cbnz w15, 0b
2407 \\1:
2408 : [ret] "={w0}" (-> u8),
2409 : [val] "{w0}" (val),
2410 [ptr] "{x1}" (ptr),
2411 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2412 : "w15", "w16", "w17", "memory"
2413 );
2414}
2415fn ldset1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2416 @setRuntimeSafety(false);
2417 __init_aarch64_have_lse_atomics();
2418
2419 return asm volatile (
2420 \\ cbz w16, 8f
2421 \\ ldsetalb w0, w0, [x1]
2422 \\ cbz wzr, 1f
2423 \\8:
2424 \\ mov w16, w0
2425 \\0:
2426 \\ ldaxrb w0, [x1]
2427 \\ orr w17, w0, w16
2428 \\ stlxrb w15, w17, [x1]
2429 \\ cbnz w15, 0b
2430 \\1:
2431 : [ret] "={w0}" (-> u8),
2432 : [val] "{w0}" (val),
2433 [ptr] "{x1}" (ptr),
2434 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2435 : "w15", "w16", "w17", "memory"
2436 );
2437}
2438fn ldset1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2439 @setRuntimeSafety(false);
2440 __init_aarch64_have_lse_atomics();
2441
2442 return asm volatile (
2443 \\ cbz w16, 8f
2444 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
2445 \\ cbz wzr, 1f
2446 \\8:
2447 \\ mov w16, w0
2448 \\0:
2449 \\ ldaxrb w0, [x1]
2450 \\ orr w17, w0, w16
2451 \\ stlxrb w15, w17, [x1]
2452 \\ cbnz w15, 0b
2453 \\1:
2454 : [ret] "={w0}" (-> u8),
2455 : [val] "{w0}" (val),
2456 [ptr] "{x1}" (ptr),
2457 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2458 : "w15", "w16", "w17", "memory"
2459 );
2460}
2461const __aarch64_ldset1_acq_rel = if (is_darwin)
2462 if (has_lse)
2463 ldset1AcqRelDarwinLse
2464 else
2465 ldset1AcqRelDarwinNolse
2466else if (has_lse)
2467 ldset1AcqRelNondarwinLse
2468else
2469 ldset1AcqRelNondarwinNolse;
2470fn cas2RelaxDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2471 @setRuntimeSafety(false);
2472 __init_aarch64_have_lse_atomics();
2473
2474 return asm volatile (
2475 \\ cbz w16, 8f
2476 \\ cash w0, w1, [x2]
2477 \\
2478 \\ cbz wzr, 1f
2479 \\8:
2480 \\ uxth w16, w0
2481 \\0:
2482 \\ ldxrh w0, [x2]
2483 \\ cmp w0, w16
2484 \\ bne 1f
2485 \\ stxrh w17, w1, [x2]
2486 \\ cbnz w17, 0b
2487 \\1:
2488 : [ret] "={w0}" (-> u16),
2489 : [expected] "{w0}" (expected),
2490 [desired] "{w1}" (desired),
2491 [ptr] "{x2}" (ptr),
2492 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2493 : "w15", "w16", "w17", "memory"
2494 );
2495}
2496fn cas2RelaxDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2497 @setRuntimeSafety(false);
2498 __init_aarch64_have_lse_atomics();
2499
2500 return asm volatile (
2501 \\ cbz w16, 8f
2502 \\ .inst 0x08a07c41 + 0x40000000 + 0x000000
2503 \\
2504 \\ cbz wzr, 1f
2505 \\8:
2506 \\ uxth w16, w0
2507 \\0:
2508 \\ ldxrh w0, [x2]
2509 \\ cmp w0, w16
2510 \\ bne 1f
2511 \\ stxrh w17, w1, [x2]
2512 \\ cbnz w17, 0b
2513 \\1:
2514 : [ret] "={w0}" (-> u16),
2515 : [expected] "{w0}" (expected),
2516 [desired] "{w1}" (desired),
2517 [ptr] "{x2}" (ptr),
2518 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2519 : "w15", "w16", "w17", "memory"
2520 );
2521}
2522fn cas2RelaxNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2523 @setRuntimeSafety(false);
2524 __init_aarch64_have_lse_atomics();
2525
2526 return asm volatile (
2527 \\ cbz w16, 8f
2528 \\ cash w0, w1, [x2]
2529 \\
2530 \\ cbz wzr, 1f
2531 \\8:
2532 \\ uxth w16, w0
2533 \\0:
2534 \\ ldxrh w0, [x2]
2535 \\ cmp w0, w16
2536 \\ bne 1f
2537 \\ stxrh w17, w1, [x2]
2538 \\ cbnz w17, 0b
2539 \\1:
2540 : [ret] "={w0}" (-> u16),
2541 : [expected] "{w0}" (expected),
2542 [desired] "{w1}" (desired),
2543 [ptr] "{x2}" (ptr),
2544 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2545 : "w15", "w16", "w17", "memory"
2546 );
2547}
2548fn cas2RelaxNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2549 @setRuntimeSafety(false);
2550 __init_aarch64_have_lse_atomics();
2551
2552 return asm volatile (
2553 \\ cbz w16, 8f
2554 \\ .inst 0x08a07c41 + 0x40000000 + 0x000000
2555 \\
2556 \\ cbz wzr, 1f
2557 \\8:
2558 \\ uxth w16, w0
2559 \\0:
2560 \\ ldxrh w0, [x2]
2561 \\ cmp w0, w16
2562 \\ bne 1f
2563 \\ stxrh w17, w1, [x2]
2564 \\ cbnz w17, 0b
2565 \\1:
2566 : [ret] "={w0}" (-> u16),
2567 : [expected] "{w0}" (expected),
2568 [desired] "{w1}" (desired),
2569 [ptr] "{x2}" (ptr),
2570 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2571 : "w15", "w16", "w17", "memory"
2572 );
2573}
2574const __aarch64_cas2_relax = if (is_darwin)
2575 if (has_lse)
2576 cas2RelaxDarwinLse
2577 else
2578 cas2RelaxDarwinNolse
2579else if (has_lse)
2580 cas2RelaxNondarwinLse
2581else
2582 cas2RelaxNondarwinNolse;
2583fn swp2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2584 @setRuntimeSafety(false);
2585 __init_aarch64_have_lse_atomics();
2586
2587 return asm volatile (
2588 \\ cbz w16, 8f
2589 \\ swph w0, w0, [x1]
2590 \\ cbz wzr, 1f
2591 \\8:
2592 \\ mov w16, w0
2593 \\0:
2594 \\ ldxrh w0, [x1]
2595 \\ stxrh w17, w16, [x1]
2596 \\ cbnz w17, 0b
2597 \\1:
2598 : [ret] "={w0}" (-> u16),
2599 : [val] "{w0}" (val),
2600 [ptr] "{x1}" (ptr),
2601 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2602 : "w15", "w16", "w17", "memory"
2603 );
2604}
2605fn swp2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2606 @setRuntimeSafety(false);
2607 __init_aarch64_have_lse_atomics();
2608
2609 return asm volatile (
2610 \\ cbz w16, 8f
2611 \\ .inst 0x38208020 + 0x40000000 + 0x000000
2612 \\ cbz wzr, 1f
2613 \\8:
2614 \\ mov w16, w0
2615 \\0:
2616 \\ ldxrh w0, [x1]
2617 \\ stxrh w17, w16, [x1]
2618 \\ cbnz w17, 0b
2619 \\1:
2620 : [ret] "={w0}" (-> u16),
2621 : [val] "{w0}" (val),
2622 [ptr] "{x1}" (ptr),
2623 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2624 : "w15", "w16", "w17", "memory"
2625 );
2626}
2627fn swp2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2628 @setRuntimeSafety(false);
2629 __init_aarch64_have_lse_atomics();
2630
2631 return asm volatile (
2632 \\ cbz w16, 8f
2633 \\ swph w0, w0, [x1]
2634 \\ cbz wzr, 1f
2635 \\8:
2636 \\ mov w16, w0
2637 \\0:
2638 \\ ldxrh w0, [x1]
2639 \\ stxrh w17, w16, [x1]
2640 \\ cbnz w17, 0b
2641 \\1:
2642 : [ret] "={w0}" (-> u16),
2643 : [val] "{w0}" (val),
2644 [ptr] "{x1}" (ptr),
2645 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2646 : "w15", "w16", "w17", "memory"
2647 );
2648}
2649fn swp2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2650 @setRuntimeSafety(false);
2651 __init_aarch64_have_lse_atomics();
2652
2653 return asm volatile (
2654 \\ cbz w16, 8f
2655 \\ .inst 0x38208020 + 0x40000000 + 0x000000
2656 \\ cbz wzr, 1f
2657 \\8:
2658 \\ mov w16, w0
2659 \\0:
2660 \\ ldxrh w0, [x1]
2661 \\ stxrh w17, w16, [x1]
2662 \\ cbnz w17, 0b
2663 \\1:
2664 : [ret] "={w0}" (-> u16),
2665 : [val] "{w0}" (val),
2666 [ptr] "{x1}" (ptr),
2667 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2668 : "w15", "w16", "w17", "memory"
2669 );
2670}
2671const __aarch64_swp2_relax = if (is_darwin)
2672 if (has_lse)
2673 swp2RelaxDarwinLse
2674 else
2675 swp2RelaxDarwinNolse
2676else if (has_lse)
2677 swp2RelaxNondarwinLse
2678else
2679 swp2RelaxNondarwinNolse;
2680fn ldadd2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2681 @setRuntimeSafety(false);
2682 __init_aarch64_have_lse_atomics();
2683
2684 return asm volatile (
2685 \\ cbz w16, 8f
2686 \\ ldaddh w0, w0, [x1]
2687 \\ cbz wzr, 1f
2688 \\8:
2689 \\ mov w16, w0
2690 \\0:
2691 \\ ldxrh w0, [x1]
2692 \\ add w17, w0, w16
2693 \\ stxrh w15, w17, [x1]
2694 \\ cbnz w15, 0b
2695 \\1:
2696 : [ret] "={w0}" (-> u16),
2697 : [val] "{w0}" (val),
2698 [ptr] "{x1}" (ptr),
2699 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2700 : "w15", "w16", "w17", "memory"
2701 );
2702}
2703fn ldadd2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2704 @setRuntimeSafety(false);
2705 __init_aarch64_have_lse_atomics();
2706
2707 return asm volatile (
2708 \\ cbz w16, 8f
2709 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
2710 \\ cbz wzr, 1f
2711 \\8:
2712 \\ mov w16, w0
2713 \\0:
2714 \\ ldxrh w0, [x1]
2715 \\ add w17, w0, w16
2716 \\ stxrh w15, w17, [x1]
2717 \\ cbnz w15, 0b
2718 \\1:
2719 : [ret] "={w0}" (-> u16),
2720 : [val] "{w0}" (val),
2721 [ptr] "{x1}" (ptr),
2722 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2723 : "w15", "w16", "w17", "memory"
2724 );
2725}
2726fn ldadd2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2727 @setRuntimeSafety(false);
2728 __init_aarch64_have_lse_atomics();
2729
2730 return asm volatile (
2731 \\ cbz w16, 8f
2732 \\ ldaddh w0, w0, [x1]
2733 \\ cbz wzr, 1f
2734 \\8:
2735 \\ mov w16, w0
2736 \\0:
2737 \\ ldxrh w0, [x1]
2738 \\ add w17, w0, w16
2739 \\ stxrh w15, w17, [x1]
2740 \\ cbnz w15, 0b
2741 \\1:
2742 : [ret] "={w0}" (-> u16),
2743 : [val] "{w0}" (val),
2744 [ptr] "{x1}" (ptr),
2745 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2746 : "w15", "w16", "w17", "memory"
2747 );
2748}
2749fn ldadd2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2750 @setRuntimeSafety(false);
2751 __init_aarch64_have_lse_atomics();
2752
2753 return asm volatile (
2754 \\ cbz w16, 8f
2755 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
2756 \\ cbz wzr, 1f
2757 \\8:
2758 \\ mov w16, w0
2759 \\0:
2760 \\ ldxrh w0, [x1]
2761 \\ add w17, w0, w16
2762 \\ stxrh w15, w17, [x1]
2763 \\ cbnz w15, 0b
2764 \\1:
2765 : [ret] "={w0}" (-> u16),
2766 : [val] "{w0}" (val),
2767 [ptr] "{x1}" (ptr),
2768 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2769 : "w15", "w16", "w17", "memory"
2770 );
2771}
2772const __aarch64_ldadd2_relax = if (is_darwin)
2773 if (has_lse)
2774 ldadd2RelaxDarwinLse
2775 else
2776 ldadd2RelaxDarwinNolse
2777else if (has_lse)
2778 ldadd2RelaxNondarwinLse
2779else
2780 ldadd2RelaxNondarwinNolse;
2781fn ldclr2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2782 @setRuntimeSafety(false);
2783 __init_aarch64_have_lse_atomics();
2784
2785 return asm volatile (
2786 \\ cbz w16, 8f
2787 \\ ldclrh w0, w0, [x1]
2788 \\ cbz wzr, 1f
2789 \\8:
2790 \\ mov w16, w0
2791 \\0:
2792 \\ ldxrh w0, [x1]
2793 \\ bic w17, w0, w16
2794 \\ stxrh w15, w17, [x1]
2795 \\ cbnz w15, 0b
2796 \\1:
2797 : [ret] "={w0}" (-> u16),
2798 : [val] "{w0}" (val),
2799 [ptr] "{x1}" (ptr),
2800 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2801 : "w15", "w16", "w17", "memory"
2802 );
2803}
2804fn ldclr2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2805 @setRuntimeSafety(false);
2806 __init_aarch64_have_lse_atomics();
2807
2808 return asm volatile (
2809 \\ cbz w16, 8f
2810 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
2811 \\ cbz wzr, 1f
2812 \\8:
2813 \\ mov w16, w0
2814 \\0:
2815 \\ ldxrh w0, [x1]
2816 \\ bic w17, w0, w16
2817 \\ stxrh w15, w17, [x1]
2818 \\ cbnz w15, 0b
2819 \\1:
2820 : [ret] "={w0}" (-> u16),
2821 : [val] "{w0}" (val),
2822 [ptr] "{x1}" (ptr),
2823 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2824 : "w15", "w16", "w17", "memory"
2825 );
2826}
2827fn ldclr2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2828 @setRuntimeSafety(false);
2829 __init_aarch64_have_lse_atomics();
2830
2831 return asm volatile (
2832 \\ cbz w16, 8f
2833 \\ ldclrh w0, w0, [x1]
2834 \\ cbz wzr, 1f
2835 \\8:
2836 \\ mov w16, w0
2837 \\0:
2838 \\ ldxrh w0, [x1]
2839 \\ bic w17, w0, w16
2840 \\ stxrh w15, w17, [x1]
2841 \\ cbnz w15, 0b
2842 \\1:
2843 : [ret] "={w0}" (-> u16),
2844 : [val] "{w0}" (val),
2845 [ptr] "{x1}" (ptr),
2846 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2847 : "w15", "w16", "w17", "memory"
2848 );
2849}
2850fn ldclr2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2851 @setRuntimeSafety(false);
2852 __init_aarch64_have_lse_atomics();
2853
2854 return asm volatile (
2855 \\ cbz w16, 8f
2856 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
2857 \\ cbz wzr, 1f
2858 \\8:
2859 \\ mov w16, w0
2860 \\0:
2861 \\ ldxrh w0, [x1]
2862 \\ bic w17, w0, w16
2863 \\ stxrh w15, w17, [x1]
2864 \\ cbnz w15, 0b
2865 \\1:
2866 : [ret] "={w0}" (-> u16),
2867 : [val] "{w0}" (val),
2868 [ptr] "{x1}" (ptr),
2869 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2870 : "w15", "w16", "w17", "memory"
2871 );
2872}
2873const __aarch64_ldclr2_relax = if (is_darwin)
2874 if (has_lse)
2875 ldclr2RelaxDarwinLse
2876 else
2877 ldclr2RelaxDarwinNolse
2878else if (has_lse)
2879 ldclr2RelaxNondarwinLse
2880else
2881 ldclr2RelaxNondarwinNolse;
2882fn ldeor2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2883 @setRuntimeSafety(false);
2884 __init_aarch64_have_lse_atomics();
2885
2886 return asm volatile (
2887 \\ cbz w16, 8f
2888 \\ ldeorh w0, w0, [x1]
2889 \\ cbz wzr, 1f
2890 \\8:
2891 \\ mov w16, w0
2892 \\0:
2893 \\ ldxrh w0, [x1]
2894 \\ eor w17, w0, w16
2895 \\ stxrh w15, w17, [x1]
2896 \\ cbnz w15, 0b
2897 \\1:
2898 : [ret] "={w0}" (-> u16),
2899 : [val] "{w0}" (val),
2900 [ptr] "{x1}" (ptr),
2901 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2902 : "w15", "w16", "w17", "memory"
2903 );
2904}
2905fn ldeor2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2906 @setRuntimeSafety(false);
2907 __init_aarch64_have_lse_atomics();
2908
2909 return asm volatile (
2910 \\ cbz w16, 8f
2911 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
2912 \\ cbz wzr, 1f
2913 \\8:
2914 \\ mov w16, w0
2915 \\0:
2916 \\ ldxrh w0, [x1]
2917 \\ eor w17, w0, w16
2918 \\ stxrh w15, w17, [x1]
2919 \\ cbnz w15, 0b
2920 \\1:
2921 : [ret] "={w0}" (-> u16),
2922 : [val] "{w0}" (val),
2923 [ptr] "{x1}" (ptr),
2924 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2925 : "w15", "w16", "w17", "memory"
2926 );
2927}
2928fn ldeor2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2929 @setRuntimeSafety(false);
2930 __init_aarch64_have_lse_atomics();
2931
2932 return asm volatile (
2933 \\ cbz w16, 8f
2934 \\ ldeorh w0, w0, [x1]
2935 \\ cbz wzr, 1f
2936 \\8:
2937 \\ mov w16, w0
2938 \\0:
2939 \\ ldxrh w0, [x1]
2940 \\ eor w17, w0, w16
2941 \\ stxrh w15, w17, [x1]
2942 \\ cbnz w15, 0b
2943 \\1:
2944 : [ret] "={w0}" (-> u16),
2945 : [val] "{w0}" (val),
2946 [ptr] "{x1}" (ptr),
2947 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2948 : "w15", "w16", "w17", "memory"
2949 );
2950}
2951fn ldeor2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2952 @setRuntimeSafety(false);
2953 __init_aarch64_have_lse_atomics();
2954
2955 return asm volatile (
2956 \\ cbz w16, 8f
2957 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
2958 \\ cbz wzr, 1f
2959 \\8:
2960 \\ mov w16, w0
2961 \\0:
2962 \\ ldxrh w0, [x1]
2963 \\ eor w17, w0, w16
2964 \\ stxrh w15, w17, [x1]
2965 \\ cbnz w15, 0b
2966 \\1:
2967 : [ret] "={w0}" (-> u16),
2968 : [val] "{w0}" (val),
2969 [ptr] "{x1}" (ptr),
2970 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2971 : "w15", "w16", "w17", "memory"
2972 );
2973}
2974const __aarch64_ldeor2_relax = if (is_darwin)
2975 if (has_lse)
2976 ldeor2RelaxDarwinLse
2977 else
2978 ldeor2RelaxDarwinNolse
2979else if (has_lse)
2980 ldeor2RelaxNondarwinLse
2981else
2982 ldeor2RelaxNondarwinNolse;
2983fn ldset2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2984 @setRuntimeSafety(false);
2985 __init_aarch64_have_lse_atomics();
2986
2987 return asm volatile (
2988 \\ cbz w16, 8f
2989 \\ ldseth w0, w0, [x1]
2990 \\ cbz wzr, 1f
2991 \\8:
2992 \\ mov w16, w0
2993 \\0:
2994 \\ ldxrh w0, [x1]
2995 \\ orr w17, w0, w16
2996 \\ stxrh w15, w17, [x1]
2997 \\ cbnz w15, 0b
2998 \\1:
2999 : [ret] "={w0}" (-> u16),
3000 : [val] "{w0}" (val),
3001 [ptr] "{x1}" (ptr),
3002 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3003 : "w15", "w16", "w17", "memory"
3004 );
3005}
3006fn ldset2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3007 @setRuntimeSafety(false);
3008 __init_aarch64_have_lse_atomics();
3009
3010 return asm volatile (
3011 \\ cbz w16, 8f
3012 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
3013 \\ cbz wzr, 1f
3014 \\8:
3015 \\ mov w16, w0
3016 \\0:
3017 \\ ldxrh w0, [x1]
3018 \\ orr w17, w0, w16
3019 \\ stxrh w15, w17, [x1]
3020 \\ cbnz w15, 0b
3021 \\1:
3022 : [ret] "={w0}" (-> u16),
3023 : [val] "{w0}" (val),
3024 [ptr] "{x1}" (ptr),
3025 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3026 : "w15", "w16", "w17", "memory"
3027 );
3028}
3029fn ldset2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3030 @setRuntimeSafety(false);
3031 __init_aarch64_have_lse_atomics();
3032
3033 return asm volatile (
3034 \\ cbz w16, 8f
3035 \\ ldseth w0, w0, [x1]
3036 \\ cbz wzr, 1f
3037 \\8:
3038 \\ mov w16, w0
3039 \\0:
3040 \\ ldxrh w0, [x1]
3041 \\ orr w17, w0, w16
3042 \\ stxrh w15, w17, [x1]
3043 \\ cbnz w15, 0b
3044 \\1:
3045 : [ret] "={w0}" (-> u16),
3046 : [val] "{w0}" (val),
3047 [ptr] "{x1}" (ptr),
3048 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3049 : "w15", "w16", "w17", "memory"
3050 );
3051}
3052fn ldset2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3053 @setRuntimeSafety(false);
3054 __init_aarch64_have_lse_atomics();
3055
3056 return asm volatile (
3057 \\ cbz w16, 8f
3058 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
3059 \\ cbz wzr, 1f
3060 \\8:
3061 \\ mov w16, w0
3062 \\0:
3063 \\ ldxrh w0, [x1]
3064 \\ orr w17, w0, w16
3065 \\ stxrh w15, w17, [x1]
3066 \\ cbnz w15, 0b
3067 \\1:
3068 : [ret] "={w0}" (-> u16),
3069 : [val] "{w0}" (val),
3070 [ptr] "{x1}" (ptr),
3071 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3072 : "w15", "w16", "w17", "memory"
3073 );
3074}
3075const __aarch64_ldset2_relax = if (is_darwin)
3076 if (has_lse)
3077 ldset2RelaxDarwinLse
3078 else
3079 ldset2RelaxDarwinNolse
3080else if (has_lse)
3081 ldset2RelaxNondarwinLse
3082else
3083 ldset2RelaxNondarwinNolse;
3084fn cas2AcqDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3085 @setRuntimeSafety(false);
3086 __init_aarch64_have_lse_atomics();
3087
3088 return asm volatile (
3089 \\ cbz w16, 8f
3090 \\ casah w0, w1, [x2]
3091 \\
3092 \\ cbz wzr, 1f
3093 \\8:
3094 \\ uxth w16, w0
3095 \\0:
3096 \\ ldaxrh w0, [x2]
3097 \\ cmp w0, w16
3098 \\ bne 1f
3099 \\ stxrh w17, w1, [x2]
3100 \\ cbnz w17, 0b
3101 \\1:
3102 : [ret] "={w0}" (-> u16),
3103 : [expected] "{w0}" (expected),
3104 [desired] "{w1}" (desired),
3105 [ptr] "{x2}" (ptr),
3106 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3107 : "w15", "w16", "w17", "memory"
3108 );
3109}
3110fn cas2AcqDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3111 @setRuntimeSafety(false);
3112 __init_aarch64_have_lse_atomics();
3113
3114 return asm volatile (
3115 \\ cbz w16, 8f
3116 \\ .inst 0x08a07c41 + 0x40000000 + 0x400000
3117 \\
3118 \\ cbz wzr, 1f
3119 \\8:
3120 \\ uxth w16, w0
3121 \\0:
3122 \\ ldaxrh w0, [x2]
3123 \\ cmp w0, w16
3124 \\ bne 1f
3125 \\ stxrh w17, w1, [x2]
3126 \\ cbnz w17, 0b
3127 \\1:
3128 : [ret] "={w0}" (-> u16),
3129 : [expected] "{w0}" (expected),
3130 [desired] "{w1}" (desired),
3131 [ptr] "{x2}" (ptr),
3132 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3133 : "w15", "w16", "w17", "memory"
3134 );
3135}
3136fn cas2AcqNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3137 @setRuntimeSafety(false);
3138 __init_aarch64_have_lse_atomics();
3139
3140 return asm volatile (
3141 \\ cbz w16, 8f
3142 \\ casah w0, w1, [x2]
3143 \\
3144 \\ cbz wzr, 1f
3145 \\8:
3146 \\ uxth w16, w0
3147 \\0:
3148 \\ ldaxrh w0, [x2]
3149 \\ cmp w0, w16
3150 \\ bne 1f
3151 \\ stxrh w17, w1, [x2]
3152 \\ cbnz w17, 0b
3153 \\1:
3154 : [ret] "={w0}" (-> u16),
3155 : [expected] "{w0}" (expected),
3156 [desired] "{w1}" (desired),
3157 [ptr] "{x2}" (ptr),
3158 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3159 : "w15", "w16", "w17", "memory"
3160 );
3161}
3162fn cas2AcqNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3163 @setRuntimeSafety(false);
3164 __init_aarch64_have_lse_atomics();
3165
3166 return asm volatile (
3167 \\ cbz w16, 8f
3168 \\ .inst 0x08a07c41 + 0x40000000 + 0x400000
3169 \\
3170 \\ cbz wzr, 1f
3171 \\8:
3172 \\ uxth w16, w0
3173 \\0:
3174 \\ ldaxrh w0, [x2]
3175 \\ cmp w0, w16
3176 \\ bne 1f
3177 \\ stxrh w17, w1, [x2]
3178 \\ cbnz w17, 0b
3179 \\1:
3180 : [ret] "={w0}" (-> u16),
3181 : [expected] "{w0}" (expected),
3182 [desired] "{w1}" (desired),
3183 [ptr] "{x2}" (ptr),
3184 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3185 : "w15", "w16", "w17", "memory"
3186 );
3187}
3188const __aarch64_cas2_acq = if (is_darwin)
3189 if (has_lse)
3190 cas2AcqDarwinLse
3191 else
3192 cas2AcqDarwinNolse
3193else if (has_lse)
3194 cas2AcqNondarwinLse
3195else
3196 cas2AcqNondarwinNolse;
3197fn swp2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3198 @setRuntimeSafety(false);
3199 __init_aarch64_have_lse_atomics();
3200
3201 return asm volatile (
3202 \\ cbz w16, 8f
3203 \\ swpah w0, w0, [x1]
3204 \\ cbz wzr, 1f
3205 \\8:
3206 \\ mov w16, w0
3207 \\0:
3208 \\ ldaxrh w0, [x1]
3209 \\ stxrh w17, w16, [x1]
3210 \\ cbnz w17, 0b
3211 \\1:
3212 : [ret] "={w0}" (-> u16),
3213 : [val] "{w0}" (val),
3214 [ptr] "{x1}" (ptr),
3215 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3216 : "w15", "w16", "w17", "memory"
3217 );
3218}
3219fn swp2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3220 @setRuntimeSafety(false);
3221 __init_aarch64_have_lse_atomics();
3222
3223 return asm volatile (
3224 \\ cbz w16, 8f
3225 \\ .inst 0x38208020 + 0x40000000 + 0x800000
3226 \\ cbz wzr, 1f
3227 \\8:
3228 \\ mov w16, w0
3229 \\0:
3230 \\ ldaxrh w0, [x1]
3231 \\ stxrh w17, w16, [x1]
3232 \\ cbnz w17, 0b
3233 \\1:
3234 : [ret] "={w0}" (-> u16),
3235 : [val] "{w0}" (val),
3236 [ptr] "{x1}" (ptr),
3237 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3238 : "w15", "w16", "w17", "memory"
3239 );
3240}
3241fn swp2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3242 @setRuntimeSafety(false);
3243 __init_aarch64_have_lse_atomics();
3244
3245 return asm volatile (
3246 \\ cbz w16, 8f
3247 \\ swpah w0, w0, [x1]
3248 \\ cbz wzr, 1f
3249 \\8:
3250 \\ mov w16, w0
3251 \\0:
3252 \\ ldaxrh w0, [x1]
3253 \\ stxrh w17, w16, [x1]
3254 \\ cbnz w17, 0b
3255 \\1:
3256 : [ret] "={w0}" (-> u16),
3257 : [val] "{w0}" (val),
3258 [ptr] "{x1}" (ptr),
3259 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3260 : "w15", "w16", "w17", "memory"
3261 );
3262}
3263fn swp2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3264 @setRuntimeSafety(false);
3265 __init_aarch64_have_lse_atomics();
3266
3267 return asm volatile (
3268 \\ cbz w16, 8f
3269 \\ .inst 0x38208020 + 0x40000000 + 0x800000
3270 \\ cbz wzr, 1f
3271 \\8:
3272 \\ mov w16, w0
3273 \\0:
3274 \\ ldaxrh w0, [x1]
3275 \\ stxrh w17, w16, [x1]
3276 \\ cbnz w17, 0b
3277 \\1:
3278 : [ret] "={w0}" (-> u16),
3279 : [val] "{w0}" (val),
3280 [ptr] "{x1}" (ptr),
3281 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3282 : "w15", "w16", "w17", "memory"
3283 );
3284}
3285const __aarch64_swp2_acq = if (is_darwin)
3286 if (has_lse)
3287 swp2AcqDarwinLse
3288 else
3289 swp2AcqDarwinNolse
3290else if (has_lse)
3291 swp2AcqNondarwinLse
3292else
3293 swp2AcqNondarwinNolse;
3294fn ldadd2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3295 @setRuntimeSafety(false);
3296 __init_aarch64_have_lse_atomics();
3297
3298 return asm volatile (
3299 \\ cbz w16, 8f
3300 \\ ldaddah w0, w0, [x1]
3301 \\ cbz wzr, 1f
3302 \\8:
3303 \\ mov w16, w0
3304 \\0:
3305 \\ ldaxrh w0, [x1]
3306 \\ add w17, w0, w16
3307 \\ stxrh w15, w17, [x1]
3308 \\ cbnz w15, 0b
3309 \\1:
3310 : [ret] "={w0}" (-> u16),
3311 : [val] "{w0}" (val),
3312 [ptr] "{x1}" (ptr),
3313 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3314 : "w15", "w16", "w17", "memory"
3315 );
3316}
3317fn ldadd2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3318 @setRuntimeSafety(false);
3319 __init_aarch64_have_lse_atomics();
3320
3321 return asm volatile (
3322 \\ cbz w16, 8f
3323 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
3324 \\ cbz wzr, 1f
3325 \\8:
3326 \\ mov w16, w0
3327 \\0:
3328 \\ ldaxrh w0, [x1]
3329 \\ add w17, w0, w16
3330 \\ stxrh w15, w17, [x1]
3331 \\ cbnz w15, 0b
3332 \\1:
3333 : [ret] "={w0}" (-> u16),
3334 : [val] "{w0}" (val),
3335 [ptr] "{x1}" (ptr),
3336 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3337 : "w15", "w16", "w17", "memory"
3338 );
3339}
3340fn ldadd2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3341 @setRuntimeSafety(false);
3342 __init_aarch64_have_lse_atomics();
3343
3344 return asm volatile (
3345 \\ cbz w16, 8f
3346 \\ ldaddah w0, w0, [x1]
3347 \\ cbz wzr, 1f
3348 \\8:
3349 \\ mov w16, w0
3350 \\0:
3351 \\ ldaxrh w0, [x1]
3352 \\ add w17, w0, w16
3353 \\ stxrh w15, w17, [x1]
3354 \\ cbnz w15, 0b
3355 \\1:
3356 : [ret] "={w0}" (-> u16),
3357 : [val] "{w0}" (val),
3358 [ptr] "{x1}" (ptr),
3359 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3360 : "w15", "w16", "w17", "memory"
3361 );
3362}
3363fn ldadd2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3364 @setRuntimeSafety(false);
3365 __init_aarch64_have_lse_atomics();
3366
3367 return asm volatile (
3368 \\ cbz w16, 8f
3369 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
3370 \\ cbz wzr, 1f
3371 \\8:
3372 \\ mov w16, w0
3373 \\0:
3374 \\ ldaxrh w0, [x1]
3375 \\ add w17, w0, w16
3376 \\ stxrh w15, w17, [x1]
3377 \\ cbnz w15, 0b
3378 \\1:
3379 : [ret] "={w0}" (-> u16),
3380 : [val] "{w0}" (val),
3381 [ptr] "{x1}" (ptr),
3382 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3383 : "w15", "w16", "w17", "memory"
3384 );
3385}
3386const __aarch64_ldadd2_acq = if (is_darwin)
3387 if (has_lse)
3388 ldadd2AcqDarwinLse
3389 else
3390 ldadd2AcqDarwinNolse
3391else if (has_lse)
3392 ldadd2AcqNondarwinLse
3393else
3394 ldadd2AcqNondarwinNolse;
3395fn ldclr2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3396 @setRuntimeSafety(false);
3397 __init_aarch64_have_lse_atomics();
3398
3399 return asm volatile (
3400 \\ cbz w16, 8f
3401 \\ ldclrah w0, w0, [x1]
3402 \\ cbz wzr, 1f
3403 \\8:
3404 \\ mov w16, w0
3405 \\0:
3406 \\ ldaxrh w0, [x1]
3407 \\ bic w17, w0, w16
3408 \\ stxrh w15, w17, [x1]
3409 \\ cbnz w15, 0b
3410 \\1:
3411 : [ret] "={w0}" (-> u16),
3412 : [val] "{w0}" (val),
3413 [ptr] "{x1}" (ptr),
3414 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3415 : "w15", "w16", "w17", "memory"
3416 );
3417}
3418fn ldclr2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3419 @setRuntimeSafety(false);
3420 __init_aarch64_have_lse_atomics();
3421
3422 return asm volatile (
3423 \\ cbz w16, 8f
3424 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
3425 \\ cbz wzr, 1f
3426 \\8:
3427 \\ mov w16, w0
3428 \\0:
3429 \\ ldaxrh w0, [x1]
3430 \\ bic w17, w0, w16
3431 \\ stxrh w15, w17, [x1]
3432 \\ cbnz w15, 0b
3433 \\1:
3434 : [ret] "={w0}" (-> u16),
3435 : [val] "{w0}" (val),
3436 [ptr] "{x1}" (ptr),
3437 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3438 : "w15", "w16", "w17", "memory"
3439 );
3440}
3441fn ldclr2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3442 @setRuntimeSafety(false);
3443 __init_aarch64_have_lse_atomics();
3444
3445 return asm volatile (
3446 \\ cbz w16, 8f
3447 \\ ldclrah w0, w0, [x1]
3448 \\ cbz wzr, 1f
3449 \\8:
3450 \\ mov w16, w0
3451 \\0:
3452 \\ ldaxrh w0, [x1]
3453 \\ bic w17, w0, w16
3454 \\ stxrh w15, w17, [x1]
3455 \\ cbnz w15, 0b
3456 \\1:
3457 : [ret] "={w0}" (-> u16),
3458 : [val] "{w0}" (val),
3459 [ptr] "{x1}" (ptr),
3460 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3461 : "w15", "w16", "w17", "memory"
3462 );
3463}
3464fn ldclr2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3465 @setRuntimeSafety(false);
3466 __init_aarch64_have_lse_atomics();
3467
3468 return asm volatile (
3469 \\ cbz w16, 8f
3470 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
3471 \\ cbz wzr, 1f
3472 \\8:
3473 \\ mov w16, w0
3474 \\0:
3475 \\ ldaxrh w0, [x1]
3476 \\ bic w17, w0, w16
3477 \\ stxrh w15, w17, [x1]
3478 \\ cbnz w15, 0b
3479 \\1:
3480 : [ret] "={w0}" (-> u16),
3481 : [val] "{w0}" (val),
3482 [ptr] "{x1}" (ptr),
3483 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3484 : "w15", "w16", "w17", "memory"
3485 );
3486}
3487const __aarch64_ldclr2_acq = if (is_darwin)
3488 if (has_lse)
3489 ldclr2AcqDarwinLse
3490 else
3491 ldclr2AcqDarwinNolse
3492else if (has_lse)
3493 ldclr2AcqNondarwinLse
3494else
3495 ldclr2AcqNondarwinNolse;
3496fn ldeor2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3497 @setRuntimeSafety(false);
3498 __init_aarch64_have_lse_atomics();
3499
3500 return asm volatile (
3501 \\ cbz w16, 8f
3502 \\ ldeorah w0, w0, [x1]
3503 \\ cbz wzr, 1f
3504 \\8:
3505 \\ mov w16, w0
3506 \\0:
3507 \\ ldaxrh w0, [x1]
3508 \\ eor w17, w0, w16
3509 \\ stxrh w15, w17, [x1]
3510 \\ cbnz w15, 0b
3511 \\1:
3512 : [ret] "={w0}" (-> u16),
3513 : [val] "{w0}" (val),
3514 [ptr] "{x1}" (ptr),
3515 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3516 : "w15", "w16", "w17", "memory"
3517 );
3518}
3519fn ldeor2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3520 @setRuntimeSafety(false);
3521 __init_aarch64_have_lse_atomics();
3522
3523 return asm volatile (
3524 \\ cbz w16, 8f
3525 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
3526 \\ cbz wzr, 1f
3527 \\8:
3528 \\ mov w16, w0
3529 \\0:
3530 \\ ldaxrh w0, [x1]
3531 \\ eor w17, w0, w16
3532 \\ stxrh w15, w17, [x1]
3533 \\ cbnz w15, 0b
3534 \\1:
3535 : [ret] "={w0}" (-> u16),
3536 : [val] "{w0}" (val),
3537 [ptr] "{x1}" (ptr),
3538 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3539 : "w15", "w16", "w17", "memory"
3540 );
3541}
3542fn ldeor2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3543 @setRuntimeSafety(false);
3544 __init_aarch64_have_lse_atomics();
3545
3546 return asm volatile (
3547 \\ cbz w16, 8f
3548 \\ ldeorah w0, w0, [x1]
3549 \\ cbz wzr, 1f
3550 \\8:
3551 \\ mov w16, w0
3552 \\0:
3553 \\ ldaxrh w0, [x1]
3554 \\ eor w17, w0, w16
3555 \\ stxrh w15, w17, [x1]
3556 \\ cbnz w15, 0b
3557 \\1:
3558 : [ret] "={w0}" (-> u16),
3559 : [val] "{w0}" (val),
3560 [ptr] "{x1}" (ptr),
3561 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3562 : "w15", "w16", "w17", "memory"
3563 );
3564}
3565fn ldeor2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3566 @setRuntimeSafety(false);
3567 __init_aarch64_have_lse_atomics();
3568
3569 return asm volatile (
3570 \\ cbz w16, 8f
3571 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
3572 \\ cbz wzr, 1f
3573 \\8:
3574 \\ mov w16, w0
3575 \\0:
3576 \\ ldaxrh w0, [x1]
3577 \\ eor w17, w0, w16
3578 \\ stxrh w15, w17, [x1]
3579 \\ cbnz w15, 0b
3580 \\1:
3581 : [ret] "={w0}" (-> u16),
3582 : [val] "{w0}" (val),
3583 [ptr] "{x1}" (ptr),
3584 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3585 : "w15", "w16", "w17", "memory"
3586 );
3587}
3588const __aarch64_ldeor2_acq = if (is_darwin)
3589 if (has_lse)
3590 ldeor2AcqDarwinLse
3591 else
3592 ldeor2AcqDarwinNolse
3593else if (has_lse)
3594 ldeor2AcqNondarwinLse
3595else
3596 ldeor2AcqNondarwinNolse;
3597fn ldset2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3598 @setRuntimeSafety(false);
3599 __init_aarch64_have_lse_atomics();
3600
3601 return asm volatile (
3602 \\ cbz w16, 8f
3603 \\ ldsetah w0, w0, [x1]
3604 \\ cbz wzr, 1f
3605 \\8:
3606 \\ mov w16, w0
3607 \\0:
3608 \\ ldaxrh w0, [x1]
3609 \\ orr w17, w0, w16
3610 \\ stxrh w15, w17, [x1]
3611 \\ cbnz w15, 0b
3612 \\1:
3613 : [ret] "={w0}" (-> u16),
3614 : [val] "{w0}" (val),
3615 [ptr] "{x1}" (ptr),
3616 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3617 : "w15", "w16", "w17", "memory"
3618 );
3619}
3620fn ldset2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3621 @setRuntimeSafety(false);
3622 __init_aarch64_have_lse_atomics();
3623
3624 return asm volatile (
3625 \\ cbz w16, 8f
3626 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
3627 \\ cbz wzr, 1f
3628 \\8:
3629 \\ mov w16, w0
3630 \\0:
3631 \\ ldaxrh w0, [x1]
3632 \\ orr w17, w0, w16
3633 \\ stxrh w15, w17, [x1]
3634 \\ cbnz w15, 0b
3635 \\1:
3636 : [ret] "={w0}" (-> u16),
3637 : [val] "{w0}" (val),
3638 [ptr] "{x1}" (ptr),
3639 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3640 : "w15", "w16", "w17", "memory"
3641 );
3642}
3643fn ldset2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3644 @setRuntimeSafety(false);
3645 __init_aarch64_have_lse_atomics();
3646
3647 return asm volatile (
3648 \\ cbz w16, 8f
3649 \\ ldsetah w0, w0, [x1]
3650 \\ cbz wzr, 1f
3651 \\8:
3652 \\ mov w16, w0
3653 \\0:
3654 \\ ldaxrh w0, [x1]
3655 \\ orr w17, w0, w16
3656 \\ stxrh w15, w17, [x1]
3657 \\ cbnz w15, 0b
3658 \\1:
3659 : [ret] "={w0}" (-> u16),
3660 : [val] "{w0}" (val),
3661 [ptr] "{x1}" (ptr),
3662 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3663 : "w15", "w16", "w17", "memory"
3664 );
3665}
3666fn ldset2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3667 @setRuntimeSafety(false);
3668 __init_aarch64_have_lse_atomics();
3669
3670 return asm volatile (
3671 \\ cbz w16, 8f
3672 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
3673 \\ cbz wzr, 1f
3674 \\8:
3675 \\ mov w16, w0
3676 \\0:
3677 \\ ldaxrh w0, [x1]
3678 \\ orr w17, w0, w16
3679 \\ stxrh w15, w17, [x1]
3680 \\ cbnz w15, 0b
3681 \\1:
3682 : [ret] "={w0}" (-> u16),
3683 : [val] "{w0}" (val),
3684 [ptr] "{x1}" (ptr),
3685 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3686 : "w15", "w16", "w17", "memory"
3687 );
3688}
3689const __aarch64_ldset2_acq = if (is_darwin)
3690 if (has_lse)
3691 ldset2AcqDarwinLse
3692 else
3693 ldset2AcqDarwinNolse
3694else if (has_lse)
3695 ldset2AcqNondarwinLse
3696else
3697 ldset2AcqNondarwinNolse;
3698fn cas2RelDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3699 @setRuntimeSafety(false);
3700 __init_aarch64_have_lse_atomics();
3701
3702 return asm volatile (
3703 \\ cbz w16, 8f
3704 \\ caslh w0, w1, [x2]
3705 \\
3706 \\ cbz wzr, 1f
3707 \\8:
3708 \\ uxth w16, w0
3709 \\0:
3710 \\ ldxrh w0, [x2]
3711 \\ cmp w0, w16
3712 \\ bne 1f
3713 \\ stlxrh w17, w1, [x2]
3714 \\ cbnz w17, 0b
3715 \\1:
3716 : [ret] "={w0}" (-> u16),
3717 : [expected] "{w0}" (expected),
3718 [desired] "{w1}" (desired),
3719 [ptr] "{x2}" (ptr),
3720 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3721 : "w15", "w16", "w17", "memory"
3722 );
3723}
3724fn cas2RelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3725 @setRuntimeSafety(false);
3726 __init_aarch64_have_lse_atomics();
3727
3728 return asm volatile (
3729 \\ cbz w16, 8f
3730 \\ .inst 0x08a07c41 + 0x40000000 + 0x008000
3731 \\
3732 \\ cbz wzr, 1f
3733 \\8:
3734 \\ uxth w16, w0
3735 \\0:
3736 \\ ldxrh w0, [x2]
3737 \\ cmp w0, w16
3738 \\ bne 1f
3739 \\ stlxrh w17, w1, [x2]
3740 \\ cbnz w17, 0b
3741 \\1:
3742 : [ret] "={w0}" (-> u16),
3743 : [expected] "{w0}" (expected),
3744 [desired] "{w1}" (desired),
3745 [ptr] "{x2}" (ptr),
3746 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3747 : "w15", "w16", "w17", "memory"
3748 );
3749}
3750fn cas2RelNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3751 @setRuntimeSafety(false);
3752 __init_aarch64_have_lse_atomics();
3753
3754 return asm volatile (
3755 \\ cbz w16, 8f
3756 \\ caslh w0, w1, [x2]
3757 \\
3758 \\ cbz wzr, 1f
3759 \\8:
3760 \\ uxth w16, w0
3761 \\0:
3762 \\ ldxrh w0, [x2]
3763 \\ cmp w0, w16
3764 \\ bne 1f
3765 \\ stlxrh w17, w1, [x2]
3766 \\ cbnz w17, 0b
3767 \\1:
3768 : [ret] "={w0}" (-> u16),
3769 : [expected] "{w0}" (expected),
3770 [desired] "{w1}" (desired),
3771 [ptr] "{x2}" (ptr),
3772 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3773 : "w15", "w16", "w17", "memory"
3774 );
3775}
3776fn cas2RelNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3777 @setRuntimeSafety(false);
3778 __init_aarch64_have_lse_atomics();
3779
3780 return asm volatile (
3781 \\ cbz w16, 8f
3782 \\ .inst 0x08a07c41 + 0x40000000 + 0x008000
3783 \\
3784 \\ cbz wzr, 1f
3785 \\8:
3786 \\ uxth w16, w0
3787 \\0:
3788 \\ ldxrh w0, [x2]
3789 \\ cmp w0, w16
3790 \\ bne 1f
3791 \\ stlxrh w17, w1, [x2]
3792 \\ cbnz w17, 0b
3793 \\1:
3794 : [ret] "={w0}" (-> u16),
3795 : [expected] "{w0}" (expected),
3796 [desired] "{w1}" (desired),
3797 [ptr] "{x2}" (ptr),
3798 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3799 : "w15", "w16", "w17", "memory"
3800 );
3801}
3802const __aarch64_cas2_rel = if (is_darwin)
3803 if (has_lse)
3804 cas2RelDarwinLse
3805 else
3806 cas2RelDarwinNolse
3807else if (has_lse)
3808 cas2RelNondarwinLse
3809else
3810 cas2RelNondarwinNolse;
3811fn swp2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3812 @setRuntimeSafety(false);
3813 __init_aarch64_have_lse_atomics();
3814
3815 return asm volatile (
3816 \\ cbz w16, 8f
3817 \\ swplh w0, w0, [x1]
3818 \\ cbz wzr, 1f
3819 \\8:
3820 \\ mov w16, w0
3821 \\0:
3822 \\ ldxrh w0, [x1]
3823 \\ stlxrh w17, w16, [x1]
3824 \\ cbnz w17, 0b
3825 \\1:
3826 : [ret] "={w0}" (-> u16),
3827 : [val] "{w0}" (val),
3828 [ptr] "{x1}" (ptr),
3829 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3830 : "w15", "w16", "w17", "memory"
3831 );
3832}
3833fn swp2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3834 @setRuntimeSafety(false);
3835 __init_aarch64_have_lse_atomics();
3836
3837 return asm volatile (
3838 \\ cbz w16, 8f
3839 \\ .inst 0x38208020 + 0x40000000 + 0x400000
3840 \\ cbz wzr, 1f
3841 \\8:
3842 \\ mov w16, w0
3843 \\0:
3844 \\ ldxrh w0, [x1]
3845 \\ stlxrh w17, w16, [x1]
3846 \\ cbnz w17, 0b
3847 \\1:
3848 : [ret] "={w0}" (-> u16),
3849 : [val] "{w0}" (val),
3850 [ptr] "{x1}" (ptr),
3851 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3852 : "w15", "w16", "w17", "memory"
3853 );
3854}
3855fn swp2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3856 @setRuntimeSafety(false);
3857 __init_aarch64_have_lse_atomics();
3858
3859 return asm volatile (
3860 \\ cbz w16, 8f
3861 \\ swplh w0, w0, [x1]
3862 \\ cbz wzr, 1f
3863 \\8:
3864 \\ mov w16, w0
3865 \\0:
3866 \\ ldxrh w0, [x1]
3867 \\ stlxrh w17, w16, [x1]
3868 \\ cbnz w17, 0b
3869 \\1:
3870 : [ret] "={w0}" (-> u16),
3871 : [val] "{w0}" (val),
3872 [ptr] "{x1}" (ptr),
3873 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3874 : "w15", "w16", "w17", "memory"
3875 );
3876}
3877fn swp2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3878 @setRuntimeSafety(false);
3879 __init_aarch64_have_lse_atomics();
3880
3881 return asm volatile (
3882 \\ cbz w16, 8f
3883 \\ .inst 0x38208020 + 0x40000000 + 0x400000
3884 \\ cbz wzr, 1f
3885 \\8:
3886 \\ mov w16, w0
3887 \\0:
3888 \\ ldxrh w0, [x1]
3889 \\ stlxrh w17, w16, [x1]
3890 \\ cbnz w17, 0b
3891 \\1:
3892 : [ret] "={w0}" (-> u16),
3893 : [val] "{w0}" (val),
3894 [ptr] "{x1}" (ptr),
3895 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3896 : "w15", "w16", "w17", "memory"
3897 );
3898}
3899const __aarch64_swp2_rel = if (is_darwin)
3900 if (has_lse)
3901 swp2RelDarwinLse
3902 else
3903 swp2RelDarwinNolse
3904else if (has_lse)
3905 swp2RelNondarwinLse
3906else
3907 swp2RelNondarwinNolse;
3908fn ldadd2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3909 @setRuntimeSafety(false);
3910 __init_aarch64_have_lse_atomics();
3911
3912 return asm volatile (
3913 \\ cbz w16, 8f
3914 \\ ldaddlh w0, w0, [x1]
3915 \\ cbz wzr, 1f
3916 \\8:
3917 \\ mov w16, w0
3918 \\0:
3919 \\ ldxrh w0, [x1]
3920 \\ add w17, w0, w16
3921 \\ stlxrh w15, w17, [x1]
3922 \\ cbnz w15, 0b
3923 \\1:
3924 : [ret] "={w0}" (-> u16),
3925 : [val] "{w0}" (val),
3926 [ptr] "{x1}" (ptr),
3927 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3928 : "w15", "w16", "w17", "memory"
3929 );
3930}
3931fn ldadd2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3932 @setRuntimeSafety(false);
3933 __init_aarch64_have_lse_atomics();
3934
3935 return asm volatile (
3936 \\ cbz w16, 8f
3937 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
3938 \\ cbz wzr, 1f
3939 \\8:
3940 \\ mov w16, w0
3941 \\0:
3942 \\ ldxrh w0, [x1]
3943 \\ add w17, w0, w16
3944 \\ stlxrh w15, w17, [x1]
3945 \\ cbnz w15, 0b
3946 \\1:
3947 : [ret] "={w0}" (-> u16),
3948 : [val] "{w0}" (val),
3949 [ptr] "{x1}" (ptr),
3950 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3951 : "w15", "w16", "w17", "memory"
3952 );
3953}
3954fn ldadd2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3955 @setRuntimeSafety(false);
3956 __init_aarch64_have_lse_atomics();
3957
3958 return asm volatile (
3959 \\ cbz w16, 8f
3960 \\ ldaddlh w0, w0, [x1]
3961 \\ cbz wzr, 1f
3962 \\8:
3963 \\ mov w16, w0
3964 \\0:
3965 \\ ldxrh w0, [x1]
3966 \\ add w17, w0, w16
3967 \\ stlxrh w15, w17, [x1]
3968 \\ cbnz w15, 0b
3969 \\1:
3970 : [ret] "={w0}" (-> u16),
3971 : [val] "{w0}" (val),
3972 [ptr] "{x1}" (ptr),
3973 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3974 : "w15", "w16", "w17", "memory"
3975 );
3976}
3977fn ldadd2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3978 @setRuntimeSafety(false);
3979 __init_aarch64_have_lse_atomics();
3980
3981 return asm volatile (
3982 \\ cbz w16, 8f
3983 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
3984 \\ cbz wzr, 1f
3985 \\8:
3986 \\ mov w16, w0
3987 \\0:
3988 \\ ldxrh w0, [x1]
3989 \\ add w17, w0, w16
3990 \\ stlxrh w15, w17, [x1]
3991 \\ cbnz w15, 0b
3992 \\1:
3993 : [ret] "={w0}" (-> u16),
3994 : [val] "{w0}" (val),
3995 [ptr] "{x1}" (ptr),
3996 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3997 : "w15", "w16", "w17", "memory"
3998 );
3999}
4000const __aarch64_ldadd2_rel = if (is_darwin)
4001 if (has_lse)
4002 ldadd2RelDarwinLse
4003 else
4004 ldadd2RelDarwinNolse
4005else if (has_lse)
4006 ldadd2RelNondarwinLse
4007else
4008 ldadd2RelNondarwinNolse;
4009fn ldclr2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4010 @setRuntimeSafety(false);
4011 __init_aarch64_have_lse_atomics();
4012
4013 return asm volatile (
4014 \\ cbz w16, 8f
4015 \\ ldclrlh w0, w0, [x1]
4016 \\ cbz wzr, 1f
4017 \\8:
4018 \\ mov w16, w0
4019 \\0:
4020 \\ ldxrh w0, [x1]
4021 \\ bic w17, w0, w16
4022 \\ stlxrh w15, w17, [x1]
4023 \\ cbnz w15, 0b
4024 \\1:
4025 : [ret] "={w0}" (-> u16),
4026 : [val] "{w0}" (val),
4027 [ptr] "{x1}" (ptr),
4028 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4029 : "w15", "w16", "w17", "memory"
4030 );
4031}
4032fn ldclr2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4033 @setRuntimeSafety(false);
4034 __init_aarch64_have_lse_atomics();
4035
4036 return asm volatile (
4037 \\ cbz w16, 8f
4038 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
4039 \\ cbz wzr, 1f
4040 \\8:
4041 \\ mov w16, w0
4042 \\0:
4043 \\ ldxrh w0, [x1]
4044 \\ bic w17, w0, w16
4045 \\ stlxrh w15, w17, [x1]
4046 \\ cbnz w15, 0b
4047 \\1:
4048 : [ret] "={w0}" (-> u16),
4049 : [val] "{w0}" (val),
4050 [ptr] "{x1}" (ptr),
4051 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4052 : "w15", "w16", "w17", "memory"
4053 );
4054}
4055fn ldclr2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4056 @setRuntimeSafety(false);
4057 __init_aarch64_have_lse_atomics();
4058
4059 return asm volatile (
4060 \\ cbz w16, 8f
4061 \\ ldclrlh w0, w0, [x1]
4062 \\ cbz wzr, 1f
4063 \\8:
4064 \\ mov w16, w0
4065 \\0:
4066 \\ ldxrh w0, [x1]
4067 \\ bic w17, w0, w16
4068 \\ stlxrh w15, w17, [x1]
4069 \\ cbnz w15, 0b
4070 \\1:
4071 : [ret] "={w0}" (-> u16),
4072 : [val] "{w0}" (val),
4073 [ptr] "{x1}" (ptr),
4074 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4075 : "w15", "w16", "w17", "memory"
4076 );
4077}
4078fn ldclr2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4079 @setRuntimeSafety(false);
4080 __init_aarch64_have_lse_atomics();
4081
4082 return asm volatile (
4083 \\ cbz w16, 8f
4084 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
4085 \\ cbz wzr, 1f
4086 \\8:
4087 \\ mov w16, w0
4088 \\0:
4089 \\ ldxrh w0, [x1]
4090 \\ bic w17, w0, w16
4091 \\ stlxrh w15, w17, [x1]
4092 \\ cbnz w15, 0b
4093 \\1:
4094 : [ret] "={w0}" (-> u16),
4095 : [val] "{w0}" (val),
4096 [ptr] "{x1}" (ptr),
4097 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4098 : "w15", "w16", "w17", "memory"
4099 );
4100}
4101const __aarch64_ldclr2_rel = if (is_darwin)
4102 if (has_lse)
4103 ldclr2RelDarwinLse
4104 else
4105 ldclr2RelDarwinNolse
4106else if (has_lse)
4107 ldclr2RelNondarwinLse
4108else
4109 ldclr2RelNondarwinNolse;
4110fn ldeor2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4111 @setRuntimeSafety(false);
4112 __init_aarch64_have_lse_atomics();
4113
4114 return asm volatile (
4115 \\ cbz w16, 8f
4116 \\ ldeorlh w0, w0, [x1]
4117 \\ cbz wzr, 1f
4118 \\8:
4119 \\ mov w16, w0
4120 \\0:
4121 \\ ldxrh w0, [x1]
4122 \\ eor w17, w0, w16
4123 \\ stlxrh w15, w17, [x1]
4124 \\ cbnz w15, 0b
4125 \\1:
4126 : [ret] "={w0}" (-> u16),
4127 : [val] "{w0}" (val),
4128 [ptr] "{x1}" (ptr),
4129 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4130 : "w15", "w16", "w17", "memory"
4131 );
4132}
4133fn ldeor2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4134 @setRuntimeSafety(false);
4135 __init_aarch64_have_lse_atomics();
4136
4137 return asm volatile (
4138 \\ cbz w16, 8f
4139 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
4140 \\ cbz wzr, 1f
4141 \\8:
4142 \\ mov w16, w0
4143 \\0:
4144 \\ ldxrh w0, [x1]
4145 \\ eor w17, w0, w16
4146 \\ stlxrh w15, w17, [x1]
4147 \\ cbnz w15, 0b
4148 \\1:
4149 : [ret] "={w0}" (-> u16),
4150 : [val] "{w0}" (val),
4151 [ptr] "{x1}" (ptr),
4152 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4153 : "w15", "w16", "w17", "memory"
4154 );
4155}
4156fn ldeor2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4157 @setRuntimeSafety(false);
4158 __init_aarch64_have_lse_atomics();
4159
4160 return asm volatile (
4161 \\ cbz w16, 8f
4162 \\ ldeorlh w0, w0, [x1]
4163 \\ cbz wzr, 1f
4164 \\8:
4165 \\ mov w16, w0
4166 \\0:
4167 \\ ldxrh w0, [x1]
4168 \\ eor w17, w0, w16
4169 \\ stlxrh w15, w17, [x1]
4170 \\ cbnz w15, 0b
4171 \\1:
4172 : [ret] "={w0}" (-> u16),
4173 : [val] "{w0}" (val),
4174 [ptr] "{x1}" (ptr),
4175 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4176 : "w15", "w16", "w17", "memory"
4177 );
4178}
4179fn ldeor2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4180 @setRuntimeSafety(false);
4181 __init_aarch64_have_lse_atomics();
4182
4183 return asm volatile (
4184 \\ cbz w16, 8f
4185 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
4186 \\ cbz wzr, 1f
4187 \\8:
4188 \\ mov w16, w0
4189 \\0:
4190 \\ ldxrh w0, [x1]
4191 \\ eor w17, w0, w16
4192 \\ stlxrh w15, w17, [x1]
4193 \\ cbnz w15, 0b
4194 \\1:
4195 : [ret] "={w0}" (-> u16),
4196 : [val] "{w0}" (val),
4197 [ptr] "{x1}" (ptr),
4198 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4199 : "w15", "w16", "w17", "memory"
4200 );
4201}
4202const __aarch64_ldeor2_rel = if (is_darwin)
4203 if (has_lse)
4204 ldeor2RelDarwinLse
4205 else
4206 ldeor2RelDarwinNolse
4207else if (has_lse)
4208 ldeor2RelNondarwinLse
4209else
4210 ldeor2RelNondarwinNolse;
4211fn ldset2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4212 @setRuntimeSafety(false);
4213 __init_aarch64_have_lse_atomics();
4214
4215 return asm volatile (
4216 \\ cbz w16, 8f
4217 \\ ldsetlh w0, w0, [x1]
4218 \\ cbz wzr, 1f
4219 \\8:
4220 \\ mov w16, w0
4221 \\0:
4222 \\ ldxrh w0, [x1]
4223 \\ orr w17, w0, w16
4224 \\ stlxrh w15, w17, [x1]
4225 \\ cbnz w15, 0b
4226 \\1:
4227 : [ret] "={w0}" (-> u16),
4228 : [val] "{w0}" (val),
4229 [ptr] "{x1}" (ptr),
4230 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4231 : "w15", "w16", "w17", "memory"
4232 );
4233}
4234fn ldset2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4235 @setRuntimeSafety(false);
4236 __init_aarch64_have_lse_atomics();
4237
4238 return asm volatile (
4239 \\ cbz w16, 8f
4240 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
4241 \\ cbz wzr, 1f
4242 \\8:
4243 \\ mov w16, w0
4244 \\0:
4245 \\ ldxrh w0, [x1]
4246 \\ orr w17, w0, w16
4247 \\ stlxrh w15, w17, [x1]
4248 \\ cbnz w15, 0b
4249 \\1:
4250 : [ret] "={w0}" (-> u16),
4251 : [val] "{w0}" (val),
4252 [ptr] "{x1}" (ptr),
4253 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4254 : "w15", "w16", "w17", "memory"
4255 );
4256}
4257fn ldset2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4258 @setRuntimeSafety(false);
4259 __init_aarch64_have_lse_atomics();
4260
4261 return asm volatile (
4262 \\ cbz w16, 8f
4263 \\ ldsetlh w0, w0, [x1]
4264 \\ cbz wzr, 1f
4265 \\8:
4266 \\ mov w16, w0
4267 \\0:
4268 \\ ldxrh w0, [x1]
4269 \\ orr w17, w0, w16
4270 \\ stlxrh w15, w17, [x1]
4271 \\ cbnz w15, 0b
4272 \\1:
4273 : [ret] "={w0}" (-> u16),
4274 : [val] "{w0}" (val),
4275 [ptr] "{x1}" (ptr),
4276 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4277 : "w15", "w16", "w17", "memory"
4278 );
4279}
4280fn ldset2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4281 @setRuntimeSafety(false);
4282 __init_aarch64_have_lse_atomics();
4283
4284 return asm volatile (
4285 \\ cbz w16, 8f
4286 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
4287 \\ cbz wzr, 1f
4288 \\8:
4289 \\ mov w16, w0
4290 \\0:
4291 \\ ldxrh w0, [x1]
4292 \\ orr w17, w0, w16
4293 \\ stlxrh w15, w17, [x1]
4294 \\ cbnz w15, 0b
4295 \\1:
4296 : [ret] "={w0}" (-> u16),
4297 : [val] "{w0}" (val),
4298 [ptr] "{x1}" (ptr),
4299 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4300 : "w15", "w16", "w17", "memory"
4301 );
4302}
4303const __aarch64_ldset2_rel = if (is_darwin)
4304 if (has_lse)
4305 ldset2RelDarwinLse
4306 else
4307 ldset2RelDarwinNolse
4308else if (has_lse)
4309 ldset2RelNondarwinLse
4310else
4311 ldset2RelNondarwinNolse;
4312fn cas2AcqRelDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4313 @setRuntimeSafety(false);
4314 __init_aarch64_have_lse_atomics();
4315
4316 return asm volatile (
4317 \\ cbz w16, 8f
4318 \\ casalh w0, w1, [x2]
4319 \\
4320 \\ cbz wzr, 1f
4321 \\8:
4322 \\ uxth w16, w0
4323 \\0:
4324 \\ ldaxrh w0, [x2]
4325 \\ cmp w0, w16
4326 \\ bne 1f
4327 \\ stlxrh w17, w1, [x2]
4328 \\ cbnz w17, 0b
4329 \\1:
4330 : [ret] "={w0}" (-> u16),
4331 : [expected] "{w0}" (expected),
4332 [desired] "{w1}" (desired),
4333 [ptr] "{x2}" (ptr),
4334 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4335 : "w15", "w16", "w17", "memory"
4336 );
4337}
4338fn cas2AcqRelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4339 @setRuntimeSafety(false);
4340 __init_aarch64_have_lse_atomics();
4341
4342 return asm volatile (
4343 \\ cbz w16, 8f
4344 \\ .inst 0x08a07c41 + 0x40000000 + 0x408000
4345 \\
4346 \\ cbz wzr, 1f
4347 \\8:
4348 \\ uxth w16, w0
4349 \\0:
4350 \\ ldaxrh w0, [x2]
4351 \\ cmp w0, w16
4352 \\ bne 1f
4353 \\ stlxrh w17, w1, [x2]
4354 \\ cbnz w17, 0b
4355 \\1:
4356 : [ret] "={w0}" (-> u16),
4357 : [expected] "{w0}" (expected),
4358 [desired] "{w1}" (desired),
4359 [ptr] "{x2}" (ptr),
4360 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4361 : "w15", "w16", "w17", "memory"
4362 );
4363}
4364fn cas2AcqRelNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4365 @setRuntimeSafety(false);
4366 __init_aarch64_have_lse_atomics();
4367
4368 return asm volatile (
4369 \\ cbz w16, 8f
4370 \\ casalh w0, w1, [x2]
4371 \\
4372 \\ cbz wzr, 1f
4373 \\8:
4374 \\ uxth w16, w0
4375 \\0:
4376 \\ ldaxrh w0, [x2]
4377 \\ cmp w0, w16
4378 \\ bne 1f
4379 \\ stlxrh w17, w1, [x2]
4380 \\ cbnz w17, 0b
4381 \\1:
4382 : [ret] "={w0}" (-> u16),
4383 : [expected] "{w0}" (expected),
4384 [desired] "{w1}" (desired),
4385 [ptr] "{x2}" (ptr),
4386 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4387 : "w15", "w16", "w17", "memory"
4388 );
4389}
4390fn cas2AcqRelNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4391 @setRuntimeSafety(false);
4392 __init_aarch64_have_lse_atomics();
4393
4394 return asm volatile (
4395 \\ cbz w16, 8f
4396 \\ .inst 0x08a07c41 + 0x40000000 + 0x408000
4397 \\
4398 \\ cbz wzr, 1f
4399 \\8:
4400 \\ uxth w16, w0
4401 \\0:
4402 \\ ldaxrh w0, [x2]
4403 \\ cmp w0, w16
4404 \\ bne 1f
4405 \\ stlxrh w17, w1, [x2]
4406 \\ cbnz w17, 0b
4407 \\1:
4408 : [ret] "={w0}" (-> u16),
4409 : [expected] "{w0}" (expected),
4410 [desired] "{w1}" (desired),
4411 [ptr] "{x2}" (ptr),
4412 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4413 : "w15", "w16", "w17", "memory"
4414 );
4415}
4416const __aarch64_cas2_acq_rel = if (is_darwin)
4417 if (has_lse)
4418 cas2AcqRelDarwinLse
4419 else
4420 cas2AcqRelDarwinNolse
4421else if (has_lse)
4422 cas2AcqRelNondarwinLse
4423else
4424 cas2AcqRelNondarwinNolse;
4425fn swp2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4426 @setRuntimeSafety(false);
4427 __init_aarch64_have_lse_atomics();
4428
4429 return asm volatile (
4430 \\ cbz w16, 8f
4431 \\ swpalh w0, w0, [x1]
4432 \\ cbz wzr, 1f
4433 \\8:
4434 \\ mov w16, w0
4435 \\0:
4436 \\ ldaxrh w0, [x1]
4437 \\ stlxrh w17, w16, [x1]
4438 \\ cbnz w17, 0b
4439 \\1:
4440 : [ret] "={w0}" (-> u16),
4441 : [val] "{w0}" (val),
4442 [ptr] "{x1}" (ptr),
4443 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4444 : "w15", "w16", "w17", "memory"
4445 );
4446}
4447fn swp2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4448 @setRuntimeSafety(false);
4449 __init_aarch64_have_lse_atomics();
4450
4451 return asm volatile (
4452 \\ cbz w16, 8f
4453 \\ .inst 0x38208020 + 0x40000000 + 0xc00000
4454 \\ cbz wzr, 1f
4455 \\8:
4456 \\ mov w16, w0
4457 \\0:
4458 \\ ldaxrh w0, [x1]
4459 \\ stlxrh w17, w16, [x1]
4460 \\ cbnz w17, 0b
4461 \\1:
4462 : [ret] "={w0}" (-> u16),
4463 : [val] "{w0}" (val),
4464 [ptr] "{x1}" (ptr),
4465 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4466 : "w15", "w16", "w17", "memory"
4467 );
4468}
4469fn swp2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4470 @setRuntimeSafety(false);
4471 __init_aarch64_have_lse_atomics();
4472
4473 return asm volatile (
4474 \\ cbz w16, 8f
4475 \\ swpalh w0, w0, [x1]
4476 \\ cbz wzr, 1f
4477 \\8:
4478 \\ mov w16, w0
4479 \\0:
4480 \\ ldaxrh w0, [x1]
4481 \\ stlxrh w17, w16, [x1]
4482 \\ cbnz w17, 0b
4483 \\1:
4484 : [ret] "={w0}" (-> u16),
4485 : [val] "{w0}" (val),
4486 [ptr] "{x1}" (ptr),
4487 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4488 : "w15", "w16", "w17", "memory"
4489 );
4490}
4491fn swp2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4492 @setRuntimeSafety(false);
4493 __init_aarch64_have_lse_atomics();
4494
4495 return asm volatile (
4496 \\ cbz w16, 8f
4497 \\ .inst 0x38208020 + 0x40000000 + 0xc00000
4498 \\ cbz wzr, 1f
4499 \\8:
4500 \\ mov w16, w0
4501 \\0:
4502 \\ ldaxrh w0, [x1]
4503 \\ stlxrh w17, w16, [x1]
4504 \\ cbnz w17, 0b
4505 \\1:
4506 : [ret] "={w0}" (-> u16),
4507 : [val] "{w0}" (val),
4508 [ptr] "{x1}" (ptr),
4509 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4510 : "w15", "w16", "w17", "memory"
4511 );
4512}
4513const __aarch64_swp2_acq_rel = if (is_darwin)
4514 if (has_lse)
4515 swp2AcqRelDarwinLse
4516 else
4517 swp2AcqRelDarwinNolse
4518else if (has_lse)
4519 swp2AcqRelNondarwinLse
4520else
4521 swp2AcqRelNondarwinNolse;
4522fn ldadd2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4523 @setRuntimeSafety(false);
4524 __init_aarch64_have_lse_atomics();
4525
4526 return asm volatile (
4527 \\ cbz w16, 8f
4528 \\ ldaddalh w0, w0, [x1]
4529 \\ cbz wzr, 1f
4530 \\8:
4531 \\ mov w16, w0
4532 \\0:
4533 \\ ldaxrh w0, [x1]
4534 \\ add w17, w0, w16
4535 \\ stlxrh w15, w17, [x1]
4536 \\ cbnz w15, 0b
4537 \\1:
4538 : [ret] "={w0}" (-> u16),
4539 : [val] "{w0}" (val),
4540 [ptr] "{x1}" (ptr),
4541 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4542 : "w15", "w16", "w17", "memory"
4543 );
4544}
4545fn ldadd2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4546 @setRuntimeSafety(false);
4547 __init_aarch64_have_lse_atomics();
4548
4549 return asm volatile (
4550 \\ cbz w16, 8f
4551 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
4552 \\ cbz wzr, 1f
4553 \\8:
4554 \\ mov w16, w0
4555 \\0:
4556 \\ ldaxrh w0, [x1]
4557 \\ add w17, w0, w16
4558 \\ stlxrh w15, w17, [x1]
4559 \\ cbnz w15, 0b
4560 \\1:
4561 : [ret] "={w0}" (-> u16),
4562 : [val] "{w0}" (val),
4563 [ptr] "{x1}" (ptr),
4564 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4565 : "w15", "w16", "w17", "memory"
4566 );
4567}
4568fn ldadd2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4569 @setRuntimeSafety(false);
4570 __init_aarch64_have_lse_atomics();
4571
4572 return asm volatile (
4573 \\ cbz w16, 8f
4574 \\ ldaddalh w0, w0, [x1]
4575 \\ cbz wzr, 1f
4576 \\8:
4577 \\ mov w16, w0
4578 \\0:
4579 \\ ldaxrh w0, [x1]
4580 \\ add w17, w0, w16
4581 \\ stlxrh w15, w17, [x1]
4582 \\ cbnz w15, 0b
4583 \\1:
4584 : [ret] "={w0}" (-> u16),
4585 : [val] "{w0}" (val),
4586 [ptr] "{x1}" (ptr),
4587 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4588 : "w15", "w16", "w17", "memory"
4589 );
4590}
4591fn ldadd2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4592 @setRuntimeSafety(false);
4593 __init_aarch64_have_lse_atomics();
4594
4595 return asm volatile (
4596 \\ cbz w16, 8f
4597 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
4598 \\ cbz wzr, 1f
4599 \\8:
4600 \\ mov w16, w0
4601 \\0:
4602 \\ ldaxrh w0, [x1]
4603 \\ add w17, w0, w16
4604 \\ stlxrh w15, w17, [x1]
4605 \\ cbnz w15, 0b
4606 \\1:
4607 : [ret] "={w0}" (-> u16),
4608 : [val] "{w0}" (val),
4609 [ptr] "{x1}" (ptr),
4610 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4611 : "w15", "w16", "w17", "memory"
4612 );
4613}
4614const __aarch64_ldadd2_acq_rel = if (is_darwin)
4615 if (has_lse)
4616 ldadd2AcqRelDarwinLse
4617 else
4618 ldadd2AcqRelDarwinNolse
4619else if (has_lse)
4620 ldadd2AcqRelNondarwinLse
4621else
4622 ldadd2AcqRelNondarwinNolse;
4623fn ldclr2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4624 @setRuntimeSafety(false);
4625 __init_aarch64_have_lse_atomics();
4626
4627 return asm volatile (
4628 \\ cbz w16, 8f
4629 \\ ldclralh w0, w0, [x1]
4630 \\ cbz wzr, 1f
4631 \\8:
4632 \\ mov w16, w0
4633 \\0:
4634 \\ ldaxrh w0, [x1]
4635 \\ bic w17, w0, w16
4636 \\ stlxrh w15, w17, [x1]
4637 \\ cbnz w15, 0b
4638 \\1:
4639 : [ret] "={w0}" (-> u16),
4640 : [val] "{w0}" (val),
4641 [ptr] "{x1}" (ptr),
4642 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4643 : "w15", "w16", "w17", "memory"
4644 );
4645}
4646fn ldclr2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4647 @setRuntimeSafety(false);
4648 __init_aarch64_have_lse_atomics();
4649
4650 return asm volatile (
4651 \\ cbz w16, 8f
4652 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
4653 \\ cbz wzr, 1f
4654 \\8:
4655 \\ mov w16, w0
4656 \\0:
4657 \\ ldaxrh w0, [x1]
4658 \\ bic w17, w0, w16
4659 \\ stlxrh w15, w17, [x1]
4660 \\ cbnz w15, 0b
4661 \\1:
4662 : [ret] "={w0}" (-> u16),
4663 : [val] "{w0}" (val),
4664 [ptr] "{x1}" (ptr),
4665 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4666 : "w15", "w16", "w17", "memory"
4667 );
4668}
4669fn ldclr2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4670 @setRuntimeSafety(false);
4671 __init_aarch64_have_lse_atomics();
4672
4673 return asm volatile (
4674 \\ cbz w16, 8f
4675 \\ ldclralh w0, w0, [x1]
4676 \\ cbz wzr, 1f
4677 \\8:
4678 \\ mov w16, w0
4679 \\0:
4680 \\ ldaxrh w0, [x1]
4681 \\ bic w17, w0, w16
4682 \\ stlxrh w15, w17, [x1]
4683 \\ cbnz w15, 0b
4684 \\1:
4685 : [ret] "={w0}" (-> u16),
4686 : [val] "{w0}" (val),
4687 [ptr] "{x1}" (ptr),
4688 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4689 : "w15", "w16", "w17", "memory"
4690 );
4691}
4692fn ldclr2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4693 @setRuntimeSafety(false);
4694 __init_aarch64_have_lse_atomics();
4695
4696 return asm volatile (
4697 \\ cbz w16, 8f
4698 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
4699 \\ cbz wzr, 1f
4700 \\8:
4701 \\ mov w16, w0
4702 \\0:
4703 \\ ldaxrh w0, [x1]
4704 \\ bic w17, w0, w16
4705 \\ stlxrh w15, w17, [x1]
4706 \\ cbnz w15, 0b
4707 \\1:
4708 : [ret] "={w0}" (-> u16),
4709 : [val] "{w0}" (val),
4710 [ptr] "{x1}" (ptr),
4711 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4712 : "w15", "w16", "w17", "memory"
4713 );
4714}
4715const __aarch64_ldclr2_acq_rel = if (is_darwin)
4716 if (has_lse)
4717 ldclr2AcqRelDarwinLse
4718 else
4719 ldclr2AcqRelDarwinNolse
4720else if (has_lse)
4721 ldclr2AcqRelNondarwinLse
4722else
4723 ldclr2AcqRelNondarwinNolse;
4724fn ldeor2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4725 @setRuntimeSafety(false);
4726 __init_aarch64_have_lse_atomics();
4727
4728 return asm volatile (
4729 \\ cbz w16, 8f
4730 \\ ldeoralh w0, w0, [x1]
4731 \\ cbz wzr, 1f
4732 \\8:
4733 \\ mov w16, w0
4734 \\0:
4735 \\ ldaxrh w0, [x1]
4736 \\ eor w17, w0, w16
4737 \\ stlxrh w15, w17, [x1]
4738 \\ cbnz w15, 0b
4739 \\1:
4740 : [ret] "={w0}" (-> u16),
4741 : [val] "{w0}" (val),
4742 [ptr] "{x1}" (ptr),
4743 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4744 : "w15", "w16", "w17", "memory"
4745 );
4746}
4747fn ldeor2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4748 @setRuntimeSafety(false);
4749 __init_aarch64_have_lse_atomics();
4750
4751 return asm volatile (
4752 \\ cbz w16, 8f
4753 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
4754 \\ cbz wzr, 1f
4755 \\8:
4756 \\ mov w16, w0
4757 \\0:
4758 \\ ldaxrh w0, [x1]
4759 \\ eor w17, w0, w16
4760 \\ stlxrh w15, w17, [x1]
4761 \\ cbnz w15, 0b
4762 \\1:
4763 : [ret] "={w0}" (-> u16),
4764 : [val] "{w0}" (val),
4765 [ptr] "{x1}" (ptr),
4766 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4767 : "w15", "w16", "w17", "memory"
4768 );
4769}
4770fn ldeor2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4771 @setRuntimeSafety(false);
4772 __init_aarch64_have_lse_atomics();
4773
4774 return asm volatile (
4775 \\ cbz w16, 8f
4776 \\ ldeoralh w0, w0, [x1]
4777 \\ cbz wzr, 1f
4778 \\8:
4779 \\ mov w16, w0
4780 \\0:
4781 \\ ldaxrh w0, [x1]
4782 \\ eor w17, w0, w16
4783 \\ stlxrh w15, w17, [x1]
4784 \\ cbnz w15, 0b
4785 \\1:
4786 : [ret] "={w0}" (-> u16),
4787 : [val] "{w0}" (val),
4788 [ptr] "{x1}" (ptr),
4789 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4790 : "w15", "w16", "w17", "memory"
4791 );
4792}
4793fn ldeor2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4794 @setRuntimeSafety(false);
4795 __init_aarch64_have_lse_atomics();
4796
4797 return asm volatile (
4798 \\ cbz w16, 8f
4799 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
4800 \\ cbz wzr, 1f
4801 \\8:
4802 \\ mov w16, w0
4803 \\0:
4804 \\ ldaxrh w0, [x1]
4805 \\ eor w17, w0, w16
4806 \\ stlxrh w15, w17, [x1]
4807 \\ cbnz w15, 0b
4808 \\1:
4809 : [ret] "={w0}" (-> u16),
4810 : [val] "{w0}" (val),
4811 [ptr] "{x1}" (ptr),
4812 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4813 : "w15", "w16", "w17", "memory"
4814 );
4815}
4816const __aarch64_ldeor2_acq_rel = if (is_darwin)
4817 if (has_lse)
4818 ldeor2AcqRelDarwinLse
4819 else
4820 ldeor2AcqRelDarwinNolse
4821else if (has_lse)
4822 ldeor2AcqRelNondarwinLse
4823else
4824 ldeor2AcqRelNondarwinNolse;
4825fn ldset2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4826 @setRuntimeSafety(false);
4827 __init_aarch64_have_lse_atomics();
4828
4829 return asm volatile (
4830 \\ cbz w16, 8f
4831 \\ ldsetalh w0, w0, [x1]
4832 \\ cbz wzr, 1f
4833 \\8:
4834 \\ mov w16, w0
4835 \\0:
4836 \\ ldaxrh w0, [x1]
4837 \\ orr w17, w0, w16
4838 \\ stlxrh w15, w17, [x1]
4839 \\ cbnz w15, 0b
4840 \\1:
4841 : [ret] "={w0}" (-> u16),
4842 : [val] "{w0}" (val),
4843 [ptr] "{x1}" (ptr),
4844 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4845 : "w15", "w16", "w17", "memory"
4846 );
4847}
4848fn ldset2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4849 @setRuntimeSafety(false);
4850 __init_aarch64_have_lse_atomics();
4851
4852 return asm volatile (
4853 \\ cbz w16, 8f
4854 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
4855 \\ cbz wzr, 1f
4856 \\8:
4857 \\ mov w16, w0
4858 \\0:
4859 \\ ldaxrh w0, [x1]
4860 \\ orr w17, w0, w16
4861 \\ stlxrh w15, w17, [x1]
4862 \\ cbnz w15, 0b
4863 \\1:
4864 : [ret] "={w0}" (-> u16),
4865 : [val] "{w0}" (val),
4866 [ptr] "{x1}" (ptr),
4867 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4868 : "w15", "w16", "w17", "memory"
4869 );
4870}
4871fn ldset2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4872 @setRuntimeSafety(false);
4873 __init_aarch64_have_lse_atomics();
4874
4875 return asm volatile (
4876 \\ cbz w16, 8f
4877 \\ ldsetalh w0, w0, [x1]
4878 \\ cbz wzr, 1f
4879 \\8:
4880 \\ mov w16, w0
4881 \\0:
4882 \\ ldaxrh w0, [x1]
4883 \\ orr w17, w0, w16
4884 \\ stlxrh w15, w17, [x1]
4885 \\ cbnz w15, 0b
4886 \\1:
4887 : [ret] "={w0}" (-> u16),
4888 : [val] "{w0}" (val),
4889 [ptr] "{x1}" (ptr),
4890 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4891 : "w15", "w16", "w17", "memory"
4892 );
4893}
4894fn ldset2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4895 @setRuntimeSafety(false);
4896 __init_aarch64_have_lse_atomics();
4897
4898 return asm volatile (
4899 \\ cbz w16, 8f
4900 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
4901 \\ cbz wzr, 1f
4902 \\8:
4903 \\ mov w16, w0
4904 \\0:
4905 \\ ldaxrh w0, [x1]
4906 \\ orr w17, w0, w16
4907 \\ stlxrh w15, w17, [x1]
4908 \\ cbnz w15, 0b
4909 \\1:
4910 : [ret] "={w0}" (-> u16),
4911 : [val] "{w0}" (val),
4912 [ptr] "{x1}" (ptr),
4913 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4914 : "w15", "w16", "w17", "memory"
4915 );
4916}
4917const __aarch64_ldset2_acq_rel = if (is_darwin)
4918 if (has_lse)
4919 ldset2AcqRelDarwinLse
4920 else
4921 ldset2AcqRelDarwinNolse
4922else if (has_lse)
4923 ldset2AcqRelNondarwinLse
4924else
4925 ldset2AcqRelNondarwinNolse;
4926fn cas4RelaxDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
4927 @setRuntimeSafety(false);
4928 __init_aarch64_have_lse_atomics();
4929
4930 return asm volatile (
4931 \\ cbz w16, 8f
4932 \\ cas w0, w1, [x2]
4933 \\
4934 \\ cbz wzr, 1f
4935 \\8:
4936 \\ mov w16, w0
4937 \\0:
4938 \\ ldxr w0, [x2]
4939 \\ cmp w0, w16
4940 \\ bne 1f
4941 \\ stxr w17, w1, [x2]
4942 \\ cbnz w17, 0b
4943 \\1:
4944 : [ret] "={w0}" (-> u32),
4945 : [expected] "{w0}" (expected),
4946 [desired] "{w1}" (desired),
4947 [ptr] "{x2}" (ptr),
4948 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4949 : "w15", "w16", "w17", "memory"
4950 );
4951}
4952fn cas4RelaxDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
4953 @setRuntimeSafety(false);
4954 __init_aarch64_have_lse_atomics();
4955
4956 return asm volatile (
4957 \\ cbz w16, 8f
4958 \\ .inst 0x08a07c41 + 0x80000000 + 0x000000
4959 \\
4960 \\ cbz wzr, 1f
4961 \\8:
4962 \\ mov w16, w0
4963 \\0:
4964 \\ ldxr w0, [x2]
4965 \\ cmp w0, w16
4966 \\ bne 1f
4967 \\ stxr w17, w1, [x2]
4968 \\ cbnz w17, 0b
4969 \\1:
4970 : [ret] "={w0}" (-> u32),
4971 : [expected] "{w0}" (expected),
4972 [desired] "{w1}" (desired),
4973 [ptr] "{x2}" (ptr),
4974 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4975 : "w15", "w16", "w17", "memory"
4976 );
4977}
4978fn cas4RelaxNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
4979 @setRuntimeSafety(false);
4980 __init_aarch64_have_lse_atomics();
4981
4982 return asm volatile (
4983 \\ cbz w16, 8f
4984 \\ cas w0, w1, [x2]
4985 \\
4986 \\ cbz wzr, 1f
4987 \\8:
4988 \\ mov w16, w0
4989 \\0:
4990 \\ ldxr w0, [x2]
4991 \\ cmp w0, w16
4992 \\ bne 1f
4993 \\ stxr w17, w1, [x2]
4994 \\ cbnz w17, 0b
4995 \\1:
4996 : [ret] "={w0}" (-> u32),
4997 : [expected] "{w0}" (expected),
4998 [desired] "{w1}" (desired),
4999 [ptr] "{x2}" (ptr),
5000 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5001 : "w15", "w16", "w17", "memory"
5002 );
5003}
5004fn cas4RelaxNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5005 @setRuntimeSafety(false);
5006 __init_aarch64_have_lse_atomics();
5007
5008 return asm volatile (
5009 \\ cbz w16, 8f
5010 \\ .inst 0x08a07c41 + 0x80000000 + 0x000000
5011 \\
5012 \\ cbz wzr, 1f
5013 \\8:
5014 \\ mov w16, w0
5015 \\0:
5016 \\ ldxr w0, [x2]
5017 \\ cmp w0, w16
5018 \\ bne 1f
5019 \\ stxr w17, w1, [x2]
5020 \\ cbnz w17, 0b
5021 \\1:
5022 : [ret] "={w0}" (-> u32),
5023 : [expected] "{w0}" (expected),
5024 [desired] "{w1}" (desired),
5025 [ptr] "{x2}" (ptr),
5026 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5027 : "w15", "w16", "w17", "memory"
5028 );
5029}
5030const __aarch64_cas4_relax = if (is_darwin)
5031 if (has_lse)
5032 cas4RelaxDarwinLse
5033 else
5034 cas4RelaxDarwinNolse
5035else if (has_lse)
5036 cas4RelaxNondarwinLse
5037else
5038 cas4RelaxNondarwinNolse;
5039fn swp4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5040 @setRuntimeSafety(false);
5041 __init_aarch64_have_lse_atomics();
5042
5043 return asm volatile (
5044 \\ cbz w16, 8f
5045 \\ swp w0, w0, [x1]
5046 \\ cbz wzr, 1f
5047 \\8:
5048 \\ mov w16, w0
5049 \\0:
5050 \\ ldxr w0, [x1]
5051 \\ stxr w17, w16, [x1]
5052 \\ cbnz w17, 0b
5053 \\1:
5054 : [ret] "={w0}" (-> u32),
5055 : [val] "{w0}" (val),
5056 [ptr] "{x1}" (ptr),
5057 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5058 : "w15", "w16", "w17", "memory"
5059 );
5060}
5061fn swp4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5062 @setRuntimeSafety(false);
5063 __init_aarch64_have_lse_atomics();
5064
5065 return asm volatile (
5066 \\ cbz w16, 8f
5067 \\ .inst 0x38208020 + 0x80000000 + 0x000000
5068 \\ cbz wzr, 1f
5069 \\8:
5070 \\ mov w16, w0
5071 \\0:
5072 \\ ldxr w0, [x1]
5073 \\ stxr w17, w16, [x1]
5074 \\ cbnz w17, 0b
5075 \\1:
5076 : [ret] "={w0}" (-> u32),
5077 : [val] "{w0}" (val),
5078 [ptr] "{x1}" (ptr),
5079 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5080 : "w15", "w16", "w17", "memory"
5081 );
5082}
5083fn swp4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5084 @setRuntimeSafety(false);
5085 __init_aarch64_have_lse_atomics();
5086
5087 return asm volatile (
5088 \\ cbz w16, 8f
5089 \\ swp w0, w0, [x1]
5090 \\ cbz wzr, 1f
5091 \\8:
5092 \\ mov w16, w0
5093 \\0:
5094 \\ ldxr w0, [x1]
5095 \\ stxr w17, w16, [x1]
5096 \\ cbnz w17, 0b
5097 \\1:
5098 : [ret] "={w0}" (-> u32),
5099 : [val] "{w0}" (val),
5100 [ptr] "{x1}" (ptr),
5101 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5102 : "w15", "w16", "w17", "memory"
5103 );
5104}
5105fn swp4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5106 @setRuntimeSafety(false);
5107 __init_aarch64_have_lse_atomics();
5108
5109 return asm volatile (
5110 \\ cbz w16, 8f
5111 \\ .inst 0x38208020 + 0x80000000 + 0x000000
5112 \\ cbz wzr, 1f
5113 \\8:
5114 \\ mov w16, w0
5115 \\0:
5116 \\ ldxr w0, [x1]
5117 \\ stxr w17, w16, [x1]
5118 \\ cbnz w17, 0b
5119 \\1:
5120 : [ret] "={w0}" (-> u32),
5121 : [val] "{w0}" (val),
5122 [ptr] "{x1}" (ptr),
5123 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5124 : "w15", "w16", "w17", "memory"
5125 );
5126}
5127const __aarch64_swp4_relax = if (is_darwin)
5128 if (has_lse)
5129 swp4RelaxDarwinLse
5130 else
5131 swp4RelaxDarwinNolse
5132else if (has_lse)
5133 swp4RelaxNondarwinLse
5134else
5135 swp4RelaxNondarwinNolse;
5136fn ldadd4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5137 @setRuntimeSafety(false);
5138 __init_aarch64_have_lse_atomics();
5139
5140 return asm volatile (
5141 \\ cbz w16, 8f
5142 \\ ldadd w0, w0, [x1]
5143 \\ cbz wzr, 1f
5144 \\8:
5145 \\ mov w16, w0
5146 \\0:
5147 \\ ldxr w0, [x1]
5148 \\ add w17, w0, w16
5149 \\ stxr w15, w17, [x1]
5150 \\ cbnz w15, 0b
5151 \\1:
5152 : [ret] "={w0}" (-> u32),
5153 : [val] "{w0}" (val),
5154 [ptr] "{x1}" (ptr),
5155 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5156 : "w15", "w16", "w17", "memory"
5157 );
5158}
5159fn ldadd4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5160 @setRuntimeSafety(false);
5161 __init_aarch64_have_lse_atomics();
5162
5163 return asm volatile (
5164 \\ cbz w16, 8f
5165 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
5166 \\ cbz wzr, 1f
5167 \\8:
5168 \\ mov w16, w0
5169 \\0:
5170 \\ ldxr w0, [x1]
5171 \\ add w17, w0, w16
5172 \\ stxr w15, w17, [x1]
5173 \\ cbnz w15, 0b
5174 \\1:
5175 : [ret] "={w0}" (-> u32),
5176 : [val] "{w0}" (val),
5177 [ptr] "{x1}" (ptr),
5178 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5179 : "w15", "w16", "w17", "memory"
5180 );
5181}
5182fn ldadd4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5183 @setRuntimeSafety(false);
5184 __init_aarch64_have_lse_atomics();
5185
5186 return asm volatile (
5187 \\ cbz w16, 8f
5188 \\ ldadd w0, w0, [x1]
5189 \\ cbz wzr, 1f
5190 \\8:
5191 \\ mov w16, w0
5192 \\0:
5193 \\ ldxr w0, [x1]
5194 \\ add w17, w0, w16
5195 \\ stxr w15, w17, [x1]
5196 \\ cbnz w15, 0b
5197 \\1:
5198 : [ret] "={w0}" (-> u32),
5199 : [val] "{w0}" (val),
5200 [ptr] "{x1}" (ptr),
5201 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5202 : "w15", "w16", "w17", "memory"
5203 );
5204}
5205fn ldadd4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5206 @setRuntimeSafety(false);
5207 __init_aarch64_have_lse_atomics();
5208
5209 return asm volatile (
5210 \\ cbz w16, 8f
5211 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
5212 \\ cbz wzr, 1f
5213 \\8:
5214 \\ mov w16, w0
5215 \\0:
5216 \\ ldxr w0, [x1]
5217 \\ add w17, w0, w16
5218 \\ stxr w15, w17, [x1]
5219 \\ cbnz w15, 0b
5220 \\1:
5221 : [ret] "={w0}" (-> u32),
5222 : [val] "{w0}" (val),
5223 [ptr] "{x1}" (ptr),
5224 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5225 : "w15", "w16", "w17", "memory"
5226 );
5227}
5228const __aarch64_ldadd4_relax = if (is_darwin)
5229 if (has_lse)
5230 ldadd4RelaxDarwinLse
5231 else
5232 ldadd4RelaxDarwinNolse
5233else if (has_lse)
5234 ldadd4RelaxNondarwinLse
5235else
5236 ldadd4RelaxNondarwinNolse;
5237fn ldclr4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5238 @setRuntimeSafety(false);
5239 __init_aarch64_have_lse_atomics();
5240
5241 return asm volatile (
5242 \\ cbz w16, 8f
5243 \\ ldclr w0, w0, [x1]
5244 \\ cbz wzr, 1f
5245 \\8:
5246 \\ mov w16, w0
5247 \\0:
5248 \\ ldxr w0, [x1]
5249 \\ bic w17, w0, w16
5250 \\ stxr w15, w17, [x1]
5251 \\ cbnz w15, 0b
5252 \\1:
5253 : [ret] "={w0}" (-> u32),
5254 : [val] "{w0}" (val),
5255 [ptr] "{x1}" (ptr),
5256 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5257 : "w15", "w16", "w17", "memory"
5258 );
5259}
5260fn ldclr4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5261 @setRuntimeSafety(false);
5262 __init_aarch64_have_lse_atomics();
5263
5264 return asm volatile (
5265 \\ cbz w16, 8f
5266 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
5267 \\ cbz wzr, 1f
5268 \\8:
5269 \\ mov w16, w0
5270 \\0:
5271 \\ ldxr w0, [x1]
5272 \\ bic w17, w0, w16
5273 \\ stxr w15, w17, [x1]
5274 \\ cbnz w15, 0b
5275 \\1:
5276 : [ret] "={w0}" (-> u32),
5277 : [val] "{w0}" (val),
5278 [ptr] "{x1}" (ptr),
5279 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5280 : "w15", "w16", "w17", "memory"
5281 );
5282}
5283fn ldclr4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5284 @setRuntimeSafety(false);
5285 __init_aarch64_have_lse_atomics();
5286
5287 return asm volatile (
5288 \\ cbz w16, 8f
5289 \\ ldclr w0, w0, [x1]
5290 \\ cbz wzr, 1f
5291 \\8:
5292 \\ mov w16, w0
5293 \\0:
5294 \\ ldxr w0, [x1]
5295 \\ bic w17, w0, w16
5296 \\ stxr w15, w17, [x1]
5297 \\ cbnz w15, 0b
5298 \\1:
5299 : [ret] "={w0}" (-> u32),
5300 : [val] "{w0}" (val),
5301 [ptr] "{x1}" (ptr),
5302 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5303 : "w15", "w16", "w17", "memory"
5304 );
5305}
5306fn ldclr4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5307 @setRuntimeSafety(false);
5308 __init_aarch64_have_lse_atomics();
5309
5310 return asm volatile (
5311 \\ cbz w16, 8f
5312 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
5313 \\ cbz wzr, 1f
5314 \\8:
5315 \\ mov w16, w0
5316 \\0:
5317 \\ ldxr w0, [x1]
5318 \\ bic w17, w0, w16
5319 \\ stxr w15, w17, [x1]
5320 \\ cbnz w15, 0b
5321 \\1:
5322 : [ret] "={w0}" (-> u32),
5323 : [val] "{w0}" (val),
5324 [ptr] "{x1}" (ptr),
5325 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5326 : "w15", "w16", "w17", "memory"
5327 );
5328}
5329const __aarch64_ldclr4_relax = if (is_darwin)
5330 if (has_lse)
5331 ldclr4RelaxDarwinLse
5332 else
5333 ldclr4RelaxDarwinNolse
5334else if (has_lse)
5335 ldclr4RelaxNondarwinLse
5336else
5337 ldclr4RelaxNondarwinNolse;
5338fn ldeor4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5339 @setRuntimeSafety(false);
5340 __init_aarch64_have_lse_atomics();
5341
5342 return asm volatile (
5343 \\ cbz w16, 8f
5344 \\ ldeor w0, w0, [x1]
5345 \\ cbz wzr, 1f
5346 \\8:
5347 \\ mov w16, w0
5348 \\0:
5349 \\ ldxr w0, [x1]
5350 \\ eor w17, w0, w16
5351 \\ stxr w15, w17, [x1]
5352 \\ cbnz w15, 0b
5353 \\1:
5354 : [ret] "={w0}" (-> u32),
5355 : [val] "{w0}" (val),
5356 [ptr] "{x1}" (ptr),
5357 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5358 : "w15", "w16", "w17", "memory"
5359 );
5360}
5361fn ldeor4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5362 @setRuntimeSafety(false);
5363 __init_aarch64_have_lse_atomics();
5364
5365 return asm volatile (
5366 \\ cbz w16, 8f
5367 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
5368 \\ cbz wzr, 1f
5369 \\8:
5370 \\ mov w16, w0
5371 \\0:
5372 \\ ldxr w0, [x1]
5373 \\ eor w17, w0, w16
5374 \\ stxr w15, w17, [x1]
5375 \\ cbnz w15, 0b
5376 \\1:
5377 : [ret] "={w0}" (-> u32),
5378 : [val] "{w0}" (val),
5379 [ptr] "{x1}" (ptr),
5380 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5381 : "w15", "w16", "w17", "memory"
5382 );
5383}
5384fn ldeor4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5385 @setRuntimeSafety(false);
5386 __init_aarch64_have_lse_atomics();
5387
5388 return asm volatile (
5389 \\ cbz w16, 8f
5390 \\ ldeor w0, w0, [x1]
5391 \\ cbz wzr, 1f
5392 \\8:
5393 \\ mov w16, w0
5394 \\0:
5395 \\ ldxr w0, [x1]
5396 \\ eor w17, w0, w16
5397 \\ stxr w15, w17, [x1]
5398 \\ cbnz w15, 0b
5399 \\1:
5400 : [ret] "={w0}" (-> u32),
5401 : [val] "{w0}" (val),
5402 [ptr] "{x1}" (ptr),
5403 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5404 : "w15", "w16", "w17", "memory"
5405 );
5406}
5407fn ldeor4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5408 @setRuntimeSafety(false);
5409 __init_aarch64_have_lse_atomics();
5410
5411 return asm volatile (
5412 \\ cbz w16, 8f
5413 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
5414 \\ cbz wzr, 1f
5415 \\8:
5416 \\ mov w16, w0
5417 \\0:
5418 \\ ldxr w0, [x1]
5419 \\ eor w17, w0, w16
5420 \\ stxr w15, w17, [x1]
5421 \\ cbnz w15, 0b
5422 \\1:
5423 : [ret] "={w0}" (-> u32),
5424 : [val] "{w0}" (val),
5425 [ptr] "{x1}" (ptr),
5426 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5427 : "w15", "w16", "w17", "memory"
5428 );
5429}
5430const __aarch64_ldeor4_relax = if (is_darwin)
5431 if (has_lse)
5432 ldeor4RelaxDarwinLse
5433 else
5434 ldeor4RelaxDarwinNolse
5435else if (has_lse)
5436 ldeor4RelaxNondarwinLse
5437else
5438 ldeor4RelaxNondarwinNolse;
5439fn ldset4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5440 @setRuntimeSafety(false);
5441 __init_aarch64_have_lse_atomics();
5442
5443 return asm volatile (
5444 \\ cbz w16, 8f
5445 \\ ldset w0, w0, [x1]
5446 \\ cbz wzr, 1f
5447 \\8:
5448 \\ mov w16, w0
5449 \\0:
5450 \\ ldxr w0, [x1]
5451 \\ orr w17, w0, w16
5452 \\ stxr w15, w17, [x1]
5453 \\ cbnz w15, 0b
5454 \\1:
5455 : [ret] "={w0}" (-> u32),
5456 : [val] "{w0}" (val),
5457 [ptr] "{x1}" (ptr),
5458 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5459 : "w15", "w16", "w17", "memory"
5460 );
5461}
5462fn ldset4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5463 @setRuntimeSafety(false);
5464 __init_aarch64_have_lse_atomics();
5465
5466 return asm volatile (
5467 \\ cbz w16, 8f
5468 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
5469 \\ cbz wzr, 1f
5470 \\8:
5471 \\ mov w16, w0
5472 \\0:
5473 \\ ldxr w0, [x1]
5474 \\ orr w17, w0, w16
5475 \\ stxr w15, w17, [x1]
5476 \\ cbnz w15, 0b
5477 \\1:
5478 : [ret] "={w0}" (-> u32),
5479 : [val] "{w0}" (val),
5480 [ptr] "{x1}" (ptr),
5481 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5482 : "w15", "w16", "w17", "memory"
5483 );
5484}
5485fn ldset4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5486 @setRuntimeSafety(false);
5487 __init_aarch64_have_lse_atomics();
5488
5489 return asm volatile (
5490 \\ cbz w16, 8f
5491 \\ ldset w0, w0, [x1]
5492 \\ cbz wzr, 1f
5493 \\8:
5494 \\ mov w16, w0
5495 \\0:
5496 \\ ldxr w0, [x1]
5497 \\ orr w17, w0, w16
5498 \\ stxr w15, w17, [x1]
5499 \\ cbnz w15, 0b
5500 \\1:
5501 : [ret] "={w0}" (-> u32),
5502 : [val] "{w0}" (val),
5503 [ptr] "{x1}" (ptr),
5504 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5505 : "w15", "w16", "w17", "memory"
5506 );
5507}
5508fn ldset4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5509 @setRuntimeSafety(false);
5510 __init_aarch64_have_lse_atomics();
5511
5512 return asm volatile (
5513 \\ cbz w16, 8f
5514 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
5515 \\ cbz wzr, 1f
5516 \\8:
5517 \\ mov w16, w0
5518 \\0:
5519 \\ ldxr w0, [x1]
5520 \\ orr w17, w0, w16
5521 \\ stxr w15, w17, [x1]
5522 \\ cbnz w15, 0b
5523 \\1:
5524 : [ret] "={w0}" (-> u32),
5525 : [val] "{w0}" (val),
5526 [ptr] "{x1}" (ptr),
5527 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5528 : "w15", "w16", "w17", "memory"
5529 );
5530}
5531const __aarch64_ldset4_relax = if (is_darwin)
5532 if (has_lse)
5533 ldset4RelaxDarwinLse
5534 else
5535 ldset4RelaxDarwinNolse
5536else if (has_lse)
5537 ldset4RelaxNondarwinLse
5538else
5539 ldset4RelaxNondarwinNolse;
5540fn cas4AcqDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5541 @setRuntimeSafety(false);
5542 __init_aarch64_have_lse_atomics();
5543
5544 return asm volatile (
5545 \\ cbz w16, 8f
5546 \\ casa w0, w1, [x2]
5547 \\
5548 \\ cbz wzr, 1f
5549 \\8:
5550 \\ mov w16, w0
5551 \\0:
5552 \\ ldaxr w0, [x2]
5553 \\ cmp w0, w16
5554 \\ bne 1f
5555 \\ stxr w17, w1, [x2]
5556 \\ cbnz w17, 0b
5557 \\1:
5558 : [ret] "={w0}" (-> u32),
5559 : [expected] "{w0}" (expected),
5560 [desired] "{w1}" (desired),
5561 [ptr] "{x2}" (ptr),
5562 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5563 : "w15", "w16", "w17", "memory"
5564 );
5565}
5566fn cas4AcqDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5567 @setRuntimeSafety(false);
5568 __init_aarch64_have_lse_atomics();
5569
5570 return asm volatile (
5571 \\ cbz w16, 8f
5572 \\ .inst 0x08a07c41 + 0x80000000 + 0x400000
5573 \\
5574 \\ cbz wzr, 1f
5575 \\8:
5576 \\ mov w16, w0
5577 \\0:
5578 \\ ldaxr w0, [x2]
5579 \\ cmp w0, w16
5580 \\ bne 1f
5581 \\ stxr w17, w1, [x2]
5582 \\ cbnz w17, 0b
5583 \\1:
5584 : [ret] "={w0}" (-> u32),
5585 : [expected] "{w0}" (expected),
5586 [desired] "{w1}" (desired),
5587 [ptr] "{x2}" (ptr),
5588 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5589 : "w15", "w16", "w17", "memory"
5590 );
5591}
5592fn cas4AcqNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5593 @setRuntimeSafety(false);
5594 __init_aarch64_have_lse_atomics();
5595
5596 return asm volatile (
5597 \\ cbz w16, 8f
5598 \\ casa w0, w1, [x2]
5599 \\
5600 \\ cbz wzr, 1f
5601 \\8:
5602 \\ mov w16, w0
5603 \\0:
5604 \\ ldaxr w0, [x2]
5605 \\ cmp w0, w16
5606 \\ bne 1f
5607 \\ stxr w17, w1, [x2]
5608 \\ cbnz w17, 0b
5609 \\1:
5610 : [ret] "={w0}" (-> u32),
5611 : [expected] "{w0}" (expected),
5612 [desired] "{w1}" (desired),
5613 [ptr] "{x2}" (ptr),
5614 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5615 : "w15", "w16", "w17", "memory"
5616 );
5617}
5618fn cas4AcqNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5619 @setRuntimeSafety(false);
5620 __init_aarch64_have_lse_atomics();
5621
5622 return asm volatile (
5623 \\ cbz w16, 8f
5624 \\ .inst 0x08a07c41 + 0x80000000 + 0x400000
5625 \\
5626 \\ cbz wzr, 1f
5627 \\8:
5628 \\ mov w16, w0
5629 \\0:
5630 \\ ldaxr w0, [x2]
5631 \\ cmp w0, w16
5632 \\ bne 1f
5633 \\ stxr w17, w1, [x2]
5634 \\ cbnz w17, 0b
5635 \\1:
5636 : [ret] "={w0}" (-> u32),
5637 : [expected] "{w0}" (expected),
5638 [desired] "{w1}" (desired),
5639 [ptr] "{x2}" (ptr),
5640 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5641 : "w15", "w16", "w17", "memory"
5642 );
5643}
5644const __aarch64_cas4_acq = if (is_darwin)
5645 if (has_lse)
5646 cas4AcqDarwinLse
5647 else
5648 cas4AcqDarwinNolse
5649else if (has_lse)
5650 cas4AcqNondarwinLse
5651else
5652 cas4AcqNondarwinNolse;
5653fn swp4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5654 @setRuntimeSafety(false);
5655 __init_aarch64_have_lse_atomics();
5656
5657 return asm volatile (
5658 \\ cbz w16, 8f
5659 \\ swpa w0, w0, [x1]
5660 \\ cbz wzr, 1f
5661 \\8:
5662 \\ mov w16, w0
5663 \\0:
5664 \\ ldaxr w0, [x1]
5665 \\ stxr w17, w16, [x1]
5666 \\ cbnz w17, 0b
5667 \\1:
5668 : [ret] "={w0}" (-> u32),
5669 : [val] "{w0}" (val),
5670 [ptr] "{x1}" (ptr),
5671 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5672 : "w15", "w16", "w17", "memory"
5673 );
5674}
5675fn swp4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5676 @setRuntimeSafety(false);
5677 __init_aarch64_have_lse_atomics();
5678
5679 return asm volatile (
5680 \\ cbz w16, 8f
5681 \\ .inst 0x38208020 + 0x80000000 + 0x800000
5682 \\ cbz wzr, 1f
5683 \\8:
5684 \\ mov w16, w0
5685 \\0:
5686 \\ ldaxr w0, [x1]
5687 \\ stxr w17, w16, [x1]
5688 \\ cbnz w17, 0b
5689 \\1:
5690 : [ret] "={w0}" (-> u32),
5691 : [val] "{w0}" (val),
5692 [ptr] "{x1}" (ptr),
5693 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5694 : "w15", "w16", "w17", "memory"
5695 );
5696}
5697fn swp4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5698 @setRuntimeSafety(false);
5699 __init_aarch64_have_lse_atomics();
5700
5701 return asm volatile (
5702 \\ cbz w16, 8f
5703 \\ swpa w0, w0, [x1]
5704 \\ cbz wzr, 1f
5705 \\8:
5706 \\ mov w16, w0
5707 \\0:
5708 \\ ldaxr w0, [x1]
5709 \\ stxr w17, w16, [x1]
5710 \\ cbnz w17, 0b
5711 \\1:
5712 : [ret] "={w0}" (-> u32),
5713 : [val] "{w0}" (val),
5714 [ptr] "{x1}" (ptr),
5715 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5716 : "w15", "w16", "w17", "memory"
5717 );
5718}
5719fn swp4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5720 @setRuntimeSafety(false);
5721 __init_aarch64_have_lse_atomics();
5722
5723 return asm volatile (
5724 \\ cbz w16, 8f
5725 \\ .inst 0x38208020 + 0x80000000 + 0x800000
5726 \\ cbz wzr, 1f
5727 \\8:
5728 \\ mov w16, w0
5729 \\0:
5730 \\ ldaxr w0, [x1]
5731 \\ stxr w17, w16, [x1]
5732 \\ cbnz w17, 0b
5733 \\1:
5734 : [ret] "={w0}" (-> u32),
5735 : [val] "{w0}" (val),
5736 [ptr] "{x1}" (ptr),
5737 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5738 : "w15", "w16", "w17", "memory"
5739 );
5740}
5741const __aarch64_swp4_acq = if (is_darwin)
5742 if (has_lse)
5743 swp4AcqDarwinLse
5744 else
5745 swp4AcqDarwinNolse
5746else if (has_lse)
5747 swp4AcqNondarwinLse
5748else
5749 swp4AcqNondarwinNolse;
5750fn ldadd4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5751 @setRuntimeSafety(false);
5752 __init_aarch64_have_lse_atomics();
5753
5754 return asm volatile (
5755 \\ cbz w16, 8f
5756 \\ ldadda w0, w0, [x1]
5757 \\ cbz wzr, 1f
5758 \\8:
5759 \\ mov w16, w0
5760 \\0:
5761 \\ ldaxr w0, [x1]
5762 \\ add w17, w0, w16
5763 \\ stxr w15, w17, [x1]
5764 \\ cbnz w15, 0b
5765 \\1:
5766 : [ret] "={w0}" (-> u32),
5767 : [val] "{w0}" (val),
5768 [ptr] "{x1}" (ptr),
5769 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5770 : "w15", "w16", "w17", "memory"
5771 );
5772}
5773fn ldadd4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5774 @setRuntimeSafety(false);
5775 __init_aarch64_have_lse_atomics();
5776
5777 return asm volatile (
5778 \\ cbz w16, 8f
5779 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
5780 \\ cbz wzr, 1f
5781 \\8:
5782 \\ mov w16, w0
5783 \\0:
5784 \\ ldaxr w0, [x1]
5785 \\ add w17, w0, w16
5786 \\ stxr w15, w17, [x1]
5787 \\ cbnz w15, 0b
5788 \\1:
5789 : [ret] "={w0}" (-> u32),
5790 : [val] "{w0}" (val),
5791 [ptr] "{x1}" (ptr),
5792 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5793 : "w15", "w16", "w17", "memory"
5794 );
5795}
5796fn ldadd4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5797 @setRuntimeSafety(false);
5798 __init_aarch64_have_lse_atomics();
5799
5800 return asm volatile (
5801 \\ cbz w16, 8f
5802 \\ ldadda w0, w0, [x1]
5803 \\ cbz wzr, 1f
5804 \\8:
5805 \\ mov w16, w0
5806 \\0:
5807 \\ ldaxr w0, [x1]
5808 \\ add w17, w0, w16
5809 \\ stxr w15, w17, [x1]
5810 \\ cbnz w15, 0b
5811 \\1:
5812 : [ret] "={w0}" (-> u32),
5813 : [val] "{w0}" (val),
5814 [ptr] "{x1}" (ptr),
5815 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5816 : "w15", "w16", "w17", "memory"
5817 );
5818}
5819fn ldadd4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5820 @setRuntimeSafety(false);
5821 __init_aarch64_have_lse_atomics();
5822
5823 return asm volatile (
5824 \\ cbz w16, 8f
5825 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
5826 \\ cbz wzr, 1f
5827 \\8:
5828 \\ mov w16, w0
5829 \\0:
5830 \\ ldaxr w0, [x1]
5831 \\ add w17, w0, w16
5832 \\ stxr w15, w17, [x1]
5833 \\ cbnz w15, 0b
5834 \\1:
5835 : [ret] "={w0}" (-> u32),
5836 : [val] "{w0}" (val),
5837 [ptr] "{x1}" (ptr),
5838 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5839 : "w15", "w16", "w17", "memory"
5840 );
5841}
5842const __aarch64_ldadd4_acq = if (is_darwin)
5843 if (has_lse)
5844 ldadd4AcqDarwinLse
5845 else
5846 ldadd4AcqDarwinNolse
5847else if (has_lse)
5848 ldadd4AcqNondarwinLse
5849else
5850 ldadd4AcqNondarwinNolse;
5851fn ldclr4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5852 @setRuntimeSafety(false);
5853 __init_aarch64_have_lse_atomics();
5854
5855 return asm volatile (
5856 \\ cbz w16, 8f
5857 \\ ldclra w0, w0, [x1]
5858 \\ cbz wzr, 1f
5859 \\8:
5860 \\ mov w16, w0
5861 \\0:
5862 \\ ldaxr w0, [x1]
5863 \\ bic w17, w0, w16
5864 \\ stxr w15, w17, [x1]
5865 \\ cbnz w15, 0b
5866 \\1:
5867 : [ret] "={w0}" (-> u32),
5868 : [val] "{w0}" (val),
5869 [ptr] "{x1}" (ptr),
5870 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5871 : "w15", "w16", "w17", "memory"
5872 );
5873}
5874fn ldclr4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5875 @setRuntimeSafety(false);
5876 __init_aarch64_have_lse_atomics();
5877
5878 return asm volatile (
5879 \\ cbz w16, 8f
5880 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
5881 \\ cbz wzr, 1f
5882 \\8:
5883 \\ mov w16, w0
5884 \\0:
5885 \\ ldaxr w0, [x1]
5886 \\ bic w17, w0, w16
5887 \\ stxr w15, w17, [x1]
5888 \\ cbnz w15, 0b
5889 \\1:
5890 : [ret] "={w0}" (-> u32),
5891 : [val] "{w0}" (val),
5892 [ptr] "{x1}" (ptr),
5893 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5894 : "w15", "w16", "w17", "memory"
5895 );
5896}
5897fn ldclr4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5898 @setRuntimeSafety(false);
5899 __init_aarch64_have_lse_atomics();
5900
5901 return asm volatile (
5902 \\ cbz w16, 8f
5903 \\ ldclra w0, w0, [x1]
5904 \\ cbz wzr, 1f
5905 \\8:
5906 \\ mov w16, w0
5907 \\0:
5908 \\ ldaxr w0, [x1]
5909 \\ bic w17, w0, w16
5910 \\ stxr w15, w17, [x1]
5911 \\ cbnz w15, 0b
5912 \\1:
5913 : [ret] "={w0}" (-> u32),
5914 : [val] "{w0}" (val),
5915 [ptr] "{x1}" (ptr),
5916 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5917 : "w15", "w16", "w17", "memory"
5918 );
5919}
5920fn ldclr4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5921 @setRuntimeSafety(false);
5922 __init_aarch64_have_lse_atomics();
5923
5924 return asm volatile (
5925 \\ cbz w16, 8f
5926 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
5927 \\ cbz wzr, 1f
5928 \\8:
5929 \\ mov w16, w0
5930 \\0:
5931 \\ ldaxr w0, [x1]
5932 \\ bic w17, w0, w16
5933 \\ stxr w15, w17, [x1]
5934 \\ cbnz w15, 0b
5935 \\1:
5936 : [ret] "={w0}" (-> u32),
5937 : [val] "{w0}" (val),
5938 [ptr] "{x1}" (ptr),
5939 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5940 : "w15", "w16", "w17", "memory"
5941 );
5942}
5943const __aarch64_ldclr4_acq = if (is_darwin)
5944 if (has_lse)
5945 ldclr4AcqDarwinLse
5946 else
5947 ldclr4AcqDarwinNolse
5948else if (has_lse)
5949 ldclr4AcqNondarwinLse
5950else
5951 ldclr4AcqNondarwinNolse;
5952fn ldeor4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5953 @setRuntimeSafety(false);
5954 __init_aarch64_have_lse_atomics();
5955
5956 return asm volatile (
5957 \\ cbz w16, 8f
5958 \\ ldeora w0, w0, [x1]
5959 \\ cbz wzr, 1f
5960 \\8:
5961 \\ mov w16, w0
5962 \\0:
5963 \\ ldaxr w0, [x1]
5964 \\ eor w17, w0, w16
5965 \\ stxr w15, w17, [x1]
5966 \\ cbnz w15, 0b
5967 \\1:
5968 : [ret] "={w0}" (-> u32),
5969 : [val] "{w0}" (val),
5970 [ptr] "{x1}" (ptr),
5971 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5972 : "w15", "w16", "w17", "memory"
5973 );
5974}
5975fn ldeor4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5976 @setRuntimeSafety(false);
5977 __init_aarch64_have_lse_atomics();
5978
5979 return asm volatile (
5980 \\ cbz w16, 8f
5981 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
5982 \\ cbz wzr, 1f
5983 \\8:
5984 \\ mov w16, w0
5985 \\0:
5986 \\ ldaxr w0, [x1]
5987 \\ eor w17, w0, w16
5988 \\ stxr w15, w17, [x1]
5989 \\ cbnz w15, 0b
5990 \\1:
5991 : [ret] "={w0}" (-> u32),
5992 : [val] "{w0}" (val),
5993 [ptr] "{x1}" (ptr),
5994 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5995 : "w15", "w16", "w17", "memory"
5996 );
5997}
5998fn ldeor4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5999 @setRuntimeSafety(false);
6000 __init_aarch64_have_lse_atomics();
6001
6002 return asm volatile (
6003 \\ cbz w16, 8f
6004 \\ ldeora w0, w0, [x1]
6005 \\ cbz wzr, 1f
6006 \\8:
6007 \\ mov w16, w0
6008 \\0:
6009 \\ ldaxr w0, [x1]
6010 \\ eor w17, w0, w16
6011 \\ stxr w15, w17, [x1]
6012 \\ cbnz w15, 0b
6013 \\1:
6014 : [ret] "={w0}" (-> u32),
6015 : [val] "{w0}" (val),
6016 [ptr] "{x1}" (ptr),
6017 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6018 : "w15", "w16", "w17", "memory"
6019 );
6020}
6021fn ldeor4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6022 @setRuntimeSafety(false);
6023 __init_aarch64_have_lse_atomics();
6024
6025 return asm volatile (
6026 \\ cbz w16, 8f
6027 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
6028 \\ cbz wzr, 1f
6029 \\8:
6030 \\ mov w16, w0
6031 \\0:
6032 \\ ldaxr w0, [x1]
6033 \\ eor w17, w0, w16
6034 \\ stxr w15, w17, [x1]
6035 \\ cbnz w15, 0b
6036 \\1:
6037 : [ret] "={w0}" (-> u32),
6038 : [val] "{w0}" (val),
6039 [ptr] "{x1}" (ptr),
6040 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6041 : "w15", "w16", "w17", "memory"
6042 );
6043}
6044const __aarch64_ldeor4_acq = if (is_darwin)
6045 if (has_lse)
6046 ldeor4AcqDarwinLse
6047 else
6048 ldeor4AcqDarwinNolse
6049else if (has_lse)
6050 ldeor4AcqNondarwinLse
6051else
6052 ldeor4AcqNondarwinNolse;
6053fn ldset4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6054 @setRuntimeSafety(false);
6055 __init_aarch64_have_lse_atomics();
6056
6057 return asm volatile (
6058 \\ cbz w16, 8f
6059 \\ ldseta w0, w0, [x1]
6060 \\ cbz wzr, 1f
6061 \\8:
6062 \\ mov w16, w0
6063 \\0:
6064 \\ ldaxr w0, [x1]
6065 \\ orr w17, w0, w16
6066 \\ stxr w15, w17, [x1]
6067 \\ cbnz w15, 0b
6068 \\1:
6069 : [ret] "={w0}" (-> u32),
6070 : [val] "{w0}" (val),
6071 [ptr] "{x1}" (ptr),
6072 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6073 : "w15", "w16", "w17", "memory"
6074 );
6075}
6076fn ldset4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6077 @setRuntimeSafety(false);
6078 __init_aarch64_have_lse_atomics();
6079
6080 return asm volatile (
6081 \\ cbz w16, 8f
6082 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
6083 \\ cbz wzr, 1f
6084 \\8:
6085 \\ mov w16, w0
6086 \\0:
6087 \\ ldaxr w0, [x1]
6088 \\ orr w17, w0, w16
6089 \\ stxr w15, w17, [x1]
6090 \\ cbnz w15, 0b
6091 \\1:
6092 : [ret] "={w0}" (-> u32),
6093 : [val] "{w0}" (val),
6094 [ptr] "{x1}" (ptr),
6095 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6096 : "w15", "w16", "w17", "memory"
6097 );
6098}
6099fn ldset4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6100 @setRuntimeSafety(false);
6101 __init_aarch64_have_lse_atomics();
6102
6103 return asm volatile (
6104 \\ cbz w16, 8f
6105 \\ ldseta w0, w0, [x1]
6106 \\ cbz wzr, 1f
6107 \\8:
6108 \\ mov w16, w0
6109 \\0:
6110 \\ ldaxr w0, [x1]
6111 \\ orr w17, w0, w16
6112 \\ stxr w15, w17, [x1]
6113 \\ cbnz w15, 0b
6114 \\1:
6115 : [ret] "={w0}" (-> u32),
6116 : [val] "{w0}" (val),
6117 [ptr] "{x1}" (ptr),
6118 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6119 : "w15", "w16", "w17", "memory"
6120 );
6121}
6122fn ldset4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6123 @setRuntimeSafety(false);
6124 __init_aarch64_have_lse_atomics();
6125
6126 return asm volatile (
6127 \\ cbz w16, 8f
6128 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
6129 \\ cbz wzr, 1f
6130 \\8:
6131 \\ mov w16, w0
6132 \\0:
6133 \\ ldaxr w0, [x1]
6134 \\ orr w17, w0, w16
6135 \\ stxr w15, w17, [x1]
6136 \\ cbnz w15, 0b
6137 \\1:
6138 : [ret] "={w0}" (-> u32),
6139 : [val] "{w0}" (val),
6140 [ptr] "{x1}" (ptr),
6141 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6142 : "w15", "w16", "w17", "memory"
6143 );
6144}
6145const __aarch64_ldset4_acq = if (is_darwin)
6146 if (has_lse)
6147 ldset4AcqDarwinLse
6148 else
6149 ldset4AcqDarwinNolse
6150else if (has_lse)
6151 ldset4AcqNondarwinLse
6152else
6153 ldset4AcqNondarwinNolse;
6154fn cas4RelDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6155 @setRuntimeSafety(false);
6156 __init_aarch64_have_lse_atomics();
6157
6158 return asm volatile (
6159 \\ cbz w16, 8f
6160 \\ casl w0, w1, [x2]
6161 \\
6162 \\ cbz wzr, 1f
6163 \\8:
6164 \\ mov w16, w0
6165 \\0:
6166 \\ ldxr w0, [x2]
6167 \\ cmp w0, w16
6168 \\ bne 1f
6169 \\ stlxr w17, w1, [x2]
6170 \\ cbnz w17, 0b
6171 \\1:
6172 : [ret] "={w0}" (-> u32),
6173 : [expected] "{w0}" (expected),
6174 [desired] "{w1}" (desired),
6175 [ptr] "{x2}" (ptr),
6176 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6177 : "w15", "w16", "w17", "memory"
6178 );
6179}
6180fn cas4RelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6181 @setRuntimeSafety(false);
6182 __init_aarch64_have_lse_atomics();
6183
6184 return asm volatile (
6185 \\ cbz w16, 8f
6186 \\ .inst 0x08a07c41 + 0x80000000 + 0x008000
6187 \\
6188 \\ cbz wzr, 1f
6189 \\8:
6190 \\ mov w16, w0
6191 \\0:
6192 \\ ldxr w0, [x2]
6193 \\ cmp w0, w16
6194 \\ bne 1f
6195 \\ stlxr w17, w1, [x2]
6196 \\ cbnz w17, 0b
6197 \\1:
6198 : [ret] "={w0}" (-> u32),
6199 : [expected] "{w0}" (expected),
6200 [desired] "{w1}" (desired),
6201 [ptr] "{x2}" (ptr),
6202 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6203 : "w15", "w16", "w17", "memory"
6204 );
6205}
6206fn cas4RelNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6207 @setRuntimeSafety(false);
6208 __init_aarch64_have_lse_atomics();
6209
6210 return asm volatile (
6211 \\ cbz w16, 8f
6212 \\ casl w0, w1, [x2]
6213 \\
6214 \\ cbz wzr, 1f
6215 \\8:
6216 \\ mov w16, w0
6217 \\0:
6218 \\ ldxr w0, [x2]
6219 \\ cmp w0, w16
6220 \\ bne 1f
6221 \\ stlxr w17, w1, [x2]
6222 \\ cbnz w17, 0b
6223 \\1:
6224 : [ret] "={w0}" (-> u32),
6225 : [expected] "{w0}" (expected),
6226 [desired] "{w1}" (desired),
6227 [ptr] "{x2}" (ptr),
6228 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6229 : "w15", "w16", "w17", "memory"
6230 );
6231}
6232fn cas4RelNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6233 @setRuntimeSafety(false);
6234 __init_aarch64_have_lse_atomics();
6235
6236 return asm volatile (
6237 \\ cbz w16, 8f
6238 \\ .inst 0x08a07c41 + 0x80000000 + 0x008000
6239 \\
6240 \\ cbz wzr, 1f
6241 \\8:
6242 \\ mov w16, w0
6243 \\0:
6244 \\ ldxr w0, [x2]
6245 \\ cmp w0, w16
6246 \\ bne 1f
6247 \\ stlxr w17, w1, [x2]
6248 \\ cbnz w17, 0b
6249 \\1:
6250 : [ret] "={w0}" (-> u32),
6251 : [expected] "{w0}" (expected),
6252 [desired] "{w1}" (desired),
6253 [ptr] "{x2}" (ptr),
6254 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6255 : "w15", "w16", "w17", "memory"
6256 );
6257}
6258const __aarch64_cas4_rel = if (is_darwin)
6259 if (has_lse)
6260 cas4RelDarwinLse
6261 else
6262 cas4RelDarwinNolse
6263else if (has_lse)
6264 cas4RelNondarwinLse
6265else
6266 cas4RelNondarwinNolse;
6267fn swp4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6268 @setRuntimeSafety(false);
6269 __init_aarch64_have_lse_atomics();
6270
6271 return asm volatile (
6272 \\ cbz w16, 8f
6273 \\ swpl w0, w0, [x1]
6274 \\ cbz wzr, 1f
6275 \\8:
6276 \\ mov w16, w0
6277 \\0:
6278 \\ ldxr w0, [x1]
6279 \\ stlxr w17, w16, [x1]
6280 \\ cbnz w17, 0b
6281 \\1:
6282 : [ret] "={w0}" (-> u32),
6283 : [val] "{w0}" (val),
6284 [ptr] "{x1}" (ptr),
6285 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6286 : "w15", "w16", "w17", "memory"
6287 );
6288}
6289fn swp4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6290 @setRuntimeSafety(false);
6291 __init_aarch64_have_lse_atomics();
6292
6293 return asm volatile (
6294 \\ cbz w16, 8f
6295 \\ .inst 0x38208020 + 0x80000000 + 0x400000
6296 \\ cbz wzr, 1f
6297 \\8:
6298 \\ mov w16, w0
6299 \\0:
6300 \\ ldxr w0, [x1]
6301 \\ stlxr w17, w16, [x1]
6302 \\ cbnz w17, 0b
6303 \\1:
6304 : [ret] "={w0}" (-> u32),
6305 : [val] "{w0}" (val),
6306 [ptr] "{x1}" (ptr),
6307 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6308 : "w15", "w16", "w17", "memory"
6309 );
6310}
6311fn swp4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6312 @setRuntimeSafety(false);
6313 __init_aarch64_have_lse_atomics();
6314
6315 return asm volatile (
6316 \\ cbz w16, 8f
6317 \\ swpl w0, w0, [x1]
6318 \\ cbz wzr, 1f
6319 \\8:
6320 \\ mov w16, w0
6321 \\0:
6322 \\ ldxr w0, [x1]
6323 \\ stlxr w17, w16, [x1]
6324 \\ cbnz w17, 0b
6325 \\1:
6326 : [ret] "={w0}" (-> u32),
6327 : [val] "{w0}" (val),
6328 [ptr] "{x1}" (ptr),
6329 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6330 : "w15", "w16", "w17", "memory"
6331 );
6332}
6333fn swp4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6334 @setRuntimeSafety(false);
6335 __init_aarch64_have_lse_atomics();
6336
6337 return asm volatile (
6338 \\ cbz w16, 8f
6339 \\ .inst 0x38208020 + 0x80000000 + 0x400000
6340 \\ cbz wzr, 1f
6341 \\8:
6342 \\ mov w16, w0
6343 \\0:
6344 \\ ldxr w0, [x1]
6345 \\ stlxr w17, w16, [x1]
6346 \\ cbnz w17, 0b
6347 \\1:
6348 : [ret] "={w0}" (-> u32),
6349 : [val] "{w0}" (val),
6350 [ptr] "{x1}" (ptr),
6351 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6352 : "w15", "w16", "w17", "memory"
6353 );
6354}
6355const __aarch64_swp4_rel = if (is_darwin)
6356 if (has_lse)
6357 swp4RelDarwinLse
6358 else
6359 swp4RelDarwinNolse
6360else if (has_lse)
6361 swp4RelNondarwinLse
6362else
6363 swp4RelNondarwinNolse;
6364fn ldadd4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6365 @setRuntimeSafety(false);
6366 __init_aarch64_have_lse_atomics();
6367
6368 return asm volatile (
6369 \\ cbz w16, 8f
6370 \\ ldaddl w0, w0, [x1]
6371 \\ cbz wzr, 1f
6372 \\8:
6373 \\ mov w16, w0
6374 \\0:
6375 \\ ldxr w0, [x1]
6376 \\ add w17, w0, w16
6377 \\ stlxr w15, w17, [x1]
6378 \\ cbnz w15, 0b
6379 \\1:
6380 : [ret] "={w0}" (-> u32),
6381 : [val] "{w0}" (val),
6382 [ptr] "{x1}" (ptr),
6383 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6384 : "w15", "w16", "w17", "memory"
6385 );
6386}
6387fn ldadd4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6388 @setRuntimeSafety(false);
6389 __init_aarch64_have_lse_atomics();
6390
6391 return asm volatile (
6392 \\ cbz w16, 8f
6393 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
6394 \\ cbz wzr, 1f
6395 \\8:
6396 \\ mov w16, w0
6397 \\0:
6398 \\ ldxr w0, [x1]
6399 \\ add w17, w0, w16
6400 \\ stlxr w15, w17, [x1]
6401 \\ cbnz w15, 0b
6402 \\1:
6403 : [ret] "={w0}" (-> u32),
6404 : [val] "{w0}" (val),
6405 [ptr] "{x1}" (ptr),
6406 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6407 : "w15", "w16", "w17", "memory"
6408 );
6409}
6410fn ldadd4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6411 @setRuntimeSafety(false);
6412 __init_aarch64_have_lse_atomics();
6413
6414 return asm volatile (
6415 \\ cbz w16, 8f
6416 \\ ldaddl w0, w0, [x1]
6417 \\ cbz wzr, 1f
6418 \\8:
6419 \\ mov w16, w0
6420 \\0:
6421 \\ ldxr w0, [x1]
6422 \\ add w17, w0, w16
6423 \\ stlxr w15, w17, [x1]
6424 \\ cbnz w15, 0b
6425 \\1:
6426 : [ret] "={w0}" (-> u32),
6427 : [val] "{w0}" (val),
6428 [ptr] "{x1}" (ptr),
6429 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6430 : "w15", "w16", "w17", "memory"
6431 );
6432}
6433fn ldadd4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6434 @setRuntimeSafety(false);
6435 __init_aarch64_have_lse_atomics();
6436
6437 return asm volatile (
6438 \\ cbz w16, 8f
6439 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
6440 \\ cbz wzr, 1f
6441 \\8:
6442 \\ mov w16, w0
6443 \\0:
6444 \\ ldxr w0, [x1]
6445 \\ add w17, w0, w16
6446 \\ stlxr w15, w17, [x1]
6447 \\ cbnz w15, 0b
6448 \\1:
6449 : [ret] "={w0}" (-> u32),
6450 : [val] "{w0}" (val),
6451 [ptr] "{x1}" (ptr),
6452 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6453 : "w15", "w16", "w17", "memory"
6454 );
6455}
6456const __aarch64_ldadd4_rel = if (is_darwin)
6457 if (has_lse)
6458 ldadd4RelDarwinLse
6459 else
6460 ldadd4RelDarwinNolse
6461else if (has_lse)
6462 ldadd4RelNondarwinLse
6463else
6464 ldadd4RelNondarwinNolse;
6465fn ldclr4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6466 @setRuntimeSafety(false);
6467 __init_aarch64_have_lse_atomics();
6468
6469 return asm volatile (
6470 \\ cbz w16, 8f
6471 \\ ldclrl w0, w0, [x1]
6472 \\ cbz wzr, 1f
6473 \\8:
6474 \\ mov w16, w0
6475 \\0:
6476 \\ ldxr w0, [x1]
6477 \\ bic w17, w0, w16
6478 \\ stlxr w15, w17, [x1]
6479 \\ cbnz w15, 0b
6480 \\1:
6481 : [ret] "={w0}" (-> u32),
6482 : [val] "{w0}" (val),
6483 [ptr] "{x1}" (ptr),
6484 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6485 : "w15", "w16", "w17", "memory"
6486 );
6487}
6488fn ldclr4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6489 @setRuntimeSafety(false);
6490 __init_aarch64_have_lse_atomics();
6491
6492 return asm volatile (
6493 \\ cbz w16, 8f
6494 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
6495 \\ cbz wzr, 1f
6496 \\8:
6497 \\ mov w16, w0
6498 \\0:
6499 \\ ldxr w0, [x1]
6500 \\ bic w17, w0, w16
6501 \\ stlxr w15, w17, [x1]
6502 \\ cbnz w15, 0b
6503 \\1:
6504 : [ret] "={w0}" (-> u32),
6505 : [val] "{w0}" (val),
6506 [ptr] "{x1}" (ptr),
6507 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6508 : "w15", "w16", "w17", "memory"
6509 );
6510}
6511fn ldclr4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6512 @setRuntimeSafety(false);
6513 __init_aarch64_have_lse_atomics();
6514
6515 return asm volatile (
6516 \\ cbz w16, 8f
6517 \\ ldclrl w0, w0, [x1]
6518 \\ cbz wzr, 1f
6519 \\8:
6520 \\ mov w16, w0
6521 \\0:
6522 \\ ldxr w0, [x1]
6523 \\ bic w17, w0, w16
6524 \\ stlxr w15, w17, [x1]
6525 \\ cbnz w15, 0b
6526 \\1:
6527 : [ret] "={w0}" (-> u32),
6528 : [val] "{w0}" (val),
6529 [ptr] "{x1}" (ptr),
6530 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6531 : "w15", "w16", "w17", "memory"
6532 );
6533}
6534fn ldclr4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6535 @setRuntimeSafety(false);
6536 __init_aarch64_have_lse_atomics();
6537
6538 return asm volatile (
6539 \\ cbz w16, 8f
6540 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
6541 \\ cbz wzr, 1f
6542 \\8:
6543 \\ mov w16, w0
6544 \\0:
6545 \\ ldxr w0, [x1]
6546 \\ bic w17, w0, w16
6547 \\ stlxr w15, w17, [x1]
6548 \\ cbnz w15, 0b
6549 \\1:
6550 : [ret] "={w0}" (-> u32),
6551 : [val] "{w0}" (val),
6552 [ptr] "{x1}" (ptr),
6553 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6554 : "w15", "w16", "w17", "memory"
6555 );
6556}
6557const __aarch64_ldclr4_rel = if (is_darwin)
6558 if (has_lse)
6559 ldclr4RelDarwinLse
6560 else
6561 ldclr4RelDarwinNolse
6562else if (has_lse)
6563 ldclr4RelNondarwinLse
6564else
6565 ldclr4RelNondarwinNolse;
6566fn ldeor4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6567 @setRuntimeSafety(false);
6568 __init_aarch64_have_lse_atomics();
6569
6570 return asm volatile (
6571 \\ cbz w16, 8f
6572 \\ ldeorl w0, w0, [x1]
6573 \\ cbz wzr, 1f
6574 \\8:
6575 \\ mov w16, w0
6576 \\0:
6577 \\ ldxr w0, [x1]
6578 \\ eor w17, w0, w16
6579 \\ stlxr w15, w17, [x1]
6580 \\ cbnz w15, 0b
6581 \\1:
6582 : [ret] "={w0}" (-> u32),
6583 : [val] "{w0}" (val),
6584 [ptr] "{x1}" (ptr),
6585 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6586 : "w15", "w16", "w17", "memory"
6587 );
6588}
6589fn ldeor4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6590 @setRuntimeSafety(false);
6591 __init_aarch64_have_lse_atomics();
6592
6593 return asm volatile (
6594 \\ cbz w16, 8f
6595 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
6596 \\ cbz wzr, 1f
6597 \\8:
6598 \\ mov w16, w0
6599 \\0:
6600 \\ ldxr w0, [x1]
6601 \\ eor w17, w0, w16
6602 \\ stlxr w15, w17, [x1]
6603 \\ cbnz w15, 0b
6604 \\1:
6605 : [ret] "={w0}" (-> u32),
6606 : [val] "{w0}" (val),
6607 [ptr] "{x1}" (ptr),
6608 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6609 : "w15", "w16", "w17", "memory"
6610 );
6611}
6612fn ldeor4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6613 @setRuntimeSafety(false);
6614 __init_aarch64_have_lse_atomics();
6615
6616 return asm volatile (
6617 \\ cbz w16, 8f
6618 \\ ldeorl w0, w0, [x1]
6619 \\ cbz wzr, 1f
6620 \\8:
6621 \\ mov w16, w0
6622 \\0:
6623 \\ ldxr w0, [x1]
6624 \\ eor w17, w0, w16
6625 \\ stlxr w15, w17, [x1]
6626 \\ cbnz w15, 0b
6627 \\1:
6628 : [ret] "={w0}" (-> u32),
6629 : [val] "{w0}" (val),
6630 [ptr] "{x1}" (ptr),
6631 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6632 : "w15", "w16", "w17", "memory"
6633 );
6634}
6635fn ldeor4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6636 @setRuntimeSafety(false);
6637 __init_aarch64_have_lse_atomics();
6638
6639 return asm volatile (
6640 \\ cbz w16, 8f
6641 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
6642 \\ cbz wzr, 1f
6643 \\8:
6644 \\ mov w16, w0
6645 \\0:
6646 \\ ldxr w0, [x1]
6647 \\ eor w17, w0, w16
6648 \\ stlxr w15, w17, [x1]
6649 \\ cbnz w15, 0b
6650 \\1:
6651 : [ret] "={w0}" (-> u32),
6652 : [val] "{w0}" (val),
6653 [ptr] "{x1}" (ptr),
6654 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6655 : "w15", "w16", "w17", "memory"
6656 );
6657}
6658const __aarch64_ldeor4_rel = if (is_darwin)
6659 if (has_lse)
6660 ldeor4RelDarwinLse
6661 else
6662 ldeor4RelDarwinNolse
6663else if (has_lse)
6664 ldeor4RelNondarwinLse
6665else
6666 ldeor4RelNondarwinNolse;
6667fn ldset4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6668 @setRuntimeSafety(false);
6669 __init_aarch64_have_lse_atomics();
6670
6671 return asm volatile (
6672 \\ cbz w16, 8f
6673 \\ ldsetl w0, w0, [x1]
6674 \\ cbz wzr, 1f
6675 \\8:
6676 \\ mov w16, w0
6677 \\0:
6678 \\ ldxr w0, [x1]
6679 \\ orr w17, w0, w16
6680 \\ stlxr w15, w17, [x1]
6681 \\ cbnz w15, 0b
6682 \\1:
6683 : [ret] "={w0}" (-> u32),
6684 : [val] "{w0}" (val),
6685 [ptr] "{x1}" (ptr),
6686 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6687 : "w15", "w16", "w17", "memory"
6688 );
6689}
6690fn ldset4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6691 @setRuntimeSafety(false);
6692 __init_aarch64_have_lse_atomics();
6693
6694 return asm volatile (
6695 \\ cbz w16, 8f
6696 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
6697 \\ cbz wzr, 1f
6698 \\8:
6699 \\ mov w16, w0
6700 \\0:
6701 \\ ldxr w0, [x1]
6702 \\ orr w17, w0, w16
6703 \\ stlxr w15, w17, [x1]
6704 \\ cbnz w15, 0b
6705 \\1:
6706 : [ret] "={w0}" (-> u32),
6707 : [val] "{w0}" (val),
6708 [ptr] "{x1}" (ptr),
6709 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6710 : "w15", "w16", "w17", "memory"
6711 );
6712}
6713fn ldset4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6714 @setRuntimeSafety(false);
6715 __init_aarch64_have_lse_atomics();
6716
6717 return asm volatile (
6718 \\ cbz w16, 8f
6719 \\ ldsetl w0, w0, [x1]
6720 \\ cbz wzr, 1f
6721 \\8:
6722 \\ mov w16, w0
6723 \\0:
6724 \\ ldxr w0, [x1]
6725 \\ orr w17, w0, w16
6726 \\ stlxr w15, w17, [x1]
6727 \\ cbnz w15, 0b
6728 \\1:
6729 : [ret] "={w0}" (-> u32),
6730 : [val] "{w0}" (val),
6731 [ptr] "{x1}" (ptr),
6732 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6733 : "w15", "w16", "w17", "memory"
6734 );
6735}
6736fn ldset4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6737 @setRuntimeSafety(false);
6738 __init_aarch64_have_lse_atomics();
6739
6740 return asm volatile (
6741 \\ cbz w16, 8f
6742 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
6743 \\ cbz wzr, 1f
6744 \\8:
6745 \\ mov w16, w0
6746 \\0:
6747 \\ ldxr w0, [x1]
6748 \\ orr w17, w0, w16
6749 \\ stlxr w15, w17, [x1]
6750 \\ cbnz w15, 0b
6751 \\1:
6752 : [ret] "={w0}" (-> u32),
6753 : [val] "{w0}" (val),
6754 [ptr] "{x1}" (ptr),
6755 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6756 : "w15", "w16", "w17", "memory"
6757 );
6758}
6759const __aarch64_ldset4_rel = if (is_darwin)
6760 if (has_lse)
6761 ldset4RelDarwinLse
6762 else
6763 ldset4RelDarwinNolse
6764else if (has_lse)
6765 ldset4RelNondarwinLse
6766else
6767 ldset4RelNondarwinNolse;
6768fn cas4AcqRelDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6769 @setRuntimeSafety(false);
6770 __init_aarch64_have_lse_atomics();
6771
6772 return asm volatile (
6773 \\ cbz w16, 8f
6774 \\ casal w0, w1, [x2]
6775 \\
6776 \\ cbz wzr, 1f
6777 \\8:
6778 \\ mov w16, w0
6779 \\0:
6780 \\ ldaxr w0, [x2]
6781 \\ cmp w0, w16
6782 \\ bne 1f
6783 \\ stlxr w17, w1, [x2]
6784 \\ cbnz w17, 0b
6785 \\1:
6786 : [ret] "={w0}" (-> u32),
6787 : [expected] "{w0}" (expected),
6788 [desired] "{w1}" (desired),
6789 [ptr] "{x2}" (ptr),
6790 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6791 : "w15", "w16", "w17", "memory"
6792 );
6793}
6794fn cas4AcqRelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6795 @setRuntimeSafety(false);
6796 __init_aarch64_have_lse_atomics();
6797
6798 return asm volatile (
6799 \\ cbz w16, 8f
6800 \\ .inst 0x08a07c41 + 0x80000000 + 0x408000
6801 \\
6802 \\ cbz wzr, 1f
6803 \\8:
6804 \\ mov w16, w0
6805 \\0:
6806 \\ ldaxr w0, [x2]
6807 \\ cmp w0, w16
6808 \\ bne 1f
6809 \\ stlxr w17, w1, [x2]
6810 \\ cbnz w17, 0b
6811 \\1:
6812 : [ret] "={w0}" (-> u32),
6813 : [expected] "{w0}" (expected),
6814 [desired] "{w1}" (desired),
6815 [ptr] "{x2}" (ptr),
6816 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6817 : "w15", "w16", "w17", "memory"
6818 );
6819}
6820fn cas4AcqRelNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6821 @setRuntimeSafety(false);
6822 __init_aarch64_have_lse_atomics();
6823
6824 return asm volatile (
6825 \\ cbz w16, 8f
6826 \\ casal w0, w1, [x2]
6827 \\
6828 \\ cbz wzr, 1f
6829 \\8:
6830 \\ mov w16, w0
6831 \\0:
6832 \\ ldaxr w0, [x2]
6833 \\ cmp w0, w16
6834 \\ bne 1f
6835 \\ stlxr w17, w1, [x2]
6836 \\ cbnz w17, 0b
6837 \\1:
6838 : [ret] "={w0}" (-> u32),
6839 : [expected] "{w0}" (expected),
6840 [desired] "{w1}" (desired),
6841 [ptr] "{x2}" (ptr),
6842 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6843 : "w15", "w16", "w17", "memory"
6844 );
6845}
6846fn cas4AcqRelNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6847 @setRuntimeSafety(false);
6848 __init_aarch64_have_lse_atomics();
6849
6850 return asm volatile (
6851 \\ cbz w16, 8f
6852 \\ .inst 0x08a07c41 + 0x80000000 + 0x408000
6853 \\
6854 \\ cbz wzr, 1f
6855 \\8:
6856 \\ mov w16, w0
6857 \\0:
6858 \\ ldaxr w0, [x2]
6859 \\ cmp w0, w16
6860 \\ bne 1f
6861 \\ stlxr w17, w1, [x2]
6862 \\ cbnz w17, 0b
6863 \\1:
6864 : [ret] "={w0}" (-> u32),
6865 : [expected] "{w0}" (expected),
6866 [desired] "{w1}" (desired),
6867 [ptr] "{x2}" (ptr),
6868 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6869 : "w15", "w16", "w17", "memory"
6870 );
6871}
6872const __aarch64_cas4_acq_rel = if (is_darwin)
6873 if (has_lse)
6874 cas4AcqRelDarwinLse
6875 else
6876 cas4AcqRelDarwinNolse
6877else if (has_lse)
6878 cas4AcqRelNondarwinLse
6879else
6880 cas4AcqRelNondarwinNolse;
6881fn swp4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6882 @setRuntimeSafety(false);
6883 __init_aarch64_have_lse_atomics();
6884
6885 return asm volatile (
6886 \\ cbz w16, 8f
6887 \\ swpal w0, w0, [x1]
6888 \\ cbz wzr, 1f
6889 \\8:
6890 \\ mov w16, w0
6891 \\0:
6892 \\ ldaxr w0, [x1]
6893 \\ stlxr w17, w16, [x1]
6894 \\ cbnz w17, 0b
6895 \\1:
6896 : [ret] "={w0}" (-> u32),
6897 : [val] "{w0}" (val),
6898 [ptr] "{x1}" (ptr),
6899 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6900 : "w15", "w16", "w17", "memory"
6901 );
6902}
6903fn swp4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6904 @setRuntimeSafety(false);
6905 __init_aarch64_have_lse_atomics();
6906
6907 return asm volatile (
6908 \\ cbz w16, 8f
6909 \\ .inst 0x38208020 + 0x80000000 + 0xc00000
6910 \\ cbz wzr, 1f
6911 \\8:
6912 \\ mov w16, w0
6913 \\0:
6914 \\ ldaxr w0, [x1]
6915 \\ stlxr w17, w16, [x1]
6916 \\ cbnz w17, 0b
6917 \\1:
6918 : [ret] "={w0}" (-> u32),
6919 : [val] "{w0}" (val),
6920 [ptr] "{x1}" (ptr),
6921 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6922 : "w15", "w16", "w17", "memory"
6923 );
6924}
6925fn swp4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6926 @setRuntimeSafety(false);
6927 __init_aarch64_have_lse_atomics();
6928
6929 return asm volatile (
6930 \\ cbz w16, 8f
6931 \\ swpal w0, w0, [x1]
6932 \\ cbz wzr, 1f
6933 \\8:
6934 \\ mov w16, w0
6935 \\0:
6936 \\ ldaxr w0, [x1]
6937 \\ stlxr w17, w16, [x1]
6938 \\ cbnz w17, 0b
6939 \\1:
6940 : [ret] "={w0}" (-> u32),
6941 : [val] "{w0}" (val),
6942 [ptr] "{x1}" (ptr),
6943 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6944 : "w15", "w16", "w17", "memory"
6945 );
6946}
6947fn swp4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6948 @setRuntimeSafety(false);
6949 __init_aarch64_have_lse_atomics();
6950
6951 return asm volatile (
6952 \\ cbz w16, 8f
6953 \\ .inst 0x38208020 + 0x80000000 + 0xc00000
6954 \\ cbz wzr, 1f
6955 \\8:
6956 \\ mov w16, w0
6957 \\0:
6958 \\ ldaxr w0, [x1]
6959 \\ stlxr w17, w16, [x1]
6960 \\ cbnz w17, 0b
6961 \\1:
6962 : [ret] "={w0}" (-> u32),
6963 : [val] "{w0}" (val),
6964 [ptr] "{x1}" (ptr),
6965 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6966 : "w15", "w16", "w17", "memory"
6967 );
6968}
6969const __aarch64_swp4_acq_rel = if (is_darwin)
6970 if (has_lse)
6971 swp4AcqRelDarwinLse
6972 else
6973 swp4AcqRelDarwinNolse
6974else if (has_lse)
6975 swp4AcqRelNondarwinLse
6976else
6977 swp4AcqRelNondarwinNolse;
6978fn ldadd4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6979 @setRuntimeSafety(false);
6980 __init_aarch64_have_lse_atomics();
6981
6982 return asm volatile (
6983 \\ cbz w16, 8f
6984 \\ ldaddal w0, w0, [x1]
6985 \\ cbz wzr, 1f
6986 \\8:
6987 \\ mov w16, w0
6988 \\0:
6989 \\ ldaxr w0, [x1]
6990 \\ add w17, w0, w16
6991 \\ stlxr w15, w17, [x1]
6992 \\ cbnz w15, 0b
6993 \\1:
6994 : [ret] "={w0}" (-> u32),
6995 : [val] "{w0}" (val),
6996 [ptr] "{x1}" (ptr),
6997 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6998 : "w15", "w16", "w17", "memory"
6999 );
7000}
7001fn ldadd4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7002 @setRuntimeSafety(false);
7003 __init_aarch64_have_lse_atomics();
7004
7005 return asm volatile (
7006 \\ cbz w16, 8f
7007 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
7008 \\ cbz wzr, 1f
7009 \\8:
7010 \\ mov w16, w0
7011 \\0:
7012 \\ ldaxr w0, [x1]
7013 \\ add w17, w0, w16
7014 \\ stlxr w15, w17, [x1]
7015 \\ cbnz w15, 0b
7016 \\1:
7017 : [ret] "={w0}" (-> u32),
7018 : [val] "{w0}" (val),
7019 [ptr] "{x1}" (ptr),
7020 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7021 : "w15", "w16", "w17", "memory"
7022 );
7023}
7024fn ldadd4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7025 @setRuntimeSafety(false);
7026 __init_aarch64_have_lse_atomics();
7027
7028 return asm volatile (
7029 \\ cbz w16, 8f
7030 \\ ldaddal w0, w0, [x1]
7031 \\ cbz wzr, 1f
7032 \\8:
7033 \\ mov w16, w0
7034 \\0:
7035 \\ ldaxr w0, [x1]
7036 \\ add w17, w0, w16
7037 \\ stlxr w15, w17, [x1]
7038 \\ cbnz w15, 0b
7039 \\1:
7040 : [ret] "={w0}" (-> u32),
7041 : [val] "{w0}" (val),
7042 [ptr] "{x1}" (ptr),
7043 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7044 : "w15", "w16", "w17", "memory"
7045 );
7046}
7047fn ldadd4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7048 @setRuntimeSafety(false);
7049 __init_aarch64_have_lse_atomics();
7050
7051 return asm volatile (
7052 \\ cbz w16, 8f
7053 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
7054 \\ cbz wzr, 1f
7055 \\8:
7056 \\ mov w16, w0
7057 \\0:
7058 \\ ldaxr w0, [x1]
7059 \\ add w17, w0, w16
7060 \\ stlxr w15, w17, [x1]
7061 \\ cbnz w15, 0b
7062 \\1:
7063 : [ret] "={w0}" (-> u32),
7064 : [val] "{w0}" (val),
7065 [ptr] "{x1}" (ptr),
7066 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7067 : "w15", "w16", "w17", "memory"
7068 );
7069}
7070const __aarch64_ldadd4_acq_rel = if (is_darwin)
7071 if (has_lse)
7072 ldadd4AcqRelDarwinLse
7073 else
7074 ldadd4AcqRelDarwinNolse
7075else if (has_lse)
7076 ldadd4AcqRelNondarwinLse
7077else
7078 ldadd4AcqRelNondarwinNolse;
7079fn ldclr4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7080 @setRuntimeSafety(false);
7081 __init_aarch64_have_lse_atomics();
7082
7083 return asm volatile (
7084 \\ cbz w16, 8f
7085 \\ ldclral w0, w0, [x1]
7086 \\ cbz wzr, 1f
7087 \\8:
7088 \\ mov w16, w0
7089 \\0:
7090 \\ ldaxr w0, [x1]
7091 \\ bic w17, w0, w16
7092 \\ stlxr w15, w17, [x1]
7093 \\ cbnz w15, 0b
7094 \\1:
7095 : [ret] "={w0}" (-> u32),
7096 : [val] "{w0}" (val),
7097 [ptr] "{x1}" (ptr),
7098 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7099 : "w15", "w16", "w17", "memory"
7100 );
7101}
7102fn ldclr4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7103 @setRuntimeSafety(false);
7104 __init_aarch64_have_lse_atomics();
7105
7106 return asm volatile (
7107 \\ cbz w16, 8f
7108 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
7109 \\ cbz wzr, 1f
7110 \\8:
7111 \\ mov w16, w0
7112 \\0:
7113 \\ ldaxr w0, [x1]
7114 \\ bic w17, w0, w16
7115 \\ stlxr w15, w17, [x1]
7116 \\ cbnz w15, 0b
7117 \\1:
7118 : [ret] "={w0}" (-> u32),
7119 : [val] "{w0}" (val),
7120 [ptr] "{x1}" (ptr),
7121 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7122 : "w15", "w16", "w17", "memory"
7123 );
7124}
7125fn ldclr4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7126 @setRuntimeSafety(false);
7127 __init_aarch64_have_lse_atomics();
7128
7129 return asm volatile (
7130 \\ cbz w16, 8f
7131 \\ ldclral w0, w0, [x1]
7132 \\ cbz wzr, 1f
7133 \\8:
7134 \\ mov w16, w0
7135 \\0:
7136 \\ ldaxr w0, [x1]
7137 \\ bic w17, w0, w16
7138 \\ stlxr w15, w17, [x1]
7139 \\ cbnz w15, 0b
7140 \\1:
7141 : [ret] "={w0}" (-> u32),
7142 : [val] "{w0}" (val),
7143 [ptr] "{x1}" (ptr),
7144 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7145 : "w15", "w16", "w17", "memory"
7146 );
7147}
7148fn ldclr4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7149 @setRuntimeSafety(false);
7150 __init_aarch64_have_lse_atomics();
7151
7152 return asm volatile (
7153 \\ cbz w16, 8f
7154 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
7155 \\ cbz wzr, 1f
7156 \\8:
7157 \\ mov w16, w0
7158 \\0:
7159 \\ ldaxr w0, [x1]
7160 \\ bic w17, w0, w16
7161 \\ stlxr w15, w17, [x1]
7162 \\ cbnz w15, 0b
7163 \\1:
7164 : [ret] "={w0}" (-> u32),
7165 : [val] "{w0}" (val),
7166 [ptr] "{x1}" (ptr),
7167 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7168 : "w15", "w16", "w17", "memory"
7169 );
7170}
7171const __aarch64_ldclr4_acq_rel = if (is_darwin)
7172 if (has_lse)
7173 ldclr4AcqRelDarwinLse
7174 else
7175 ldclr4AcqRelDarwinNolse
7176else if (has_lse)
7177 ldclr4AcqRelNondarwinLse
7178else
7179 ldclr4AcqRelNondarwinNolse;
7180fn ldeor4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7181 @setRuntimeSafety(false);
7182 __init_aarch64_have_lse_atomics();
7183
7184 return asm volatile (
7185 \\ cbz w16, 8f
7186 \\ ldeoral w0, w0, [x1]
7187 \\ cbz wzr, 1f
7188 \\8:
7189 \\ mov w16, w0
7190 \\0:
7191 \\ ldaxr w0, [x1]
7192 \\ eor w17, w0, w16
7193 \\ stlxr w15, w17, [x1]
7194 \\ cbnz w15, 0b
7195 \\1:
7196 : [ret] "={w0}" (-> u32),
7197 : [val] "{w0}" (val),
7198 [ptr] "{x1}" (ptr),
7199 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7200 : "w15", "w16", "w17", "memory"
7201 );
7202}
7203fn ldeor4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7204 @setRuntimeSafety(false);
7205 __init_aarch64_have_lse_atomics();
7206
7207 return asm volatile (
7208 \\ cbz w16, 8f
7209 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
7210 \\ cbz wzr, 1f
7211 \\8:
7212 \\ mov w16, w0
7213 \\0:
7214 \\ ldaxr w0, [x1]
7215 \\ eor w17, w0, w16
7216 \\ stlxr w15, w17, [x1]
7217 \\ cbnz w15, 0b
7218 \\1:
7219 : [ret] "={w0}" (-> u32),
7220 : [val] "{w0}" (val),
7221 [ptr] "{x1}" (ptr),
7222 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7223 : "w15", "w16", "w17", "memory"
7224 );
7225}
7226fn ldeor4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7227 @setRuntimeSafety(false);
7228 __init_aarch64_have_lse_atomics();
7229
7230 return asm volatile (
7231 \\ cbz w16, 8f
7232 \\ ldeoral w0, w0, [x1]
7233 \\ cbz wzr, 1f
7234 \\8:
7235 \\ mov w16, w0
7236 \\0:
7237 \\ ldaxr w0, [x1]
7238 \\ eor w17, w0, w16
7239 \\ stlxr w15, w17, [x1]
7240 \\ cbnz w15, 0b
7241 \\1:
7242 : [ret] "={w0}" (-> u32),
7243 : [val] "{w0}" (val),
7244 [ptr] "{x1}" (ptr),
7245 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7246 : "w15", "w16", "w17", "memory"
7247 );
7248}
7249fn ldeor4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7250 @setRuntimeSafety(false);
7251 __init_aarch64_have_lse_atomics();
7252
7253 return asm volatile (
7254 \\ cbz w16, 8f
7255 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
7256 \\ cbz wzr, 1f
7257 \\8:
7258 \\ mov w16, w0
7259 \\0:
7260 \\ ldaxr w0, [x1]
7261 \\ eor w17, w0, w16
7262 \\ stlxr w15, w17, [x1]
7263 \\ cbnz w15, 0b
7264 \\1:
7265 : [ret] "={w0}" (-> u32),
7266 : [val] "{w0}" (val),
7267 [ptr] "{x1}" (ptr),
7268 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7269 : "w15", "w16", "w17", "memory"
7270 );
7271}
7272const __aarch64_ldeor4_acq_rel = if (is_darwin)
7273 if (has_lse)
7274 ldeor4AcqRelDarwinLse
7275 else
7276 ldeor4AcqRelDarwinNolse
7277else if (has_lse)
7278 ldeor4AcqRelNondarwinLse
7279else
7280 ldeor4AcqRelNondarwinNolse;
7281fn ldset4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7282 @setRuntimeSafety(false);
7283 __init_aarch64_have_lse_atomics();
7284
7285 return asm volatile (
7286 \\ cbz w16, 8f
7287 \\ ldsetal w0, w0, [x1]
7288 \\ cbz wzr, 1f
7289 \\8:
7290 \\ mov w16, w0
7291 \\0:
7292 \\ ldaxr w0, [x1]
7293 \\ orr w17, w0, w16
7294 \\ stlxr w15, w17, [x1]
7295 \\ cbnz w15, 0b
7296 \\1:
7297 : [ret] "={w0}" (-> u32),
7298 : [val] "{w0}" (val),
7299 [ptr] "{x1}" (ptr),
7300 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7301 : "w15", "w16", "w17", "memory"
7302 );
7303}
7304fn ldset4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7305 @setRuntimeSafety(false);
7306 __init_aarch64_have_lse_atomics();
7307
7308 return asm volatile (
7309 \\ cbz w16, 8f
7310 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
7311 \\ cbz wzr, 1f
7312 \\8:
7313 \\ mov w16, w0
7314 \\0:
7315 \\ ldaxr w0, [x1]
7316 \\ orr w17, w0, w16
7317 \\ stlxr w15, w17, [x1]
7318 \\ cbnz w15, 0b
7319 \\1:
7320 : [ret] "={w0}" (-> u32),
7321 : [val] "{w0}" (val),
7322 [ptr] "{x1}" (ptr),
7323 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7324 : "w15", "w16", "w17", "memory"
7325 );
7326}
7327fn ldset4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7328 @setRuntimeSafety(false);
7329 __init_aarch64_have_lse_atomics();
7330
7331 return asm volatile (
7332 \\ cbz w16, 8f
7333 \\ ldsetal w0, w0, [x1]
7334 \\ cbz wzr, 1f
7335 \\8:
7336 \\ mov w16, w0
7337 \\0:
7338 \\ ldaxr w0, [x1]
7339 \\ orr w17, w0, w16
7340 \\ stlxr w15, w17, [x1]
7341 \\ cbnz w15, 0b
7342 \\1:
7343 : [ret] "={w0}" (-> u32),
7344 : [val] "{w0}" (val),
7345 [ptr] "{x1}" (ptr),
7346 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7347 : "w15", "w16", "w17", "memory"
7348 );
7349}
7350fn ldset4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7351 @setRuntimeSafety(false);
7352 __init_aarch64_have_lse_atomics();
7353
7354 return asm volatile (
7355 \\ cbz w16, 8f
7356 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
7357 \\ cbz wzr, 1f
7358 \\8:
7359 \\ mov w16, w0
7360 \\0:
7361 \\ ldaxr w0, [x1]
7362 \\ orr w17, w0, w16
7363 \\ stlxr w15, w17, [x1]
7364 \\ cbnz w15, 0b
7365 \\1:
7366 : [ret] "={w0}" (-> u32),
7367 : [val] "{w0}" (val),
7368 [ptr] "{x1}" (ptr),
7369 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7370 : "w15", "w16", "w17", "memory"
7371 );
7372}
7373const __aarch64_ldset4_acq_rel = if (is_darwin)
7374 if (has_lse)
7375 ldset4AcqRelDarwinLse
7376 else
7377 ldset4AcqRelDarwinNolse
7378else if (has_lse)
7379 ldset4AcqRelNondarwinLse
7380else
7381 ldset4AcqRelNondarwinNolse;
7382fn cas8RelaxDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7383 @setRuntimeSafety(false);
7384 __init_aarch64_have_lse_atomics();
7385
7386 return asm volatile (
7387 \\ cbz w16, 8f
7388 \\ cas x0, x1, [x2]
7389 \\
7390 \\ cbz wzr, 1f
7391 \\8:
7392 \\ mov x16, x0
7393 \\0:
7394 \\ ldxr x0, [x2]
7395 \\ cmp x0, x16
7396 \\ bne 1f
7397 \\ stxr w17, x1, [x2]
7398 \\ cbnz w17, 0b
7399 \\1:
7400 : [ret] "={x0}" (-> u64),
7401 : [expected] "{x0}" (expected),
7402 [desired] "{x1}" (desired),
7403 [ptr] "{x2}" (ptr),
7404 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7405 : "w15", "w16", "w17", "memory"
7406 );
7407}
7408fn cas8RelaxDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7409 @setRuntimeSafety(false);
7410 __init_aarch64_have_lse_atomics();
7411
7412 return asm volatile (
7413 \\ cbz w16, 8f
7414 \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
7415 \\
7416 \\ cbz wzr, 1f
7417 \\8:
7418 \\ mov x16, x0
7419 \\0:
7420 \\ ldxr x0, [x2]
7421 \\ cmp x0, x16
7422 \\ bne 1f
7423 \\ stxr w17, x1, [x2]
7424 \\ cbnz w17, 0b
7425 \\1:
7426 : [ret] "={x0}" (-> u64),
7427 : [expected] "{x0}" (expected),
7428 [desired] "{x1}" (desired),
7429 [ptr] "{x2}" (ptr),
7430 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7431 : "w15", "w16", "w17", "memory"
7432 );
7433}
7434fn cas8RelaxNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7435 @setRuntimeSafety(false);
7436 __init_aarch64_have_lse_atomics();
7437
7438 return asm volatile (
7439 \\ cbz w16, 8f
7440 \\ cas x0, x1, [x2]
7441 \\
7442 \\ cbz wzr, 1f
7443 \\8:
7444 \\ mov x16, x0
7445 \\0:
7446 \\ ldxr x0, [x2]
7447 \\ cmp x0, x16
7448 \\ bne 1f
7449 \\ stxr w17, x1, [x2]
7450 \\ cbnz w17, 0b
7451 \\1:
7452 : [ret] "={x0}" (-> u64),
7453 : [expected] "{x0}" (expected),
7454 [desired] "{x1}" (desired),
7455 [ptr] "{x2}" (ptr),
7456 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7457 : "w15", "w16", "w17", "memory"
7458 );
7459}
7460fn cas8RelaxNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7461 @setRuntimeSafety(false);
7462 __init_aarch64_have_lse_atomics();
7463
7464 return asm volatile (
7465 \\ cbz w16, 8f
7466 \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
7467 \\
7468 \\ cbz wzr, 1f
7469 \\8:
7470 \\ mov x16, x0
7471 \\0:
7472 \\ ldxr x0, [x2]
7473 \\ cmp x0, x16
7474 \\ bne 1f
7475 \\ stxr w17, x1, [x2]
7476 \\ cbnz w17, 0b
7477 \\1:
7478 : [ret] "={x0}" (-> u64),
7479 : [expected] "{x0}" (expected),
7480 [desired] "{x1}" (desired),
7481 [ptr] "{x2}" (ptr),
7482 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7483 : "w15", "w16", "w17", "memory"
7484 );
7485}
7486const __aarch64_cas8_relax = if (is_darwin)
7487 if (has_lse)
7488 cas8RelaxDarwinLse
7489 else
7490 cas8RelaxDarwinNolse
7491else if (has_lse)
7492 cas8RelaxNondarwinLse
7493else
7494 cas8RelaxNondarwinNolse;
7495fn swp8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7496 @setRuntimeSafety(false);
7497 __init_aarch64_have_lse_atomics();
7498
7499 return asm volatile (
7500 \\ cbz w16, 8f
7501 \\ swp x0, x0, [x1]
7502 \\ cbz wzr, 1f
7503 \\8:
7504 \\ mov x16, x0
7505 \\0:
7506 \\ ldxr x0, [x1]
7507 \\ stxr w17, x16, [x1]
7508 \\ cbnz w17, 0b
7509 \\1:
7510 : [ret] "={x0}" (-> u64),
7511 : [val] "{x0}" (val),
7512 [ptr] "{x1}" (ptr),
7513 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7514 : "w15", "w16", "w17", "memory"
7515 );
7516}
7517fn swp8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7518 @setRuntimeSafety(false);
7519 __init_aarch64_have_lse_atomics();
7520
7521 return asm volatile (
7522 \\ cbz w16, 8f
7523 \\ .inst 0x38208020 + 0xc0000000 + 0x000000
7524 \\ cbz wzr, 1f
7525 \\8:
7526 \\ mov x16, x0
7527 \\0:
7528 \\ ldxr x0, [x1]
7529 \\ stxr w17, x16, [x1]
7530 \\ cbnz w17, 0b
7531 \\1:
7532 : [ret] "={x0}" (-> u64),
7533 : [val] "{x0}" (val),
7534 [ptr] "{x1}" (ptr),
7535 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7536 : "w15", "w16", "w17", "memory"
7537 );
7538}
7539fn swp8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7540 @setRuntimeSafety(false);
7541 __init_aarch64_have_lse_atomics();
7542
7543 return asm volatile (
7544 \\ cbz w16, 8f
7545 \\ swp x0, x0, [x1]
7546 \\ cbz wzr, 1f
7547 \\8:
7548 \\ mov x16, x0
7549 \\0:
7550 \\ ldxr x0, [x1]
7551 \\ stxr w17, x16, [x1]
7552 \\ cbnz w17, 0b
7553 \\1:
7554 : [ret] "={x0}" (-> u64),
7555 : [val] "{x0}" (val),
7556 [ptr] "{x1}" (ptr),
7557 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7558 : "w15", "w16", "w17", "memory"
7559 );
7560}
7561fn swp8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7562 @setRuntimeSafety(false);
7563 __init_aarch64_have_lse_atomics();
7564
7565 return asm volatile (
7566 \\ cbz w16, 8f
7567 \\ .inst 0x38208020 + 0xc0000000 + 0x000000
7568 \\ cbz wzr, 1f
7569 \\8:
7570 \\ mov x16, x0
7571 \\0:
7572 \\ ldxr x0, [x1]
7573 \\ stxr w17, x16, [x1]
7574 \\ cbnz w17, 0b
7575 \\1:
7576 : [ret] "={x0}" (-> u64),
7577 : [val] "{x0}" (val),
7578 [ptr] "{x1}" (ptr),
7579 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7580 : "w15", "w16", "w17", "memory"
7581 );
7582}
7583const __aarch64_swp8_relax = if (is_darwin)
7584 if (has_lse)
7585 swp8RelaxDarwinLse
7586 else
7587 swp8RelaxDarwinNolse
7588else if (has_lse)
7589 swp8RelaxNondarwinLse
7590else
7591 swp8RelaxNondarwinNolse;
7592fn ldadd8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7593 @setRuntimeSafety(false);
7594 __init_aarch64_have_lse_atomics();
7595
7596 return asm volatile (
7597 \\ cbz w16, 8f
7598 \\ ldadd x0, x0, [x1]
7599 \\ cbz wzr, 1f
7600 \\8:
7601 \\ mov x16, x0
7602 \\0:
7603 \\ ldxr x0, [x1]
7604 \\ add x17, x0, x16
7605 \\ stxr w15, x17, [x1]
7606 \\ cbnz w15, 0b
7607 \\1:
7608 : [ret] "={x0}" (-> u64),
7609 : [val] "{x0}" (val),
7610 [ptr] "{x1}" (ptr),
7611 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7612 : "w15", "w16", "w17", "memory"
7613 );
7614}
7615fn ldadd8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7616 @setRuntimeSafety(false);
7617 __init_aarch64_have_lse_atomics();
7618
7619 return asm volatile (
7620 \\ cbz w16, 8f
7621 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
7622 \\ cbz wzr, 1f
7623 \\8:
7624 \\ mov x16, x0
7625 \\0:
7626 \\ ldxr x0, [x1]
7627 \\ add x17, x0, x16
7628 \\ stxr w15, x17, [x1]
7629 \\ cbnz w15, 0b
7630 \\1:
7631 : [ret] "={x0}" (-> u64),
7632 : [val] "{x0}" (val),
7633 [ptr] "{x1}" (ptr),
7634 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7635 : "w15", "w16", "w17", "memory"
7636 );
7637}
7638fn ldadd8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7639 @setRuntimeSafety(false);
7640 __init_aarch64_have_lse_atomics();
7641
7642 return asm volatile (
7643 \\ cbz w16, 8f
7644 \\ ldadd x0, x0, [x1]
7645 \\ cbz wzr, 1f
7646 \\8:
7647 \\ mov x16, x0
7648 \\0:
7649 \\ ldxr x0, [x1]
7650 \\ add x17, x0, x16
7651 \\ stxr w15, x17, [x1]
7652 \\ cbnz w15, 0b
7653 \\1:
7654 : [ret] "={x0}" (-> u64),
7655 : [val] "{x0}" (val),
7656 [ptr] "{x1}" (ptr),
7657 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7658 : "w15", "w16", "w17", "memory"
7659 );
7660}
7661fn ldadd8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7662 @setRuntimeSafety(false);
7663 __init_aarch64_have_lse_atomics();
7664
7665 return asm volatile (
7666 \\ cbz w16, 8f
7667 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
7668 \\ cbz wzr, 1f
7669 \\8:
7670 \\ mov x16, x0
7671 \\0:
7672 \\ ldxr x0, [x1]
7673 \\ add x17, x0, x16
7674 \\ stxr w15, x17, [x1]
7675 \\ cbnz w15, 0b
7676 \\1:
7677 : [ret] "={x0}" (-> u64),
7678 : [val] "{x0}" (val),
7679 [ptr] "{x1}" (ptr),
7680 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7681 : "w15", "w16", "w17", "memory"
7682 );
7683}
7684const __aarch64_ldadd8_relax = if (is_darwin)
7685 if (has_lse)
7686 ldadd8RelaxDarwinLse
7687 else
7688 ldadd8RelaxDarwinNolse
7689else if (has_lse)
7690 ldadd8RelaxNondarwinLse
7691else
7692 ldadd8RelaxNondarwinNolse;
7693fn ldclr8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7694 @setRuntimeSafety(false);
7695 __init_aarch64_have_lse_atomics();
7696
7697 return asm volatile (
7698 \\ cbz w16, 8f
7699 \\ ldclr x0, x0, [x1]
7700 \\ cbz wzr, 1f
7701 \\8:
7702 \\ mov x16, x0
7703 \\0:
7704 \\ ldxr x0, [x1]
7705 \\ bic x17, x0, x16
7706 \\ stxr w15, x17, [x1]
7707 \\ cbnz w15, 0b
7708 \\1:
7709 : [ret] "={x0}" (-> u64),
7710 : [val] "{x0}" (val),
7711 [ptr] "{x1}" (ptr),
7712 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7713 : "w15", "w16", "w17", "memory"
7714 );
7715}
7716fn ldclr8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7717 @setRuntimeSafety(false);
7718 __init_aarch64_have_lse_atomics();
7719
7720 return asm volatile (
7721 \\ cbz w16, 8f
7722 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
7723 \\ cbz wzr, 1f
7724 \\8:
7725 \\ mov x16, x0
7726 \\0:
7727 \\ ldxr x0, [x1]
7728 \\ bic x17, x0, x16
7729 \\ stxr w15, x17, [x1]
7730 \\ cbnz w15, 0b
7731 \\1:
7732 : [ret] "={x0}" (-> u64),
7733 : [val] "{x0}" (val),
7734 [ptr] "{x1}" (ptr),
7735 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7736 : "w15", "w16", "w17", "memory"
7737 );
7738}
7739fn ldclr8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7740 @setRuntimeSafety(false);
7741 __init_aarch64_have_lse_atomics();
7742
7743 return asm volatile (
7744 \\ cbz w16, 8f
7745 \\ ldclr x0, x0, [x1]
7746 \\ cbz wzr, 1f
7747 \\8:
7748 \\ mov x16, x0
7749 \\0:
7750 \\ ldxr x0, [x1]
7751 \\ bic x17, x0, x16
7752 \\ stxr w15, x17, [x1]
7753 \\ cbnz w15, 0b
7754 \\1:
7755 : [ret] "={x0}" (-> u64),
7756 : [val] "{x0}" (val),
7757 [ptr] "{x1}" (ptr),
7758 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7759 : "w15", "w16", "w17", "memory"
7760 );
7761}
7762fn ldclr8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7763 @setRuntimeSafety(false);
7764 __init_aarch64_have_lse_atomics();
7765
7766 return asm volatile (
7767 \\ cbz w16, 8f
7768 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
7769 \\ cbz wzr, 1f
7770 \\8:
7771 \\ mov x16, x0
7772 \\0:
7773 \\ ldxr x0, [x1]
7774 \\ bic x17, x0, x16
7775 \\ stxr w15, x17, [x1]
7776 \\ cbnz w15, 0b
7777 \\1:
7778 : [ret] "={x0}" (-> u64),
7779 : [val] "{x0}" (val),
7780 [ptr] "{x1}" (ptr),
7781 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7782 : "w15", "w16", "w17", "memory"
7783 );
7784}
7785const __aarch64_ldclr8_relax = if (is_darwin)
7786 if (has_lse)
7787 ldclr8RelaxDarwinLse
7788 else
7789 ldclr8RelaxDarwinNolse
7790else if (has_lse)
7791 ldclr8RelaxNondarwinLse
7792else
7793 ldclr8RelaxNondarwinNolse;
7794fn ldeor8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7795 @setRuntimeSafety(false);
7796 __init_aarch64_have_lse_atomics();
7797
7798 return asm volatile (
7799 \\ cbz w16, 8f
7800 \\ ldeor x0, x0, [x1]
7801 \\ cbz wzr, 1f
7802 \\8:
7803 \\ mov x16, x0
7804 \\0:
7805 \\ ldxr x0, [x1]
7806 \\ eor x17, x0, x16
7807 \\ stxr w15, x17, [x1]
7808 \\ cbnz w15, 0b
7809 \\1:
7810 : [ret] "={x0}" (-> u64),
7811 : [val] "{x0}" (val),
7812 [ptr] "{x1}" (ptr),
7813 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7814 : "w15", "w16", "w17", "memory"
7815 );
7816}
7817fn ldeor8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7818 @setRuntimeSafety(false);
7819 __init_aarch64_have_lse_atomics();
7820
7821 return asm volatile (
7822 \\ cbz w16, 8f
7823 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
7824 \\ cbz wzr, 1f
7825 \\8:
7826 \\ mov x16, x0
7827 \\0:
7828 \\ ldxr x0, [x1]
7829 \\ eor x17, x0, x16
7830 \\ stxr w15, x17, [x1]
7831 \\ cbnz w15, 0b
7832 \\1:
7833 : [ret] "={x0}" (-> u64),
7834 : [val] "{x0}" (val),
7835 [ptr] "{x1}" (ptr),
7836 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7837 : "w15", "w16", "w17", "memory"
7838 );
7839}
7840fn ldeor8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7841 @setRuntimeSafety(false);
7842 __init_aarch64_have_lse_atomics();
7843
7844 return asm volatile (
7845 \\ cbz w16, 8f
7846 \\ ldeor x0, x0, [x1]
7847 \\ cbz wzr, 1f
7848 \\8:
7849 \\ mov x16, x0
7850 \\0:
7851 \\ ldxr x0, [x1]
7852 \\ eor x17, x0, x16
7853 \\ stxr w15, x17, [x1]
7854 \\ cbnz w15, 0b
7855 \\1:
7856 : [ret] "={x0}" (-> u64),
7857 : [val] "{x0}" (val),
7858 [ptr] "{x1}" (ptr),
7859 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7860 : "w15", "w16", "w17", "memory"
7861 );
7862}
7863fn ldeor8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7864 @setRuntimeSafety(false);
7865 __init_aarch64_have_lse_atomics();
7866
7867 return asm volatile (
7868 \\ cbz w16, 8f
7869 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
7870 \\ cbz wzr, 1f
7871 \\8:
7872 \\ mov x16, x0
7873 \\0:
7874 \\ ldxr x0, [x1]
7875 \\ eor x17, x0, x16
7876 \\ stxr w15, x17, [x1]
7877 \\ cbnz w15, 0b
7878 \\1:
7879 : [ret] "={x0}" (-> u64),
7880 : [val] "{x0}" (val),
7881 [ptr] "{x1}" (ptr),
7882 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7883 : "w15", "w16", "w17", "memory"
7884 );
7885}
7886const __aarch64_ldeor8_relax = if (is_darwin)
7887 if (has_lse)
7888 ldeor8RelaxDarwinLse
7889 else
7890 ldeor8RelaxDarwinNolse
7891else if (has_lse)
7892 ldeor8RelaxNondarwinLse
7893else
7894 ldeor8RelaxNondarwinNolse;
7895fn ldset8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7896 @setRuntimeSafety(false);
7897 __init_aarch64_have_lse_atomics();
7898
7899 return asm volatile (
7900 \\ cbz w16, 8f
7901 \\ ldset x0, x0, [x1]
7902 \\ cbz wzr, 1f
7903 \\8:
7904 \\ mov x16, x0
7905 \\0:
7906 \\ ldxr x0, [x1]
7907 \\ orr x17, x0, x16
7908 \\ stxr w15, x17, [x1]
7909 \\ cbnz w15, 0b
7910 \\1:
7911 : [ret] "={x0}" (-> u64),
7912 : [val] "{x0}" (val),
7913 [ptr] "{x1}" (ptr),
7914 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7915 : "w15", "w16", "w17", "memory"
7916 );
7917}
7918fn ldset8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7919 @setRuntimeSafety(false);
7920 __init_aarch64_have_lse_atomics();
7921
7922 return asm volatile (
7923 \\ cbz w16, 8f
7924 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
7925 \\ cbz wzr, 1f
7926 \\8:
7927 \\ mov x16, x0
7928 \\0:
7929 \\ ldxr x0, [x1]
7930 \\ orr x17, x0, x16
7931 \\ stxr w15, x17, [x1]
7932 \\ cbnz w15, 0b
7933 \\1:
7934 : [ret] "={x0}" (-> u64),
7935 : [val] "{x0}" (val),
7936 [ptr] "{x1}" (ptr),
7937 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7938 : "w15", "w16", "w17", "memory"
7939 );
7940}
7941fn ldset8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7942 @setRuntimeSafety(false);
7943 __init_aarch64_have_lse_atomics();
7944
7945 return asm volatile (
7946 \\ cbz w16, 8f
7947 \\ ldset x0, x0, [x1]
7948 \\ cbz wzr, 1f
7949 \\8:
7950 \\ mov x16, x0
7951 \\0:
7952 \\ ldxr x0, [x1]
7953 \\ orr x17, x0, x16
7954 \\ stxr w15, x17, [x1]
7955 \\ cbnz w15, 0b
7956 \\1:
7957 : [ret] "={x0}" (-> u64),
7958 : [val] "{x0}" (val),
7959 [ptr] "{x1}" (ptr),
7960 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7961 : "w15", "w16", "w17", "memory"
7962 );
7963}
7964fn ldset8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7965 @setRuntimeSafety(false);
7966 __init_aarch64_have_lse_atomics();
7967
7968 return asm volatile (
7969 \\ cbz w16, 8f
7970 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
7971 \\ cbz wzr, 1f
7972 \\8:
7973 \\ mov x16, x0
7974 \\0:
7975 \\ ldxr x0, [x1]
7976 \\ orr x17, x0, x16
7977 \\ stxr w15, x17, [x1]
7978 \\ cbnz w15, 0b
7979 \\1:
7980 : [ret] "={x0}" (-> u64),
7981 : [val] "{x0}" (val),
7982 [ptr] "{x1}" (ptr),
7983 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7984 : "w15", "w16", "w17", "memory"
7985 );
7986}
7987const __aarch64_ldset8_relax = if (is_darwin)
7988 if (has_lse)
7989 ldset8RelaxDarwinLse
7990 else
7991 ldset8RelaxDarwinNolse
7992else if (has_lse)
7993 ldset8RelaxNondarwinLse
7994else
7995 ldset8RelaxNondarwinNolse;
7996fn cas8AcqDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7997 @setRuntimeSafety(false);
7998 __init_aarch64_have_lse_atomics();
7999
8000 return asm volatile (
8001 \\ cbz w16, 8f
8002 \\ casa x0, x1, [x2]
8003 \\
8004 \\ cbz wzr, 1f
8005 \\8:
8006 \\ mov x16, x0
8007 \\0:
8008 \\ ldaxr x0, [x2]
8009 \\ cmp x0, x16
8010 \\ bne 1f
8011 \\ stxr w17, x1, [x2]
8012 \\ cbnz w17, 0b
8013 \\1:
8014 : [ret] "={x0}" (-> u64),
8015 : [expected] "{x0}" (expected),
8016 [desired] "{x1}" (desired),
8017 [ptr] "{x2}" (ptr),
8018 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8019 : "w15", "w16", "w17", "memory"
8020 );
8021}
8022fn cas8AcqDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8023 @setRuntimeSafety(false);
8024 __init_aarch64_have_lse_atomics();
8025
8026 return asm volatile (
8027 \\ cbz w16, 8f
8028 \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
8029 \\
8030 \\ cbz wzr, 1f
8031 \\8:
8032 \\ mov x16, x0
8033 \\0:
8034 \\ ldaxr x0, [x2]
8035 \\ cmp x0, x16
8036 \\ bne 1f
8037 \\ stxr w17, x1, [x2]
8038 \\ cbnz w17, 0b
8039 \\1:
8040 : [ret] "={x0}" (-> u64),
8041 : [expected] "{x0}" (expected),
8042 [desired] "{x1}" (desired),
8043 [ptr] "{x2}" (ptr),
8044 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8045 : "w15", "w16", "w17", "memory"
8046 );
8047}
8048fn cas8AcqNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8049 @setRuntimeSafety(false);
8050 __init_aarch64_have_lse_atomics();
8051
8052 return asm volatile (
8053 \\ cbz w16, 8f
8054 \\ casa x0, x1, [x2]
8055 \\
8056 \\ cbz wzr, 1f
8057 \\8:
8058 \\ mov x16, x0
8059 \\0:
8060 \\ ldaxr x0, [x2]
8061 \\ cmp x0, x16
8062 \\ bne 1f
8063 \\ stxr w17, x1, [x2]
8064 \\ cbnz w17, 0b
8065 \\1:
8066 : [ret] "={x0}" (-> u64),
8067 : [expected] "{x0}" (expected),
8068 [desired] "{x1}" (desired),
8069 [ptr] "{x2}" (ptr),
8070 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8071 : "w15", "w16", "w17", "memory"
8072 );
8073}
8074fn cas8AcqNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8075 @setRuntimeSafety(false);
8076 __init_aarch64_have_lse_atomics();
8077
8078 return asm volatile (
8079 \\ cbz w16, 8f
8080 \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
8081 \\
8082 \\ cbz wzr, 1f
8083 \\8:
8084 \\ mov x16, x0
8085 \\0:
8086 \\ ldaxr x0, [x2]
8087 \\ cmp x0, x16
8088 \\ bne 1f
8089 \\ stxr w17, x1, [x2]
8090 \\ cbnz w17, 0b
8091 \\1:
8092 : [ret] "={x0}" (-> u64),
8093 : [expected] "{x0}" (expected),
8094 [desired] "{x1}" (desired),
8095 [ptr] "{x2}" (ptr),
8096 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8097 : "w15", "w16", "w17", "memory"
8098 );
8099}
8100const __aarch64_cas8_acq = if (is_darwin)
8101 if (has_lse)
8102 cas8AcqDarwinLse
8103 else
8104 cas8AcqDarwinNolse
8105else if (has_lse)
8106 cas8AcqNondarwinLse
8107else
8108 cas8AcqNondarwinNolse;
8109fn swp8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8110 @setRuntimeSafety(false);
8111 __init_aarch64_have_lse_atomics();
8112
8113 return asm volatile (
8114 \\ cbz w16, 8f
8115 \\ swpa x0, x0, [x1]
8116 \\ cbz wzr, 1f
8117 \\8:
8118 \\ mov x16, x0
8119 \\0:
8120 \\ ldaxr x0, [x1]
8121 \\ stxr w17, x16, [x1]
8122 \\ cbnz w17, 0b
8123 \\1:
8124 : [ret] "={x0}" (-> u64),
8125 : [val] "{x0}" (val),
8126 [ptr] "{x1}" (ptr),
8127 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8128 : "w15", "w16", "w17", "memory"
8129 );
8130}
8131fn swp8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8132 @setRuntimeSafety(false);
8133 __init_aarch64_have_lse_atomics();
8134
8135 return asm volatile (
8136 \\ cbz w16, 8f
8137 \\ .inst 0x38208020 + 0xc0000000 + 0x800000
8138 \\ cbz wzr, 1f
8139 \\8:
8140 \\ mov x16, x0
8141 \\0:
8142 \\ ldaxr x0, [x1]
8143 \\ stxr w17, x16, [x1]
8144 \\ cbnz w17, 0b
8145 \\1:
8146 : [ret] "={x0}" (-> u64),
8147 : [val] "{x0}" (val),
8148 [ptr] "{x1}" (ptr),
8149 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8150 : "w15", "w16", "w17", "memory"
8151 );
8152}
8153fn swp8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8154 @setRuntimeSafety(false);
8155 __init_aarch64_have_lse_atomics();
8156
8157 return asm volatile (
8158 \\ cbz w16, 8f
8159 \\ swpa x0, x0, [x1]
8160 \\ cbz wzr, 1f
8161 \\8:
8162 \\ mov x16, x0
8163 \\0:
8164 \\ ldaxr x0, [x1]
8165 \\ stxr w17, x16, [x1]
8166 \\ cbnz w17, 0b
8167 \\1:
8168 : [ret] "={x0}" (-> u64),
8169 : [val] "{x0}" (val),
8170 [ptr] "{x1}" (ptr),
8171 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8172 : "w15", "w16", "w17", "memory"
8173 );
8174}
8175fn swp8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8176 @setRuntimeSafety(false);
8177 __init_aarch64_have_lse_atomics();
8178
8179 return asm volatile (
8180 \\ cbz w16, 8f
8181 \\ .inst 0x38208020 + 0xc0000000 + 0x800000
8182 \\ cbz wzr, 1f
8183 \\8:
8184 \\ mov x16, x0
8185 \\0:
8186 \\ ldaxr x0, [x1]
8187 \\ stxr w17, x16, [x1]
8188 \\ cbnz w17, 0b
8189 \\1:
8190 : [ret] "={x0}" (-> u64),
8191 : [val] "{x0}" (val),
8192 [ptr] "{x1}" (ptr),
8193 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8194 : "w15", "w16", "w17", "memory"
8195 );
8196}
8197const __aarch64_swp8_acq = if (is_darwin)
8198 if (has_lse)
8199 swp8AcqDarwinLse
8200 else
8201 swp8AcqDarwinNolse
8202else if (has_lse)
8203 swp8AcqNondarwinLse
8204else
8205 swp8AcqNondarwinNolse;
8206fn ldadd8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8207 @setRuntimeSafety(false);
8208 __init_aarch64_have_lse_atomics();
8209
8210 return asm volatile (
8211 \\ cbz w16, 8f
8212 \\ ldadda x0, x0, [x1]
8213 \\ cbz wzr, 1f
8214 \\8:
8215 \\ mov x16, x0
8216 \\0:
8217 \\ ldaxr x0, [x1]
8218 \\ add x17, x0, x16
8219 \\ stxr w15, x17, [x1]
8220 \\ cbnz w15, 0b
8221 \\1:
8222 : [ret] "={x0}" (-> u64),
8223 : [val] "{x0}" (val),
8224 [ptr] "{x1}" (ptr),
8225 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8226 : "w15", "w16", "w17", "memory"
8227 );
8228}
8229fn ldadd8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8230 @setRuntimeSafety(false);
8231 __init_aarch64_have_lse_atomics();
8232
8233 return asm volatile (
8234 \\ cbz w16, 8f
8235 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
8236 \\ cbz wzr, 1f
8237 \\8:
8238 \\ mov x16, x0
8239 \\0:
8240 \\ ldaxr x0, [x1]
8241 \\ add x17, x0, x16
8242 \\ stxr w15, x17, [x1]
8243 \\ cbnz w15, 0b
8244 \\1:
8245 : [ret] "={x0}" (-> u64),
8246 : [val] "{x0}" (val),
8247 [ptr] "{x1}" (ptr),
8248 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8249 : "w15", "w16", "w17", "memory"
8250 );
8251}
8252fn ldadd8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8253 @setRuntimeSafety(false);
8254 __init_aarch64_have_lse_atomics();
8255
8256 return asm volatile (
8257 \\ cbz w16, 8f
8258 \\ ldadda x0, x0, [x1]
8259 \\ cbz wzr, 1f
8260 \\8:
8261 \\ mov x16, x0
8262 \\0:
8263 \\ ldaxr x0, [x1]
8264 \\ add x17, x0, x16
8265 \\ stxr w15, x17, [x1]
8266 \\ cbnz w15, 0b
8267 \\1:
8268 : [ret] "={x0}" (-> u64),
8269 : [val] "{x0}" (val),
8270 [ptr] "{x1}" (ptr),
8271 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8272 : "w15", "w16", "w17", "memory"
8273 );
8274}
8275fn ldadd8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8276 @setRuntimeSafety(false);
8277 __init_aarch64_have_lse_atomics();
8278
8279 return asm volatile (
8280 \\ cbz w16, 8f
8281 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
8282 \\ cbz wzr, 1f
8283 \\8:
8284 \\ mov x16, x0
8285 \\0:
8286 \\ ldaxr x0, [x1]
8287 \\ add x17, x0, x16
8288 \\ stxr w15, x17, [x1]
8289 \\ cbnz w15, 0b
8290 \\1:
8291 : [ret] "={x0}" (-> u64),
8292 : [val] "{x0}" (val),
8293 [ptr] "{x1}" (ptr),
8294 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8295 : "w15", "w16", "w17", "memory"
8296 );
8297}
8298const __aarch64_ldadd8_acq = if (is_darwin)
8299 if (has_lse)
8300 ldadd8AcqDarwinLse
8301 else
8302 ldadd8AcqDarwinNolse
8303else if (has_lse)
8304 ldadd8AcqNondarwinLse
8305else
8306 ldadd8AcqNondarwinNolse;
8307fn ldclr8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8308 @setRuntimeSafety(false);
8309 __init_aarch64_have_lse_atomics();
8310
8311 return asm volatile (
8312 \\ cbz w16, 8f
8313 \\ ldclra x0, x0, [x1]
8314 \\ cbz wzr, 1f
8315 \\8:
8316 \\ mov x16, x0
8317 \\0:
8318 \\ ldaxr x0, [x1]
8319 \\ bic x17, x0, x16
8320 \\ stxr w15, x17, [x1]
8321 \\ cbnz w15, 0b
8322 \\1:
8323 : [ret] "={x0}" (-> u64),
8324 : [val] "{x0}" (val),
8325 [ptr] "{x1}" (ptr),
8326 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8327 : "w15", "w16", "w17", "memory"
8328 );
8329}
8330fn ldclr8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8331 @setRuntimeSafety(false);
8332 __init_aarch64_have_lse_atomics();
8333
8334 return asm volatile (
8335 \\ cbz w16, 8f
8336 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
8337 \\ cbz wzr, 1f
8338 \\8:
8339 \\ mov x16, x0
8340 \\0:
8341 \\ ldaxr x0, [x1]
8342 \\ bic x17, x0, x16
8343 \\ stxr w15, x17, [x1]
8344 \\ cbnz w15, 0b
8345 \\1:
8346 : [ret] "={x0}" (-> u64),
8347 : [val] "{x0}" (val),
8348 [ptr] "{x1}" (ptr),
8349 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8350 : "w15", "w16", "w17", "memory"
8351 );
8352}
8353fn ldclr8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8354 @setRuntimeSafety(false);
8355 __init_aarch64_have_lse_atomics();
8356
8357 return asm volatile (
8358 \\ cbz w16, 8f
8359 \\ ldclra x0, x0, [x1]
8360 \\ cbz wzr, 1f
8361 \\8:
8362 \\ mov x16, x0
8363 \\0:
8364 \\ ldaxr x0, [x1]
8365 \\ bic x17, x0, x16
8366 \\ stxr w15, x17, [x1]
8367 \\ cbnz w15, 0b
8368 \\1:
8369 : [ret] "={x0}" (-> u64),
8370 : [val] "{x0}" (val),
8371 [ptr] "{x1}" (ptr),
8372 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8373 : "w15", "w16", "w17", "memory"
8374 );
8375}
8376fn ldclr8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8377 @setRuntimeSafety(false);
8378 __init_aarch64_have_lse_atomics();
8379
8380 return asm volatile (
8381 \\ cbz w16, 8f
8382 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
8383 \\ cbz wzr, 1f
8384 \\8:
8385 \\ mov x16, x0
8386 \\0:
8387 \\ ldaxr x0, [x1]
8388 \\ bic x17, x0, x16
8389 \\ stxr w15, x17, [x1]
8390 \\ cbnz w15, 0b
8391 \\1:
8392 : [ret] "={x0}" (-> u64),
8393 : [val] "{x0}" (val),
8394 [ptr] "{x1}" (ptr),
8395 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8396 : "w15", "w16", "w17", "memory"
8397 );
8398}
8399const __aarch64_ldclr8_acq = if (is_darwin)
8400 if (has_lse)
8401 ldclr8AcqDarwinLse
8402 else
8403 ldclr8AcqDarwinNolse
8404else if (has_lse)
8405 ldclr8AcqNondarwinLse
8406else
8407 ldclr8AcqNondarwinNolse;
8408fn ldeor8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8409 @setRuntimeSafety(false);
8410 __init_aarch64_have_lse_atomics();
8411
8412 return asm volatile (
8413 \\ cbz w16, 8f
8414 \\ ldeora x0, x0, [x1]
8415 \\ cbz wzr, 1f
8416 \\8:
8417 \\ mov x16, x0
8418 \\0:
8419 \\ ldaxr x0, [x1]
8420 \\ eor x17, x0, x16
8421 \\ stxr w15, x17, [x1]
8422 \\ cbnz w15, 0b
8423 \\1:
8424 : [ret] "={x0}" (-> u64),
8425 : [val] "{x0}" (val),
8426 [ptr] "{x1}" (ptr),
8427 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8428 : "w15", "w16", "w17", "memory"
8429 );
8430}
8431fn ldeor8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8432 @setRuntimeSafety(false);
8433 __init_aarch64_have_lse_atomics();
8434
8435 return asm volatile (
8436 \\ cbz w16, 8f
8437 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
8438 \\ cbz wzr, 1f
8439 \\8:
8440 \\ mov x16, x0
8441 \\0:
8442 \\ ldaxr x0, [x1]
8443 \\ eor x17, x0, x16
8444 \\ stxr w15, x17, [x1]
8445 \\ cbnz w15, 0b
8446 \\1:
8447 : [ret] "={x0}" (-> u64),
8448 : [val] "{x0}" (val),
8449 [ptr] "{x1}" (ptr),
8450 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8451 : "w15", "w16", "w17", "memory"
8452 );
8453}
8454fn ldeor8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8455 @setRuntimeSafety(false);
8456 __init_aarch64_have_lse_atomics();
8457
8458 return asm volatile (
8459 \\ cbz w16, 8f
8460 \\ ldeora x0, x0, [x1]
8461 \\ cbz wzr, 1f
8462 \\8:
8463 \\ mov x16, x0
8464 \\0:
8465 \\ ldaxr x0, [x1]
8466 \\ eor x17, x0, x16
8467 \\ stxr w15, x17, [x1]
8468 \\ cbnz w15, 0b
8469 \\1:
8470 : [ret] "={x0}" (-> u64),
8471 : [val] "{x0}" (val),
8472 [ptr] "{x1}" (ptr),
8473 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8474 : "w15", "w16", "w17", "memory"
8475 );
8476}
8477fn ldeor8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8478 @setRuntimeSafety(false);
8479 __init_aarch64_have_lse_atomics();
8480
8481 return asm volatile (
8482 \\ cbz w16, 8f
8483 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
8484 \\ cbz wzr, 1f
8485 \\8:
8486 \\ mov x16, x0
8487 \\0:
8488 \\ ldaxr x0, [x1]
8489 \\ eor x17, x0, x16
8490 \\ stxr w15, x17, [x1]
8491 \\ cbnz w15, 0b
8492 \\1:
8493 : [ret] "={x0}" (-> u64),
8494 : [val] "{x0}" (val),
8495 [ptr] "{x1}" (ptr),
8496 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8497 : "w15", "w16", "w17", "memory"
8498 );
8499}
8500const __aarch64_ldeor8_acq = if (is_darwin)
8501 if (has_lse)
8502 ldeor8AcqDarwinLse
8503 else
8504 ldeor8AcqDarwinNolse
8505else if (has_lse)
8506 ldeor8AcqNondarwinLse
8507else
8508 ldeor8AcqNondarwinNolse;
8509fn ldset8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8510 @setRuntimeSafety(false);
8511 __init_aarch64_have_lse_atomics();
8512
8513 return asm volatile (
8514 \\ cbz w16, 8f
8515 \\ ldseta x0, x0, [x1]
8516 \\ cbz wzr, 1f
8517 \\8:
8518 \\ mov x16, x0
8519 \\0:
8520 \\ ldaxr x0, [x1]
8521 \\ orr x17, x0, x16
8522 \\ stxr w15, x17, [x1]
8523 \\ cbnz w15, 0b
8524 \\1:
8525 : [ret] "={x0}" (-> u64),
8526 : [val] "{x0}" (val),
8527 [ptr] "{x1}" (ptr),
8528 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8529 : "w15", "w16", "w17", "memory"
8530 );
8531}
8532fn ldset8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8533 @setRuntimeSafety(false);
8534 __init_aarch64_have_lse_atomics();
8535
8536 return asm volatile (
8537 \\ cbz w16, 8f
8538 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
8539 \\ cbz wzr, 1f
8540 \\8:
8541 \\ mov x16, x0
8542 \\0:
8543 \\ ldaxr x0, [x1]
8544 \\ orr x17, x0, x16
8545 \\ stxr w15, x17, [x1]
8546 \\ cbnz w15, 0b
8547 \\1:
8548 : [ret] "={x0}" (-> u64),
8549 : [val] "{x0}" (val),
8550 [ptr] "{x1}" (ptr),
8551 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8552 : "w15", "w16", "w17", "memory"
8553 );
8554}
8555fn ldset8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8556 @setRuntimeSafety(false);
8557 __init_aarch64_have_lse_atomics();
8558
8559 return asm volatile (
8560 \\ cbz w16, 8f
8561 \\ ldseta x0, x0, [x1]
8562 \\ cbz wzr, 1f
8563 \\8:
8564 \\ mov x16, x0
8565 \\0:
8566 \\ ldaxr x0, [x1]
8567 \\ orr x17, x0, x16
8568 \\ stxr w15, x17, [x1]
8569 \\ cbnz w15, 0b
8570 \\1:
8571 : [ret] "={x0}" (-> u64),
8572 : [val] "{x0}" (val),
8573 [ptr] "{x1}" (ptr),
8574 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8575 : "w15", "w16", "w17", "memory"
8576 );
8577}
8578fn ldset8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8579 @setRuntimeSafety(false);
8580 __init_aarch64_have_lse_atomics();
8581
8582 return asm volatile (
8583 \\ cbz w16, 8f
8584 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
8585 \\ cbz wzr, 1f
8586 \\8:
8587 \\ mov x16, x0
8588 \\0:
8589 \\ ldaxr x0, [x1]
8590 \\ orr x17, x0, x16
8591 \\ stxr w15, x17, [x1]
8592 \\ cbnz w15, 0b
8593 \\1:
8594 : [ret] "={x0}" (-> u64),
8595 : [val] "{x0}" (val),
8596 [ptr] "{x1}" (ptr),
8597 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8598 : "w15", "w16", "w17", "memory"
8599 );
8600}
8601const __aarch64_ldset8_acq = if (is_darwin)
8602 if (has_lse)
8603 ldset8AcqDarwinLse
8604 else
8605 ldset8AcqDarwinNolse
8606else if (has_lse)
8607 ldset8AcqNondarwinLse
8608else
8609 ldset8AcqNondarwinNolse;
8610fn cas8RelDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8611 @setRuntimeSafety(false);
8612 __init_aarch64_have_lse_atomics();
8613
8614 return asm volatile (
8615 \\ cbz w16, 8f
8616 \\ casl x0, x1, [x2]
8617 \\
8618 \\ cbz wzr, 1f
8619 \\8:
8620 \\ mov x16, x0
8621 \\0:
8622 \\ ldxr x0, [x2]
8623 \\ cmp x0, x16
8624 \\ bne 1f
8625 \\ stlxr w17, x1, [x2]
8626 \\ cbnz w17, 0b
8627 \\1:
8628 : [ret] "={x0}" (-> u64),
8629 : [expected] "{x0}" (expected),
8630 [desired] "{x1}" (desired),
8631 [ptr] "{x2}" (ptr),
8632 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8633 : "w15", "w16", "w17", "memory"
8634 );
8635}
8636fn cas8RelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8637 @setRuntimeSafety(false);
8638 __init_aarch64_have_lse_atomics();
8639
8640 return asm volatile (
8641 \\ cbz w16, 8f
8642 \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
8643 \\
8644 \\ cbz wzr, 1f
8645 \\8:
8646 \\ mov x16, x0
8647 \\0:
8648 \\ ldxr x0, [x2]
8649 \\ cmp x0, x16
8650 \\ bne 1f
8651 \\ stlxr w17, x1, [x2]
8652 \\ cbnz w17, 0b
8653 \\1:
8654 : [ret] "={x0}" (-> u64),
8655 : [expected] "{x0}" (expected),
8656 [desired] "{x1}" (desired),
8657 [ptr] "{x2}" (ptr),
8658 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8659 : "w15", "w16", "w17", "memory"
8660 );
8661}
8662fn cas8RelNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8663 @setRuntimeSafety(false);
8664 __init_aarch64_have_lse_atomics();
8665
8666 return asm volatile (
8667 \\ cbz w16, 8f
8668 \\ casl x0, x1, [x2]
8669 \\
8670 \\ cbz wzr, 1f
8671 \\8:
8672 \\ mov x16, x0
8673 \\0:
8674 \\ ldxr x0, [x2]
8675 \\ cmp x0, x16
8676 \\ bne 1f
8677 \\ stlxr w17, x1, [x2]
8678 \\ cbnz w17, 0b
8679 \\1:
8680 : [ret] "={x0}" (-> u64),
8681 : [expected] "{x0}" (expected),
8682 [desired] "{x1}" (desired),
8683 [ptr] "{x2}" (ptr),
8684 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8685 : "w15", "w16", "w17", "memory"
8686 );
8687}
8688fn cas8RelNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8689 @setRuntimeSafety(false);
8690 __init_aarch64_have_lse_atomics();
8691
8692 return asm volatile (
8693 \\ cbz w16, 8f
8694 \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
8695 \\
8696 \\ cbz wzr, 1f
8697 \\8:
8698 \\ mov x16, x0
8699 \\0:
8700 \\ ldxr x0, [x2]
8701 \\ cmp x0, x16
8702 \\ bne 1f
8703 \\ stlxr w17, x1, [x2]
8704 \\ cbnz w17, 0b
8705 \\1:
8706 : [ret] "={x0}" (-> u64),
8707 : [expected] "{x0}" (expected),
8708 [desired] "{x1}" (desired),
8709 [ptr] "{x2}" (ptr),
8710 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8711 : "w15", "w16", "w17", "memory"
8712 );
8713}
8714const __aarch64_cas8_rel = if (is_darwin)
8715 if (has_lse)
8716 cas8RelDarwinLse
8717 else
8718 cas8RelDarwinNolse
8719else if (has_lse)
8720 cas8RelNondarwinLse
8721else
8722 cas8RelNondarwinNolse;
8723fn swp8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8724 @setRuntimeSafety(false);
8725 __init_aarch64_have_lse_atomics();
8726
8727 return asm volatile (
8728 \\ cbz w16, 8f
8729 \\ swpl x0, x0, [x1]
8730 \\ cbz wzr, 1f
8731 \\8:
8732 \\ mov x16, x0
8733 \\0:
8734 \\ ldxr x0, [x1]
8735 \\ stlxr w17, x16, [x1]
8736 \\ cbnz w17, 0b
8737 \\1:
8738 : [ret] "={x0}" (-> u64),
8739 : [val] "{x0}" (val),
8740 [ptr] "{x1}" (ptr),
8741 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8742 : "w15", "w16", "w17", "memory"
8743 );
8744}
8745fn swp8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8746 @setRuntimeSafety(false);
8747 __init_aarch64_have_lse_atomics();
8748
8749 return asm volatile (
8750 \\ cbz w16, 8f
8751 \\ .inst 0x38208020 + 0xc0000000 + 0x400000
8752 \\ cbz wzr, 1f
8753 \\8:
8754 \\ mov x16, x0
8755 \\0:
8756 \\ ldxr x0, [x1]
8757 \\ stlxr w17, x16, [x1]
8758 \\ cbnz w17, 0b
8759 \\1:
8760 : [ret] "={x0}" (-> u64),
8761 : [val] "{x0}" (val),
8762 [ptr] "{x1}" (ptr),
8763 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8764 : "w15", "w16", "w17", "memory"
8765 );
8766}
8767fn swp8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8768 @setRuntimeSafety(false);
8769 __init_aarch64_have_lse_atomics();
8770
8771 return asm volatile (
8772 \\ cbz w16, 8f
8773 \\ swpl x0, x0, [x1]
8774 \\ cbz wzr, 1f
8775 \\8:
8776 \\ mov x16, x0
8777 \\0:
8778 \\ ldxr x0, [x1]
8779 \\ stlxr w17, x16, [x1]
8780 \\ cbnz w17, 0b
8781 \\1:
8782 : [ret] "={x0}" (-> u64),
8783 : [val] "{x0}" (val),
8784 [ptr] "{x1}" (ptr),
8785 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8786 : "w15", "w16", "w17", "memory"
8787 );
8788}
8789fn swp8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8790 @setRuntimeSafety(false);
8791 __init_aarch64_have_lse_atomics();
8792
8793 return asm volatile (
8794 \\ cbz w16, 8f
8795 \\ .inst 0x38208020 + 0xc0000000 + 0x400000
8796 \\ cbz wzr, 1f
8797 \\8:
8798 \\ mov x16, x0
8799 \\0:
8800 \\ ldxr x0, [x1]
8801 \\ stlxr w17, x16, [x1]
8802 \\ cbnz w17, 0b
8803 \\1:
8804 : [ret] "={x0}" (-> u64),
8805 : [val] "{x0}" (val),
8806 [ptr] "{x1}" (ptr),
8807 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8808 : "w15", "w16", "w17", "memory"
8809 );
8810}
8811const __aarch64_swp8_rel = if (is_darwin)
8812 if (has_lse)
8813 swp8RelDarwinLse
8814 else
8815 swp8RelDarwinNolse
8816else if (has_lse)
8817 swp8RelNondarwinLse
8818else
8819 swp8RelNondarwinNolse;
8820fn ldadd8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8821 @setRuntimeSafety(false);
8822 __init_aarch64_have_lse_atomics();
8823
8824 return asm volatile (
8825 \\ cbz w16, 8f
8826 \\ ldaddl x0, x0, [x1]
8827 \\ cbz wzr, 1f
8828 \\8:
8829 \\ mov x16, x0
8830 \\0:
8831 \\ ldxr x0, [x1]
8832 \\ add x17, x0, x16
8833 \\ stlxr w15, x17, [x1]
8834 \\ cbnz w15, 0b
8835 \\1:
8836 : [ret] "={x0}" (-> u64),
8837 : [val] "{x0}" (val),
8838 [ptr] "{x1}" (ptr),
8839 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8840 : "w15", "w16", "w17", "memory"
8841 );
8842}
8843fn ldadd8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8844 @setRuntimeSafety(false);
8845 __init_aarch64_have_lse_atomics();
8846
8847 return asm volatile (
8848 \\ cbz w16, 8f
8849 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
8850 \\ cbz wzr, 1f
8851 \\8:
8852 \\ mov x16, x0
8853 \\0:
8854 \\ ldxr x0, [x1]
8855 \\ add x17, x0, x16
8856 \\ stlxr w15, x17, [x1]
8857 \\ cbnz w15, 0b
8858 \\1:
8859 : [ret] "={x0}" (-> u64),
8860 : [val] "{x0}" (val),
8861 [ptr] "{x1}" (ptr),
8862 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8863 : "w15", "w16", "w17", "memory"
8864 );
8865}
8866fn ldadd8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8867 @setRuntimeSafety(false);
8868 __init_aarch64_have_lse_atomics();
8869
8870 return asm volatile (
8871 \\ cbz w16, 8f
8872 \\ ldaddl x0, x0, [x1]
8873 \\ cbz wzr, 1f
8874 \\8:
8875 \\ mov x16, x0
8876 \\0:
8877 \\ ldxr x0, [x1]
8878 \\ add x17, x0, x16
8879 \\ stlxr w15, x17, [x1]
8880 \\ cbnz w15, 0b
8881 \\1:
8882 : [ret] "={x0}" (-> u64),
8883 : [val] "{x0}" (val),
8884 [ptr] "{x1}" (ptr),
8885 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8886 : "w15", "w16", "w17", "memory"
8887 );
8888}
8889fn ldadd8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8890 @setRuntimeSafety(false);
8891 __init_aarch64_have_lse_atomics();
8892
8893 return asm volatile (
8894 \\ cbz w16, 8f
8895 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
8896 \\ cbz wzr, 1f
8897 \\8:
8898 \\ mov x16, x0
8899 \\0:
8900 \\ ldxr x0, [x1]
8901 \\ add x17, x0, x16
8902 \\ stlxr w15, x17, [x1]
8903 \\ cbnz w15, 0b
8904 \\1:
8905 : [ret] "={x0}" (-> u64),
8906 : [val] "{x0}" (val),
8907 [ptr] "{x1}" (ptr),
8908 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8909 : "w15", "w16", "w17", "memory"
8910 );
8911}
8912const __aarch64_ldadd8_rel = if (is_darwin)
8913 if (has_lse)
8914 ldadd8RelDarwinLse
8915 else
8916 ldadd8RelDarwinNolse
8917else if (has_lse)
8918 ldadd8RelNondarwinLse
8919else
8920 ldadd8RelNondarwinNolse;
8921fn ldclr8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8922 @setRuntimeSafety(false);
8923 __init_aarch64_have_lse_atomics();
8924
8925 return asm volatile (
8926 \\ cbz w16, 8f
8927 \\ ldclrl x0, x0, [x1]
8928 \\ cbz wzr, 1f
8929 \\8:
8930 \\ mov x16, x0
8931 \\0:
8932 \\ ldxr x0, [x1]
8933 \\ bic x17, x0, x16
8934 \\ stlxr w15, x17, [x1]
8935 \\ cbnz w15, 0b
8936 \\1:
8937 : [ret] "={x0}" (-> u64),
8938 : [val] "{x0}" (val),
8939 [ptr] "{x1}" (ptr),
8940 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8941 : "w15", "w16", "w17", "memory"
8942 );
8943}
8944fn ldclr8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8945 @setRuntimeSafety(false);
8946 __init_aarch64_have_lse_atomics();
8947
8948 return asm volatile (
8949 \\ cbz w16, 8f
8950 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
8951 \\ cbz wzr, 1f
8952 \\8:
8953 \\ mov x16, x0
8954 \\0:
8955 \\ ldxr x0, [x1]
8956 \\ bic x17, x0, x16
8957 \\ stlxr w15, x17, [x1]
8958 \\ cbnz w15, 0b
8959 \\1:
8960 : [ret] "={x0}" (-> u64),
8961 : [val] "{x0}" (val),
8962 [ptr] "{x1}" (ptr),
8963 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8964 : "w15", "w16", "w17", "memory"
8965 );
8966}
8967fn ldclr8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8968 @setRuntimeSafety(false);
8969 __init_aarch64_have_lse_atomics();
8970
8971 return asm volatile (
8972 \\ cbz w16, 8f
8973 \\ ldclrl x0, x0, [x1]
8974 \\ cbz wzr, 1f
8975 \\8:
8976 \\ mov x16, x0
8977 \\0:
8978 \\ ldxr x0, [x1]
8979 \\ bic x17, x0, x16
8980 \\ stlxr w15, x17, [x1]
8981 \\ cbnz w15, 0b
8982 \\1:
8983 : [ret] "={x0}" (-> u64),
8984 : [val] "{x0}" (val),
8985 [ptr] "{x1}" (ptr),
8986 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8987 : "w15", "w16", "w17", "memory"
8988 );
8989}
8990fn ldclr8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8991 @setRuntimeSafety(false);
8992 __init_aarch64_have_lse_atomics();
8993
8994 return asm volatile (
8995 \\ cbz w16, 8f
8996 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
8997 \\ cbz wzr, 1f
8998 \\8:
8999 \\ mov x16, x0
9000 \\0:
9001 \\ ldxr x0, [x1]
9002 \\ bic x17, x0, x16
9003 \\ stlxr w15, x17, [x1]
9004 \\ cbnz w15, 0b
9005 \\1:
9006 : [ret] "={x0}" (-> u64),
9007 : [val] "{x0}" (val),
9008 [ptr] "{x1}" (ptr),
9009 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9010 : "w15", "w16", "w17", "memory"
9011 );
9012}
9013const __aarch64_ldclr8_rel = if (is_darwin)
9014 if (has_lse)
9015 ldclr8RelDarwinLse
9016 else
9017 ldclr8RelDarwinNolse
9018else if (has_lse)
9019 ldclr8RelNondarwinLse
9020else
9021 ldclr8RelNondarwinNolse;
9022fn ldeor8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9023 @setRuntimeSafety(false);
9024 __init_aarch64_have_lse_atomics();
9025
9026 return asm volatile (
9027 \\ cbz w16, 8f
9028 \\ ldeorl x0, x0, [x1]
9029 \\ cbz wzr, 1f
9030 \\8:
9031 \\ mov x16, x0
9032 \\0:
9033 \\ ldxr x0, [x1]
9034 \\ eor x17, x0, x16
9035 \\ stlxr w15, x17, [x1]
9036 \\ cbnz w15, 0b
9037 \\1:
9038 : [ret] "={x0}" (-> u64),
9039 : [val] "{x0}" (val),
9040 [ptr] "{x1}" (ptr),
9041 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9042 : "w15", "w16", "w17", "memory"
9043 );
9044}
9045fn ldeor8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9046 @setRuntimeSafety(false);
9047 __init_aarch64_have_lse_atomics();
9048
9049 return asm volatile (
9050 \\ cbz w16, 8f
9051 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
9052 \\ cbz wzr, 1f
9053 \\8:
9054 \\ mov x16, x0
9055 \\0:
9056 \\ ldxr x0, [x1]
9057 \\ eor x17, x0, x16
9058 \\ stlxr w15, x17, [x1]
9059 \\ cbnz w15, 0b
9060 \\1:
9061 : [ret] "={x0}" (-> u64),
9062 : [val] "{x0}" (val),
9063 [ptr] "{x1}" (ptr),
9064 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9065 : "w15", "w16", "w17", "memory"
9066 );
9067}
9068fn ldeor8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9069 @setRuntimeSafety(false);
9070 __init_aarch64_have_lse_atomics();
9071
9072 return asm volatile (
9073 \\ cbz w16, 8f
9074 \\ ldeorl x0, x0, [x1]
9075 \\ cbz wzr, 1f
9076 \\8:
9077 \\ mov x16, x0
9078 \\0:
9079 \\ ldxr x0, [x1]
9080 \\ eor x17, x0, x16
9081 \\ stlxr w15, x17, [x1]
9082 \\ cbnz w15, 0b
9083 \\1:
9084 : [ret] "={x0}" (-> u64),
9085 : [val] "{x0}" (val),
9086 [ptr] "{x1}" (ptr),
9087 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9088 : "w15", "w16", "w17", "memory"
9089 );
9090}
9091fn ldeor8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9092 @setRuntimeSafety(false);
9093 __init_aarch64_have_lse_atomics();
9094
9095 return asm volatile (
9096 \\ cbz w16, 8f
9097 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
9098 \\ cbz wzr, 1f
9099 \\8:
9100 \\ mov x16, x0
9101 \\0:
9102 \\ ldxr x0, [x1]
9103 \\ eor x17, x0, x16
9104 \\ stlxr w15, x17, [x1]
9105 \\ cbnz w15, 0b
9106 \\1:
9107 : [ret] "={x0}" (-> u64),
9108 : [val] "{x0}" (val),
9109 [ptr] "{x1}" (ptr),
9110 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9111 : "w15", "w16", "w17", "memory"
9112 );
9113}
9114const __aarch64_ldeor8_rel = if (is_darwin)
9115 if (has_lse)
9116 ldeor8RelDarwinLse
9117 else
9118 ldeor8RelDarwinNolse
9119else if (has_lse)
9120 ldeor8RelNondarwinLse
9121else
9122 ldeor8RelNondarwinNolse;
9123fn ldset8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9124 @setRuntimeSafety(false);
9125 __init_aarch64_have_lse_atomics();
9126
9127 return asm volatile (
9128 \\ cbz w16, 8f
9129 \\ ldsetl x0, x0, [x1]
9130 \\ cbz wzr, 1f
9131 \\8:
9132 \\ mov x16, x0
9133 \\0:
9134 \\ ldxr x0, [x1]
9135 \\ orr x17, x0, x16
9136 \\ stlxr w15, x17, [x1]
9137 \\ cbnz w15, 0b
9138 \\1:
9139 : [ret] "={x0}" (-> u64),
9140 : [val] "{x0}" (val),
9141 [ptr] "{x1}" (ptr),
9142 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9143 : "w15", "w16", "w17", "memory"
9144 );
9145}
9146fn ldset8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9147 @setRuntimeSafety(false);
9148 __init_aarch64_have_lse_atomics();
9149
9150 return asm volatile (
9151 \\ cbz w16, 8f
9152 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
9153 \\ cbz wzr, 1f
9154 \\8:
9155 \\ mov x16, x0
9156 \\0:
9157 \\ ldxr x0, [x1]
9158 \\ orr x17, x0, x16
9159 \\ stlxr w15, x17, [x1]
9160 \\ cbnz w15, 0b
9161 \\1:
9162 : [ret] "={x0}" (-> u64),
9163 : [val] "{x0}" (val),
9164 [ptr] "{x1}" (ptr),
9165 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9166 : "w15", "w16", "w17", "memory"
9167 );
9168}
9169fn ldset8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9170 @setRuntimeSafety(false);
9171 __init_aarch64_have_lse_atomics();
9172
9173 return asm volatile (
9174 \\ cbz w16, 8f
9175 \\ ldsetl x0, x0, [x1]
9176 \\ cbz wzr, 1f
9177 \\8:
9178 \\ mov x16, x0
9179 \\0:
9180 \\ ldxr x0, [x1]
9181 \\ orr x17, x0, x16
9182 \\ stlxr w15, x17, [x1]
9183 \\ cbnz w15, 0b
9184 \\1:
9185 : [ret] "={x0}" (-> u64),
9186 : [val] "{x0}" (val),
9187 [ptr] "{x1}" (ptr),
9188 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9189 : "w15", "w16", "w17", "memory"
9190 );
9191}
9192fn ldset8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9193 @setRuntimeSafety(false);
9194 __init_aarch64_have_lse_atomics();
9195
9196 return asm volatile (
9197 \\ cbz w16, 8f
9198 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
9199 \\ cbz wzr, 1f
9200 \\8:
9201 \\ mov x16, x0
9202 \\0:
9203 \\ ldxr x0, [x1]
9204 \\ orr x17, x0, x16
9205 \\ stlxr w15, x17, [x1]
9206 \\ cbnz w15, 0b
9207 \\1:
9208 : [ret] "={x0}" (-> u64),
9209 : [val] "{x0}" (val),
9210 [ptr] "{x1}" (ptr),
9211 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9212 : "w15", "w16", "w17", "memory"
9213 );
9214}
9215const __aarch64_ldset8_rel = if (is_darwin)
9216 if (has_lse)
9217 ldset8RelDarwinLse
9218 else
9219 ldset8RelDarwinNolse
9220else if (has_lse)
9221 ldset8RelNondarwinLse
9222else
9223 ldset8RelNondarwinNolse;
9224fn cas8AcqRelDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9225 @setRuntimeSafety(false);
9226 __init_aarch64_have_lse_atomics();
9227
9228 return asm volatile (
9229 \\ cbz w16, 8f
9230 \\ casal x0, x1, [x2]
9231 \\
9232 \\ cbz wzr, 1f
9233 \\8:
9234 \\ mov x16, x0
9235 \\0:
9236 \\ ldaxr x0, [x2]
9237 \\ cmp x0, x16
9238 \\ bne 1f
9239 \\ stlxr w17, x1, [x2]
9240 \\ cbnz w17, 0b
9241 \\1:
9242 : [ret] "={x0}" (-> u64),
9243 : [expected] "{x0}" (expected),
9244 [desired] "{x1}" (desired),
9245 [ptr] "{x2}" (ptr),
9246 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9247 : "w15", "w16", "w17", "memory"
9248 );
9249}
9250fn cas8AcqRelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9251 @setRuntimeSafety(false);
9252 __init_aarch64_have_lse_atomics();
9253
9254 return asm volatile (
9255 \\ cbz w16, 8f
9256 \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
9257 \\
9258 \\ cbz wzr, 1f
9259 \\8:
9260 \\ mov x16, x0
9261 \\0:
9262 \\ ldaxr x0, [x2]
9263 \\ cmp x0, x16
9264 \\ bne 1f
9265 \\ stlxr w17, x1, [x2]
9266 \\ cbnz w17, 0b
9267 \\1:
9268 : [ret] "={x0}" (-> u64),
9269 : [expected] "{x0}" (expected),
9270 [desired] "{x1}" (desired),
9271 [ptr] "{x2}" (ptr),
9272 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9273 : "w15", "w16", "w17", "memory"
9274 );
9275}
9276fn cas8AcqRelNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9277 @setRuntimeSafety(false);
9278 __init_aarch64_have_lse_atomics();
9279
9280 return asm volatile (
9281 \\ cbz w16, 8f
9282 \\ casal x0, x1, [x2]
9283 \\
9284 \\ cbz wzr, 1f
9285 \\8:
9286 \\ mov x16, x0
9287 \\0:
9288 \\ ldaxr x0, [x2]
9289 \\ cmp x0, x16
9290 \\ bne 1f
9291 \\ stlxr w17, x1, [x2]
9292 \\ cbnz w17, 0b
9293 \\1:
9294 : [ret] "={x0}" (-> u64),
9295 : [expected] "{x0}" (expected),
9296 [desired] "{x1}" (desired),
9297 [ptr] "{x2}" (ptr),
9298 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9299 : "w15", "w16", "w17", "memory"
9300 );
9301}
9302fn cas8AcqRelNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9303 @setRuntimeSafety(false);
9304 __init_aarch64_have_lse_atomics();
9305
9306 return asm volatile (
9307 \\ cbz w16, 8f
9308 \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
9309 \\
9310 \\ cbz wzr, 1f
9311 \\8:
9312 \\ mov x16, x0
9313 \\0:
9314 \\ ldaxr x0, [x2]
9315 \\ cmp x0, x16
9316 \\ bne 1f
9317 \\ stlxr w17, x1, [x2]
9318 \\ cbnz w17, 0b
9319 \\1:
9320 : [ret] "={x0}" (-> u64),
9321 : [expected] "{x0}" (expected),
9322 [desired] "{x1}" (desired),
9323 [ptr] "{x2}" (ptr),
9324 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9325 : "w15", "w16", "w17", "memory"
9326 );
9327}
9328const __aarch64_cas8_acq_rel = if (is_darwin)
9329 if (has_lse)
9330 cas8AcqRelDarwinLse
9331 else
9332 cas8AcqRelDarwinNolse
9333else if (has_lse)
9334 cas8AcqRelNondarwinLse
9335else
9336 cas8AcqRelNondarwinNolse;
9337fn swp8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9338 @setRuntimeSafety(false);
9339 __init_aarch64_have_lse_atomics();
9340
9341 return asm volatile (
9342 \\ cbz w16, 8f
9343 \\ swpal x0, x0, [x1]
9344 \\ cbz wzr, 1f
9345 \\8:
9346 \\ mov x16, x0
9347 \\0:
9348 \\ ldaxr x0, [x1]
9349 \\ stlxr w17, x16, [x1]
9350 \\ cbnz w17, 0b
9351 \\1:
9352 : [ret] "={x0}" (-> u64),
9353 : [val] "{x0}" (val),
9354 [ptr] "{x1}" (ptr),
9355 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9356 : "w15", "w16", "w17", "memory"
9357 );
9358}
9359fn swp8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9360 @setRuntimeSafety(false);
9361 __init_aarch64_have_lse_atomics();
9362
9363 return asm volatile (
9364 \\ cbz w16, 8f
9365 \\ .inst 0x38208020 + 0xc0000000 + 0xc00000
9366 \\ cbz wzr, 1f
9367 \\8:
9368 \\ mov x16, x0
9369 \\0:
9370 \\ ldaxr x0, [x1]
9371 \\ stlxr w17, x16, [x1]
9372 \\ cbnz w17, 0b
9373 \\1:
9374 : [ret] "={x0}" (-> u64),
9375 : [val] "{x0}" (val),
9376 [ptr] "{x1}" (ptr),
9377 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9378 : "w15", "w16", "w17", "memory"
9379 );
9380}
9381fn swp8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9382 @setRuntimeSafety(false);
9383 __init_aarch64_have_lse_atomics();
9384
9385 return asm volatile (
9386 \\ cbz w16, 8f
9387 \\ swpal x0, x0, [x1]
9388 \\ cbz wzr, 1f
9389 \\8:
9390 \\ mov x16, x0
9391 \\0:
9392 \\ ldaxr x0, [x1]
9393 \\ stlxr w17, x16, [x1]
9394 \\ cbnz w17, 0b
9395 \\1:
9396 : [ret] "={x0}" (-> u64),
9397 : [val] "{x0}" (val),
9398 [ptr] "{x1}" (ptr),
9399 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9400 : "w15", "w16", "w17", "memory"
9401 );
9402}
9403fn swp8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9404 @setRuntimeSafety(false);
9405 __init_aarch64_have_lse_atomics();
9406
9407 return asm volatile (
9408 \\ cbz w16, 8f
9409 \\ .inst 0x38208020 + 0xc0000000 + 0xc00000
9410 \\ cbz wzr, 1f
9411 \\8:
9412 \\ mov x16, x0
9413 \\0:
9414 \\ ldaxr x0, [x1]
9415 \\ stlxr w17, x16, [x1]
9416 \\ cbnz w17, 0b
9417 \\1:
9418 : [ret] "={x0}" (-> u64),
9419 : [val] "{x0}" (val),
9420 [ptr] "{x1}" (ptr),
9421 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9422 : "w15", "w16", "w17", "memory"
9423 );
9424}
9425const __aarch64_swp8_acq_rel = if (is_darwin)
9426 if (has_lse)
9427 swp8AcqRelDarwinLse
9428 else
9429 swp8AcqRelDarwinNolse
9430else if (has_lse)
9431 swp8AcqRelNondarwinLse
9432else
9433 swp8AcqRelNondarwinNolse;
9434fn ldadd8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9435 @setRuntimeSafety(false);
9436 __init_aarch64_have_lse_atomics();
9437
9438 return asm volatile (
9439 \\ cbz w16, 8f
9440 \\ ldaddal x0, x0, [x1]
9441 \\ cbz wzr, 1f
9442 \\8:
9443 \\ mov x16, x0
9444 \\0:
9445 \\ ldaxr x0, [x1]
9446 \\ add x17, x0, x16
9447 \\ stlxr w15, x17, [x1]
9448 \\ cbnz w15, 0b
9449 \\1:
9450 : [ret] "={x0}" (-> u64),
9451 : [val] "{x0}" (val),
9452 [ptr] "{x1}" (ptr),
9453 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9454 : "w15", "w16", "w17", "memory"
9455 );
9456}
9457fn ldadd8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9458 @setRuntimeSafety(false);
9459 __init_aarch64_have_lse_atomics();
9460
9461 return asm volatile (
9462 \\ cbz w16, 8f
9463 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
9464 \\ cbz wzr, 1f
9465 \\8:
9466 \\ mov x16, x0
9467 \\0:
9468 \\ ldaxr x0, [x1]
9469 \\ add x17, x0, x16
9470 \\ stlxr w15, x17, [x1]
9471 \\ cbnz w15, 0b
9472 \\1:
9473 : [ret] "={x0}" (-> u64),
9474 : [val] "{x0}" (val),
9475 [ptr] "{x1}" (ptr),
9476 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9477 : "w15", "w16", "w17", "memory"
9478 );
9479}
9480fn ldadd8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9481 @setRuntimeSafety(false);
9482 __init_aarch64_have_lse_atomics();
9483
9484 return asm volatile (
9485 \\ cbz w16, 8f
9486 \\ ldaddal x0, x0, [x1]
9487 \\ cbz wzr, 1f
9488 \\8:
9489 \\ mov x16, x0
9490 \\0:
9491 \\ ldaxr x0, [x1]
9492 \\ add x17, x0, x16
9493 \\ stlxr w15, x17, [x1]
9494 \\ cbnz w15, 0b
9495 \\1:
9496 : [ret] "={x0}" (-> u64),
9497 : [val] "{x0}" (val),
9498 [ptr] "{x1}" (ptr),
9499 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9500 : "w15", "w16", "w17", "memory"
9501 );
9502}
9503fn ldadd8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9504 @setRuntimeSafety(false);
9505 __init_aarch64_have_lse_atomics();
9506
9507 return asm volatile (
9508 \\ cbz w16, 8f
9509 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
9510 \\ cbz wzr, 1f
9511 \\8:
9512 \\ mov x16, x0
9513 \\0:
9514 \\ ldaxr x0, [x1]
9515 \\ add x17, x0, x16
9516 \\ stlxr w15, x17, [x1]
9517 \\ cbnz w15, 0b
9518 \\1:
9519 : [ret] "={x0}" (-> u64),
9520 : [val] "{x0}" (val),
9521 [ptr] "{x1}" (ptr),
9522 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9523 : "w15", "w16", "w17", "memory"
9524 );
9525}
9526const __aarch64_ldadd8_acq_rel = if (is_darwin)
9527 if (has_lse)
9528 ldadd8AcqRelDarwinLse
9529 else
9530 ldadd8AcqRelDarwinNolse
9531else if (has_lse)
9532 ldadd8AcqRelNondarwinLse
9533else
9534 ldadd8AcqRelNondarwinNolse;
9535fn ldclr8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9536 @setRuntimeSafety(false);
9537 __init_aarch64_have_lse_atomics();
9538
9539 return asm volatile (
9540 \\ cbz w16, 8f
9541 \\ ldclral x0, x0, [x1]
9542 \\ cbz wzr, 1f
9543 \\8:
9544 \\ mov x16, x0
9545 \\0:
9546 \\ ldaxr x0, [x1]
9547 \\ bic x17, x0, x16
9548 \\ stlxr w15, x17, [x1]
9549 \\ cbnz w15, 0b
9550 \\1:
9551 : [ret] "={x0}" (-> u64),
9552 : [val] "{x0}" (val),
9553 [ptr] "{x1}" (ptr),
9554 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9555 : "w15", "w16", "w17", "memory"
9556 );
9557}
9558fn ldclr8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9559 @setRuntimeSafety(false);
9560 __init_aarch64_have_lse_atomics();
9561
9562 return asm volatile (
9563 \\ cbz w16, 8f
9564 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
9565 \\ cbz wzr, 1f
9566 \\8:
9567 \\ mov x16, x0
9568 \\0:
9569 \\ ldaxr x0, [x1]
9570 \\ bic x17, x0, x16
9571 \\ stlxr w15, x17, [x1]
9572 \\ cbnz w15, 0b
9573 \\1:
9574 : [ret] "={x0}" (-> u64),
9575 : [val] "{x0}" (val),
9576 [ptr] "{x1}" (ptr),
9577 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9578 : "w15", "w16", "w17", "memory"
9579 );
9580}
9581fn ldclr8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9582 @setRuntimeSafety(false);
9583 __init_aarch64_have_lse_atomics();
9584
9585 return asm volatile (
9586 \\ cbz w16, 8f
9587 \\ ldclral x0, x0, [x1]
9588 \\ cbz wzr, 1f
9589 \\8:
9590 \\ mov x16, x0
9591 \\0:
9592 \\ ldaxr x0, [x1]
9593 \\ bic x17, x0, x16
9594 \\ stlxr w15, x17, [x1]
9595 \\ cbnz w15, 0b
9596 \\1:
9597 : [ret] "={x0}" (-> u64),
9598 : [val] "{x0}" (val),
9599 [ptr] "{x1}" (ptr),
9600 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9601 : "w15", "w16", "w17", "memory"
9602 );
9603}
9604fn ldclr8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9605 @setRuntimeSafety(false);
9606 __init_aarch64_have_lse_atomics();
9607
9608 return asm volatile (
9609 \\ cbz w16, 8f
9610 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
9611 \\ cbz wzr, 1f
9612 \\8:
9613 \\ mov x16, x0
9614 \\0:
9615 \\ ldaxr x0, [x1]
9616 \\ bic x17, x0, x16
9617 \\ stlxr w15, x17, [x1]
9618 \\ cbnz w15, 0b
9619 \\1:
9620 : [ret] "={x0}" (-> u64),
9621 : [val] "{x0}" (val),
9622 [ptr] "{x1}" (ptr),
9623 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9624 : "w15", "w16", "w17", "memory"
9625 );
9626}
9627const __aarch64_ldclr8_acq_rel = if (is_darwin)
9628 if (has_lse)
9629 ldclr8AcqRelDarwinLse
9630 else
9631 ldclr8AcqRelDarwinNolse
9632else if (has_lse)
9633 ldclr8AcqRelNondarwinLse
9634else
9635 ldclr8AcqRelNondarwinNolse;
9636fn ldeor8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9637 @setRuntimeSafety(false);
9638 __init_aarch64_have_lse_atomics();
9639
9640 return asm volatile (
9641 \\ cbz w16, 8f
9642 \\ ldeoral x0, x0, [x1]
9643 \\ cbz wzr, 1f
9644 \\8:
9645 \\ mov x16, x0
9646 \\0:
9647 \\ ldaxr x0, [x1]
9648 \\ eor x17, x0, x16
9649 \\ stlxr w15, x17, [x1]
9650 \\ cbnz w15, 0b
9651 \\1:
9652 : [ret] "={x0}" (-> u64),
9653 : [val] "{x0}" (val),
9654 [ptr] "{x1}" (ptr),
9655 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9656 : "w15", "w16", "w17", "memory"
9657 );
9658}
9659fn ldeor8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9660 @setRuntimeSafety(false);
9661 __init_aarch64_have_lse_atomics();
9662
9663 return asm volatile (
9664 \\ cbz w16, 8f
9665 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
9666 \\ cbz wzr, 1f
9667 \\8:
9668 \\ mov x16, x0
9669 \\0:
9670 \\ ldaxr x0, [x1]
9671 \\ eor x17, x0, x16
9672 \\ stlxr w15, x17, [x1]
9673 \\ cbnz w15, 0b
9674 \\1:
9675 : [ret] "={x0}" (-> u64),
9676 : [val] "{x0}" (val),
9677 [ptr] "{x1}" (ptr),
9678 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9679 : "w15", "w16", "w17", "memory"
9680 );
9681}
9682fn ldeor8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9683 @setRuntimeSafety(false);
9684 __init_aarch64_have_lse_atomics();
9685
9686 return asm volatile (
9687 \\ cbz w16, 8f
9688 \\ ldeoral x0, x0, [x1]
9689 \\ cbz wzr, 1f
9690 \\8:
9691 \\ mov x16, x0
9692 \\0:
9693 \\ ldaxr x0, [x1]
9694 \\ eor x17, x0, x16
9695 \\ stlxr w15, x17, [x1]
9696 \\ cbnz w15, 0b
9697 \\1:
9698 : [ret] "={x0}" (-> u64),
9699 : [val] "{x0}" (val),
9700 [ptr] "{x1}" (ptr),
9701 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9702 : "w15", "w16", "w17", "memory"
9703 );
9704}
9705fn ldeor8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9706 @setRuntimeSafety(false);
9707 __init_aarch64_have_lse_atomics();
9708
9709 return asm volatile (
9710 \\ cbz w16, 8f
9711 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
9712 \\ cbz wzr, 1f
9713 \\8:
9714 \\ mov x16, x0
9715 \\0:
9716 \\ ldaxr x0, [x1]
9717 \\ eor x17, x0, x16
9718 \\ stlxr w15, x17, [x1]
9719 \\ cbnz w15, 0b
9720 \\1:
9721 : [ret] "={x0}" (-> u64),
9722 : [val] "{x0}" (val),
9723 [ptr] "{x1}" (ptr),
9724 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9725 : "w15", "w16", "w17", "memory"
9726 );
9727}
9728const __aarch64_ldeor8_acq_rel = if (is_darwin)
9729 if (has_lse)
9730 ldeor8AcqRelDarwinLse
9731 else
9732 ldeor8AcqRelDarwinNolse
9733else if (has_lse)
9734 ldeor8AcqRelNondarwinLse
9735else
9736 ldeor8AcqRelNondarwinNolse;
9737fn ldset8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9738 @setRuntimeSafety(false);
9739 __init_aarch64_have_lse_atomics();
9740
9741 return asm volatile (
9742 \\ cbz w16, 8f
9743 \\ ldsetal x0, x0, [x1]
9744 \\ cbz wzr, 1f
9745 \\8:
9746 \\ mov x16, x0
9747 \\0:
9748 \\ ldaxr x0, [x1]
9749 \\ orr x17, x0, x16
9750 \\ stlxr w15, x17, [x1]
9751 \\ cbnz w15, 0b
9752 \\1:
9753 : [ret] "={x0}" (-> u64),
9754 : [val] "{x0}" (val),
9755 [ptr] "{x1}" (ptr),
9756 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9757 : "w15", "w16", "w17", "memory"
9758 );
9759}
9760fn ldset8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9761 @setRuntimeSafety(false);
9762 __init_aarch64_have_lse_atomics();
9763
9764 return asm volatile (
9765 \\ cbz w16, 8f
9766 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
9767 \\ cbz wzr, 1f
9768 \\8:
9769 \\ mov x16, x0
9770 \\0:
9771 \\ ldaxr x0, [x1]
9772 \\ orr x17, x0, x16
9773 \\ stlxr w15, x17, [x1]
9774 \\ cbnz w15, 0b
9775 \\1:
9776 : [ret] "={x0}" (-> u64),
9777 : [val] "{x0}" (val),
9778 [ptr] "{x1}" (ptr),
9779 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9780 : "w15", "w16", "w17", "memory"
9781 );
9782}
9783fn ldset8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9784 @setRuntimeSafety(false);
9785 __init_aarch64_have_lse_atomics();
9786
9787 return asm volatile (
9788 \\ cbz w16, 8f
9789 \\ ldsetal x0, x0, [x1]
9790 \\ cbz wzr, 1f
9791 \\8:
9792 \\ mov x16, x0
9793 \\0:
9794 \\ ldaxr x0, [x1]
9795 \\ orr x17, x0, x16
9796 \\ stlxr w15, x17, [x1]
9797 \\ cbnz w15, 0b
9798 \\1:
9799 : [ret] "={x0}" (-> u64),
9800 : [val] "{x0}" (val),
9801 [ptr] "{x1}" (ptr),
9802 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9803 : "w15", "w16", "w17", "memory"
9804 );
9805}
9806fn ldset8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9807 @setRuntimeSafety(false);
9808 __init_aarch64_have_lse_atomics();
9809
9810 return asm volatile (
9811 \\ cbz w16, 8f
9812 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
9813 \\ cbz wzr, 1f
9814 \\8:
9815 \\ mov x16, x0
9816 \\0:
9817 \\ ldaxr x0, [x1]
9818 \\ orr x17, x0, x16
9819 \\ stlxr w15, x17, [x1]
9820 \\ cbnz w15, 0b
9821 \\1:
9822 : [ret] "={x0}" (-> u64),
9823 : [val] "{x0}" (val),
9824 [ptr] "{x1}" (ptr),
9825 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9826 : "w15", "w16", "w17", "memory"
9827 );
9828}
9829const __aarch64_ldset8_acq_rel = if (is_darwin)
9830 if (has_lse)
9831 ldset8AcqRelDarwinLse
9832 else
9833 ldset8AcqRelDarwinNolse
9834else if (has_lse)
9835 ldset8AcqRelNondarwinLse
9836else
9837 ldset8AcqRelNondarwinNolse;
9838fn cas16RelaxDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
9839 @setRuntimeSafety(false);
9840 __init_aarch64_have_lse_atomics();
9841
9842 return asm volatile (
9843 \\ cbz w16, 8f
9844 \\ casp x0, x1, x2, x3, [x4]
9845 \\
9846 \\ cbz wzr, 1f
9847 \\8:
9848 \\ mov x16, x0
9849 \\ mov x17, x1
9850 \\0:
9851 \\ ldxp x0, x1, [x4]
9852 \\ cmp x0, x16
9853 \\ ccmp x1, x17, #0, eq
9854 \\ bne 1f
9855 \\ stxp w15, x2, x3, [x4]
9856 \\ cbnz w15, 0b
9857 \\1:
9858 : [ret] "={x0}" (-> u128),
9859 : [expected] "{x0}" (expected),
9860 [desired] "{x1}" (desired),
9861 [ptr] "{x2}" (ptr),
9862 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9863 : "w15", "w16", "w17", "memory"
9864 );
9865}
9866fn cas16RelaxDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
9867 @setRuntimeSafety(false);
9868 __init_aarch64_have_lse_atomics();
9869
9870 return asm volatile (
9871 \\ cbz w16, 8f
9872 \\ .inst 0x48207c82 + 0x000000
9873 \\
9874 \\ cbz wzr, 1f
9875 \\8:
9876 \\ mov x16, x0
9877 \\ mov x17, x1
9878 \\0:
9879 \\ ldxp x0, x1, [x4]
9880 \\ cmp x0, x16
9881 \\ ccmp x1, x17, #0, eq
9882 \\ bne 1f
9883 \\ stxp w15, x2, x3, [x4]
9884 \\ cbnz w15, 0b
9885 \\1:
9886 : [ret] "={x0}" (-> u128),
9887 : [expected] "{x0}" (expected),
9888 [desired] "{x1}" (desired),
9889 [ptr] "{x2}" (ptr),
9890 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9891 : "w15", "w16", "w17", "memory"
9892 );
9893}
9894fn cas16RelaxNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
9895 @setRuntimeSafety(false);
9896 __init_aarch64_have_lse_atomics();
9897
9898 return asm volatile (
9899 \\ cbz w16, 8f
9900 \\ casp x0, x1, x2, x3, [x4]
9901 \\
9902 \\ cbz wzr, 1f
9903 \\8:
9904 \\ mov x16, x0
9905 \\ mov x17, x1
9906 \\0:
9907 \\ ldxp x0, x1, [x4]
9908 \\ cmp x0, x16
9909 \\ ccmp x1, x17, #0, eq
9910 \\ bne 1f
9911 \\ stxp w15, x2, x3, [x4]
9912 \\ cbnz w15, 0b
9913 \\1:
9914 : [ret] "={x0}" (-> u128),
9915 : [expected] "{x0}" (expected),
9916 [desired] "{x1}" (desired),
9917 [ptr] "{x2}" (ptr),
9918 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9919 : "w15", "w16", "w17", "memory"
9920 );
9921}
9922fn cas16RelaxNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
9923 @setRuntimeSafety(false);
9924 __init_aarch64_have_lse_atomics();
9925
9926 return asm volatile (
9927 \\ cbz w16, 8f
9928 \\ .inst 0x48207c82 + 0x000000
9929 \\
9930 \\ cbz wzr, 1f
9931 \\8:
9932 \\ mov x16, x0
9933 \\ mov x17, x1
9934 \\0:
9935 \\ ldxp x0, x1, [x4]
9936 \\ cmp x0, x16
9937 \\ ccmp x1, x17, #0, eq
9938 \\ bne 1f
9939 \\ stxp w15, x2, x3, [x4]
9940 \\ cbnz w15, 0b
9941 \\1:
9942 : [ret] "={x0}" (-> u128),
9943 : [expected] "{x0}" (expected),
9944 [desired] "{x1}" (desired),
9945 [ptr] "{x2}" (ptr),
9946 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9947 : "w15", "w16", "w17", "memory"
9948 );
9949}
9950const __aarch64_cas16_relax = if (is_darwin)
9951 if (has_lse)
9952 cas16RelaxDarwinLse
9953 else
9954 cas16RelaxDarwinNolse
9955else if (has_lse)
9956 cas16RelaxNondarwinLse
9957else
9958 cas16RelaxNondarwinNolse;
9959fn cas16AcqDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
9960 @setRuntimeSafety(false);
9961 __init_aarch64_have_lse_atomics();
9962
9963 return asm volatile (
9964 \\ cbz w16, 8f
9965 \\ caspa x0, x1, x2, x3, [x4]
9966 \\
9967 \\ cbz wzr, 1f
9968 \\8:
9969 \\ mov x16, x0
9970 \\ mov x17, x1
9971 \\0:
9972 \\ ldaxp x0, x1, [x4]
9973 \\ cmp x0, x16
9974 \\ ccmp x1, x17, #0, eq
9975 \\ bne 1f
9976 \\ stxp w15, x2, x3, [x4]
9977 \\ cbnz w15, 0b
9978 \\1:
9979 : [ret] "={x0}" (-> u128),
9980 : [expected] "{x0}" (expected),
9981 [desired] "{x1}" (desired),
9982 [ptr] "{x2}" (ptr),
9983 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9984 : "w15", "w16", "w17", "memory"
9985 );
9986}
9987fn cas16AcqDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
9988 @setRuntimeSafety(false);
9989 __init_aarch64_have_lse_atomics();
9990
9991 return asm volatile (
9992 \\ cbz w16, 8f
9993 \\ .inst 0x48207c82 + 0x400000
9994 \\
9995 \\ cbz wzr, 1f
9996 \\8:
9997 \\ mov x16, x0
9998 \\ mov x17, x1
9999 \\0:
10000 \\ ldaxp x0, x1, [x4]
10001 \\ cmp x0, x16
10002 \\ ccmp x1, x17, #0, eq
10003 \\ bne 1f
10004 \\ stxp w15, x2, x3, [x4]
10005 \\ cbnz w15, 0b
10006 \\1:
10007 : [ret] "={x0}" (-> u128),
10008 : [expected] "{x0}" (expected),
10009 [desired] "{x1}" (desired),
10010 [ptr] "{x2}" (ptr),
10011 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10012 : "w15", "w16", "w17", "memory"
10013 );
10014}
10015fn cas16AcqNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10016 @setRuntimeSafety(false);
10017 __init_aarch64_have_lse_atomics();
10018
10019 return asm volatile (
10020 \\ cbz w16, 8f
10021 \\ caspa x0, x1, x2, x3, [x4]
10022 \\
10023 \\ cbz wzr, 1f
10024 \\8:
10025 \\ mov x16, x0
10026 \\ mov x17, x1
10027 \\0:
10028 \\ ldaxp x0, x1, [x4]
10029 \\ cmp x0, x16
10030 \\ ccmp x1, x17, #0, eq
10031 \\ bne 1f
10032 \\ stxp w15, x2, x3, [x4]
10033 \\ cbnz w15, 0b
10034 \\1:
10035 : [ret] "={x0}" (-> u128),
10036 : [expected] "{x0}" (expected),
10037 [desired] "{x1}" (desired),
10038 [ptr] "{x2}" (ptr),
10039 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10040 : "w15", "w16", "w17", "memory"
10041 );
10042}
10043fn cas16AcqNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10044 @setRuntimeSafety(false);
10045 __init_aarch64_have_lse_atomics();
10046
10047 return asm volatile (
10048 \\ cbz w16, 8f
10049 \\ .inst 0x48207c82 + 0x400000
10050 \\
10051 \\ cbz wzr, 1f
10052 \\8:
10053 \\ mov x16, x0
10054 \\ mov x17, x1
10055 \\0:
10056 \\ ldaxp x0, x1, [x4]
10057 \\ cmp x0, x16
10058 \\ ccmp x1, x17, #0, eq
10059 \\ bne 1f
10060 \\ stxp w15, x2, x3, [x4]
10061 \\ cbnz w15, 0b
10062 \\1:
10063 : [ret] "={x0}" (-> u128),
10064 : [expected] "{x0}" (expected),
10065 [desired] "{x1}" (desired),
10066 [ptr] "{x2}" (ptr),
10067 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10068 : "w15", "w16", "w17", "memory"
10069 );
10070}
10071const __aarch64_cas16_acq = if (is_darwin)
10072 if (has_lse)
10073 cas16AcqDarwinLse
10074 else
10075 cas16AcqDarwinNolse
10076else if (has_lse)
10077 cas16AcqNondarwinLse
10078else
10079 cas16AcqNondarwinNolse;
10080fn cas16RelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10081 @setRuntimeSafety(false);
10082 __init_aarch64_have_lse_atomics();
10083
10084 return asm volatile (
10085 \\ cbz w16, 8f
10086 \\ caspl x0, x1, x2, x3, [x4]
10087 \\
10088 \\ cbz wzr, 1f
10089 \\8:
10090 \\ mov x16, x0
10091 \\ mov x17, x1
10092 \\0:
10093 \\ ldxp x0, x1, [x4]
10094 \\ cmp x0, x16
10095 \\ ccmp x1, x17, #0, eq
10096 \\ bne 1f
10097 \\ stlxp w15, x2, x3, [x4]
10098 \\ cbnz w15, 0b
10099 \\1:
10100 : [ret] "={x0}" (-> u128),
10101 : [expected] "{x0}" (expected),
10102 [desired] "{x1}" (desired),
10103 [ptr] "{x2}" (ptr),
10104 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10105 : "w15", "w16", "w17", "memory"
10106 );
10107}
10108fn cas16RelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10109 @setRuntimeSafety(false);
10110 __init_aarch64_have_lse_atomics();
10111
10112 return asm volatile (
10113 \\ cbz w16, 8f
10114 \\ .inst 0x48207c82 + 0x008000
10115 \\
10116 \\ cbz wzr, 1f
10117 \\8:
10118 \\ mov x16, x0
10119 \\ mov x17, x1
10120 \\0:
10121 \\ ldxp x0, x1, [x4]
10122 \\ cmp x0, x16
10123 \\ ccmp x1, x17, #0, eq
10124 \\ bne 1f
10125 \\ stlxp w15, x2, x3, [x4]
10126 \\ cbnz w15, 0b
10127 \\1:
10128 : [ret] "={x0}" (-> u128),
10129 : [expected] "{x0}" (expected),
10130 [desired] "{x1}" (desired),
10131 [ptr] "{x2}" (ptr),
10132 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10133 : "w15", "w16", "w17", "memory"
10134 );
10135}
10136fn cas16RelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10137 @setRuntimeSafety(false);
10138 __init_aarch64_have_lse_atomics();
10139
10140 return asm volatile (
10141 \\ cbz w16, 8f
10142 \\ caspl x0, x1, x2, x3, [x4]
10143 \\
10144 \\ cbz wzr, 1f
10145 \\8:
10146 \\ mov x16, x0
10147 \\ mov x17, x1
10148 \\0:
10149 \\ ldxp x0, x1, [x4]
10150 \\ cmp x0, x16
10151 \\ ccmp x1, x17, #0, eq
10152 \\ bne 1f
10153 \\ stlxp w15, x2, x3, [x4]
10154 \\ cbnz w15, 0b
10155 \\1:
10156 : [ret] "={x0}" (-> u128),
10157 : [expected] "{x0}" (expected),
10158 [desired] "{x1}" (desired),
10159 [ptr] "{x2}" (ptr),
10160 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10161 : "w15", "w16", "w17", "memory"
10162 );
10163}
10164fn cas16RelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10165 @setRuntimeSafety(false);
10166 __init_aarch64_have_lse_atomics();
10167
10168 return asm volatile (
10169 \\ cbz w16, 8f
10170 \\ .inst 0x48207c82 + 0x008000
10171 \\
10172 \\ cbz wzr, 1f
10173 \\8:
10174 \\ mov x16, x0
10175 \\ mov x17, x1
10176 \\0:
10177 \\ ldxp x0, x1, [x4]
10178 \\ cmp x0, x16
10179 \\ ccmp x1, x17, #0, eq
10180 \\ bne 1f
10181 \\ stlxp w15, x2, x3, [x4]
10182 \\ cbnz w15, 0b
10183 \\1:
10184 : [ret] "={x0}" (-> u128),
10185 : [expected] "{x0}" (expected),
10186 [desired] "{x1}" (desired),
10187 [ptr] "{x2}" (ptr),
10188 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10189 : "w15", "w16", "w17", "memory"
10190 );
10191}
10192const __aarch64_cas16_rel = if (is_darwin)
10193 if (has_lse)
10194 cas16RelDarwinLse
10195 else
10196 cas16RelDarwinNolse
10197else if (has_lse)
10198 cas16RelNondarwinLse
10199else
10200 cas16RelNondarwinNolse;
10201fn cas16AcqRelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10202 @setRuntimeSafety(false);
10203 __init_aarch64_have_lse_atomics();
10204
10205 return asm volatile (
10206 \\ cbz w16, 8f
10207 \\ caspal x0, x1, x2, x3, [x4]
10208 \\
10209 \\ cbz wzr, 1f
10210 \\8:
10211 \\ mov x16, x0
10212 \\ mov x17, x1
10213 \\0:
10214 \\ ldaxp x0, x1, [x4]
10215 \\ cmp x0, x16
10216 \\ ccmp x1, x17, #0, eq
10217 \\ bne 1f
10218 \\ stlxp w15, x2, x3, [x4]
10219 \\ cbnz w15, 0b
10220 \\1:
10221 : [ret] "={x0}" (-> u128),
10222 : [expected] "{x0}" (expected),
10223 [desired] "{x1}" (desired),
10224 [ptr] "{x2}" (ptr),
10225 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10226 : "w15", "w16", "w17", "memory"
10227 );
10228}
10229fn cas16AcqRelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10230 @setRuntimeSafety(false);
10231 __init_aarch64_have_lse_atomics();
10232
10233 return asm volatile (
10234 \\ cbz w16, 8f
10235 \\ .inst 0x48207c82 + 0x408000
10236 \\
10237 \\ cbz wzr, 1f
10238 \\8:
10239 \\ mov x16, x0
10240 \\ mov x17, x1
10241 \\0:
10242 \\ ldaxp x0, x1, [x4]
10243 \\ cmp x0, x16
10244 \\ ccmp x1, x17, #0, eq
10245 \\ bne 1f
10246 \\ stlxp w15, x2, x3, [x4]
10247 \\ cbnz w15, 0b
10248 \\1:
10249 : [ret] "={x0}" (-> u128),
10250 : [expected] "{x0}" (expected),
10251 [desired] "{x1}" (desired),
10252 [ptr] "{x2}" (ptr),
10253 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10254 : "w15", "w16", "w17", "memory"
10255 );
10256}
10257fn cas16AcqRelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10258 @setRuntimeSafety(false);
10259 __init_aarch64_have_lse_atomics();
10260
10261 return asm volatile (
10262 \\ cbz w16, 8f
10263 \\ caspal x0, x1, x2, x3, [x4]
10264 \\
10265 \\ cbz wzr, 1f
10266 \\8:
10267 \\ mov x16, x0
10268 \\ mov x17, x1
10269 \\0:
10270 \\ ldaxp x0, x1, [x4]
10271 \\ cmp x0, x16
10272 \\ ccmp x1, x17, #0, eq
10273 \\ bne 1f
10274 \\ stlxp w15, x2, x3, [x4]
10275 \\ cbnz w15, 0b
10276 \\1:
10277 : [ret] "={x0}" (-> u128),
10278 : [expected] "{x0}" (expected),
10279 [desired] "{x1}" (desired),
10280 [ptr] "{x2}" (ptr),
10281 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10282 : "w15", "w16", "w17", "memory"
10283 );
10284}
10285fn cas16AcqRelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10286 @setRuntimeSafety(false);
10287 __init_aarch64_have_lse_atomics();
10288
10289 return asm volatile (
10290 \\ cbz w16, 8f
10291 \\ .inst 0x48207c82 + 0x408000
10292 \\
10293 \\ cbz wzr, 1f
10294 \\8:
10295 \\ mov x16, x0
10296 \\ mov x17, x1
10297 \\0:
10298 \\ ldaxp x0, x1, [x4]
10299 \\ cmp x0, x16
10300 \\ ccmp x1, x17, #0, eq
10301 \\ bne 1f
10302 \\ stlxp w15, x2, x3, [x4]
10303 \\ cbnz w15, 0b
10304 \\1:
10305 : [ret] "={x0}" (-> u128),
10306 : [expected] "{x0}" (expected),
10307 [desired] "{x1}" (desired),
10308 [ptr] "{x2}" (ptr),
10309 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10310 : "w15", "w16", "w17", "memory"
10311 );
10312}
10313const __aarch64_cas16_acq_rel = if (is_darwin)
10314 if (has_lse)
10315 cas16AcqRelDarwinLse
10316 else
10317 cas16AcqRelDarwinNolse
10318else if (has_lse)
10319 cas16AcqRelNondarwinLse
10320else
10321 cas16AcqRelNondarwinNolse;
10322//TODO: Add linksection once implemented and remove init at writeFunction
10323fn __init_aarch64_have_lse_atomics() callconv(.C) void {
10324 const AT_HWCAP = 16;
10325 const HWCAP_ATOMICS = 1 << 8;
10326 const hwcap = std.os.linux.getauxval(AT_HWCAP);
10327 __aarch64_have_lse_atomics = @boolToInt((hwcap & HWCAP_ATOMICS) != 0);
10328}
10329
10330var __aarch64_have_lse_atomics: u8 = @boolToInt(has_lse);
10331
10332comptime {
10333 if (arch.isAARCH64()) {
10334 @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage });
10335 @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage });
10336 @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage });
10337 @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage });
10338 @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage });
10339 @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage });
10340 @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage });
10341 @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage });
10342 @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage });
10343 @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage });
10344 @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage });
10345 @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage });
10346 @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage });
10347 @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage });
10348 @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage });
10349 @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage });
10350 @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage });
10351 @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage });
10352 @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage });
10353 @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage });
10354 @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage });
10355 @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage });
10356 @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage });
10357 @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage });
10358 @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage });
10359 @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage });
10360 @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage });
10361 @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage });
10362 @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage });
10363 @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage });
10364 @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage });
10365 @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage });
10366 @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage });
10367 @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage });
10368 @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage });
10369 @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage });
10370 @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage });
10371 @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage });
10372 @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage });
10373 @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage });
10374 @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage });
10375 @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage });
10376 @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage });
10377 @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage });
10378 @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage });
10379 @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage });
10380 @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage });
10381 @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage });
10382 @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage });
10383 @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage });
10384 @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage });
10385 @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage });
10386 @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage });
10387 @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage });
10388 @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage });
10389 @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage });
10390 @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage });
10391 @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage });
10392 @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage });
10393 @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage });
10394 @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage });
10395 @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage });
10396 @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage });
10397 @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage });
10398 @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage });
10399 @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage });
10400 @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage });
10401 @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage });
10402 @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage });
10403 @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage });
10404 @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage });
10405 @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage });
10406 @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage });
10407 @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage });
10408 @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage });
10409 @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage });
10410 @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage });
10411 @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage });
10412 @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage });
10413 @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage });
10414 @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage });
10415 @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage });
10416 @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage });
10417 @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage });
10418 @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage });
10419 @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage });
10420 @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage });
10421 @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage });
10422 @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage });
10423 @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage });
10424 @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage });
10425 @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage });
10426 @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage });
10427 @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage });
10428 @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage });
10429 @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage });
10430 @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage });
10431 @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage });
10432 @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage });
10433 @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage });
10434 }
10435}
tools/gen_outline_atomics.zig created+594
...@@ -0,0 +1,594 @@
1const std = @import("std");
2const Allocator = std.mem.Allocator;
3
4const AtomicOp = enum {
5 cas,
6 swp,
7 ldadd,
8 ldclr,
9 ldeor,
10 ldset,
11};
12
13pub fn main() !void {
14 var gpa = std.heap.GeneralPurposeAllocator(.{}){};
15 defer std.debug.assert(!gpa.deinit());
16 var allocator = gpa.allocator();
17
18 const args = try std.process.argsAlloc(allocator);
19 defer std.process.argsFree(allocator, args);
20 if (args.len != 2) {
21 usageAndExit(std.io.getStdErr(), args[0], 1);
22 }
23
24 var file = try std.fs.cwd().createFile(args[1], .{ .truncate = true });
25
26 try file.writeAll(
27 \\const std = @import("std");
28 \\const builtin = @import("builtin");
29 \\const arch = builtin.cpu.arch;
30 \\const is_test = builtin.is_test;
31 \\const target = std.Target;
32 \\const os_tag = builtin.os.tag;
33 \\const is_darwin = target.Os.Tag.isDarwin(os_tag);
34 \\const has_lse = target.aarch64.featureSetHas(builtin.target.cpu.features, .lse);
35 \\const linkage = if (is_test)
36 \\ std.builtin.GlobalLinkage.Internal
37 \\else
38 \\ std.builtin.GlobalLinkage.Strong;
39 \\
40 \\
41 );
42
43 for ([_]N{ .one, .two, .four, .eight, .sixteen }) |n| {
44 for ([_]Ordering{ .relax, .acq, .rel, .acq_rel }) |order| {
45 for ([_]AtomicOp{ .cas, .swp, .ldadd, .ldclr, .ldeor, .ldset }) |pat| {
46 if (pat == .cas or n != .sixteen) {
47 for ([_]bool{ true, false }) |darwin| {
48 for ([_]bool{ true, false }) |lse| {
49 const darwin_name = if (darwin) "Darwin" else "Nondarwin";
50 const lse_name = if (lse) "Lse" else "Nolse";
51 var buf: [100:0]u8 = undefined;
52 const name = try std.fmt.bufPrintZ(&buf, "{s}{s}{s}{s}{s}", .{ @tagName(pat), n.toBytes(), order.capName(), darwin_name, lse_name });
53 const body = switch (pat) {
54 .cas => try generateCas(&allocator, n, order, lse),
55 .swp => try generateSwp(&allocator, n, order, lse),
56 .ldadd => try generateLd(&allocator, n, order, .ldadd, lse),
57 .ldclr => try generateLd(&allocator, n, order, .ldclr, lse),
58 .ldeor => try generateLd(&allocator, n, order, .ldeor, lse),
59 .ldset => try generateLd(&allocator, n, order, .ldset, lse),
60 };
61 defer allocator.destroy(body.ptr);
62 try writeFunction(&file, name, pat, n, body);
63 }
64 }
65 try writeExport(&file, @tagName(pat), n.toBytes(), order);
66 }
67 }
68 }
69 }
70
71 try file.writeAll(
72 \\//TODO: Add linksection once implemented and remove init at writeFunction
73 \\fn __init_aarch64_have_lse_atomics() callconv(.C) void {
74 \\ const AT_HWCAP = 16;
75 \\ const HWCAP_ATOMICS = 1 << 8;
76 \\ const hwcap = std.os.linux.getauxval(AT_HWCAP);
77 \\ __aarch64_have_lse_atomics = @boolToInt((hwcap & HWCAP_ATOMICS) != 0);
78 \\}
79 \\
80 \\var __aarch64_have_lse_atomics: u8 = @boolToInt(has_lse);
81 \\
82 \\comptime {
83 \\ if (arch.isAARCH64()) {
84 \\ @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage });
85 \\ @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage });
86 \\ @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage });
87 \\ @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage });
88 \\ @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage });
89 \\ @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage });
90 \\ @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage });
91 \\ @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage });
92 \\ @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage });
93 \\ @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage });
94 \\ @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage });
95 \\ @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage });
96 \\ @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage });
97 \\ @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage });
98 \\ @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage });
99 \\ @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage });
100 \\ @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage });
101 \\ @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage });
102 \\ @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage });
103 \\ @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage });
104 \\ @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage });
105 \\ @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage });
106 \\ @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage });
107 \\ @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage });
108 \\ @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage });
109 \\ @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage });
110 \\ @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage });
111 \\ @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage });
112 \\ @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage });
113 \\ @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage });
114 \\ @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage });
115 \\ @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage });
116 \\ @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage });
117 \\ @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage });
118 \\ @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage });
119 \\ @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage });
120 \\ @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage });
121 \\ @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage });
122 \\ @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage });
123 \\ @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage });
124 \\ @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage });
125 \\ @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage });
126 \\ @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage });
127 \\ @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage });
128 \\ @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage });
129 \\ @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage });
130 \\ @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage });
131 \\ @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage });
132 \\ @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage });
133 \\ @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage });
134 \\ @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage });
135 \\ @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage });
136 \\ @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage });
137 \\ @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage });
138 \\ @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage });
139 \\ @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage });
140 \\ @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage });
141 \\ @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage });
142 \\ @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage });
143 \\ @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage });
144 \\ @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage });
145 \\ @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage });
146 \\ @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage });
147 \\ @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage });
148 \\ @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage });
149 \\ @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage });
150 \\ @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage });
151 \\ @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage });
152 \\ @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage });
153 \\ @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage });
154 \\ @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage });
155 \\ @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage });
156 \\ @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage });
157 \\ @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage });
158 \\ @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage });
159 \\ @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage });
160 \\ @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage });
161 \\ @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage });
162 \\ @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage });
163 \\ @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage });
164 \\ @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage });
165 \\ @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage });
166 \\ @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage });
167 \\ @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage });
168 \\ @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage });
169 \\ @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage });
170 \\ @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage });
171 \\ @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage });
172 \\ @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage });
173 \\ @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage });
174 \\ @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage });
175 \\ @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage });
176 \\ @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage });
177 \\ @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage });
178 \\ @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage });
179 \\ @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage });
180 \\ @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage });
181 \\ @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage });
182 \\ @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage });
183 \\ @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage });
184 \\ }
185 \\}
186 \\
187 );
188}
189
190fn usageAndExit(file: std.fs.File, arg0: []const u8, code: u8) noreturn {
191 file.writer().print(
192 \\Usage: {s} /path/to/lib/compiler_rt/lse_atomics.zig
193 \\
194 \\Generates outline atomics for compiler-rt.
195 \\
196 , .{arg0}) catch std.process.exit(1);
197 std.process.exit(code);
198}
199
200fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, body: [:0]const u8) !void {
201 var fn_buf: [100:0]u8 = undefined;
202 const fn_sig = if (op != .cas)
203 try std.fmt.bufPrintZ(&fn_buf, "fn {[name]s}(val: u{[n]s}, ptr: *u{[n]s}) callconv(.C) u{[n]s} {{", .{ .name = name, .n = n.toBits() })
204 else
205 try std.fmt.bufPrintZ(&fn_buf, "fn {[name]s}(expected: u{[n]s}, desired: u{[n]s}, ptr: *u{[n]s}) callconv(.C) u{[n]s} {{", .{ .name = name, .n = n.toBits() });
206 try file.writeAll(fn_sig);
207 try file.writeAll(
208 \\
209 \\ @setRuntimeSafety(false);
210 \\ __init_aarch64_have_lse_atomics();
211 \\
212 \\ return asm volatile (
213 \\
214 );
215 var iter = std.mem.split(u8, body, "\n");
216 while (iter.next()) |line| {
217 try file.writeAll(" \\\\");
218 try file.writeAll(line);
219 try file.writeAll("\n");
220 }
221 var constraint_buf: [500:0]u8 = undefined;
222 const constraints = if (op != .cas)
223 try std.fmt.bufPrintZ(&constraint_buf,
224 \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]s}),
225 \\ : [val] "{{{[reg]s}0}}" (val),
226 \\ [ptr] "{{x1}}" (ptr),
227 \\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
228 \\ : "w15", "w16", "w17", "memory"
229 \\
230 , .{ .reg = n.register(), .ty = n.toBits() })
231 else
232 try std.fmt.bufPrintZ(&constraint_buf,
233 \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]s}),
234 \\ : [expected] "{{{[reg]s}0}}" (expected),
235 \\ [desired] "{{{[reg]s}1}}" (desired),
236 \\ [ptr] "{{x2}}" (ptr),
237 \\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
238 \\ : "w15", "w16", "w17", "memory"
239 \\
240 , .{ .reg = n.register(), .ty = n.toBits() });
241
242 try file.writeAll(constraints);
243 try file.writeAll(
244 \\ );
245 \\
246 );
247 try file.writeAll("}\n");
248}
249
250fn writeExport(file: *std.fs.File, pat: [:0]const u8, n: [:0]const u8, order: Ordering) !void {
251 var darwin_lse_buf: [100:0]u8 = undefined;
252 var darwin_nolse_buf: [100:0]u8 = undefined;
253 var nodarwin_lse_buf: [100:0]u8 = undefined;
254 var nodarwin_nolse_buf: [100:0]u8 = undefined;
255 var name_buf: [100:0]u8 = undefined;
256 const darwin_lse = try std.fmt.bufPrintZ(&darwin_lse_buf, "{s}{s}{s}DarwinLse", .{ pat, n, order.capName() });
257 const darwin_nolse = try std.fmt.bufPrintZ(&darwin_nolse_buf, "{s}{s}{s}DarwinNolse", .{ pat, n, order.capName() });
258 const nodarwin_lse = try std.fmt.bufPrintZ(&nodarwin_lse_buf, "{s}{s}{s}NondarwinLse", .{ pat, n, order.capName() });
259 const nodarwin_nolse = try std.fmt.bufPrintZ(&nodarwin_nolse_buf, "{s}{s}{s}NondarwinNolse", .{ pat, n, order.capName() });
260 const name = try std.fmt.bufPrintZ(&name_buf, "__aarch64_{s}{s}_{s}", .{ pat, n, @tagName(order) });
261 try file.writeAll("const ");
262 try file.writeAll(name);
263 try file.writeAll(
264 \\ = if (is_darwin)
265 \\ if (has_lse)
266 \\
267 );
268 try file.writeAll(darwin_lse);
269 try file.writeAll(
270 \\
271 \\ else
272 \\
273 );
274 try file.writeAll(darwin_nolse);
275 try file.writeAll(
276 \\
277 \\else if (has_lse)
278 \\
279 );
280 try file.writeAll(nodarwin_lse);
281 try file.writeAll(
282 \\
283 \\else
284 \\
285 );
286 try file.writeAll(nodarwin_nolse);
287 try file.writeAll(
288 \\;
289 \\
290 );
291}
292
293const N = enum(u8) {
294 one = 1,
295 two = 2,
296 four = 4,
297 eight = 8,
298 sixteen = 16,
299
300 const Defines = struct {
301 s: [:0]const u8,
302 uxt: [:0]const u8,
303 b: [:0]const u8,
304 };
305 fn defines(self: @This()) Defines {
306 const s = switch (self) {
307 .one => "b",
308 .two => "h",
309 else => "",
310 };
311 const uxt = switch (self) {
312 .one => "uxtb",
313 .two => "uxth",
314 .four, .eight, .sixteen => "mov",
315 };
316 const b = switch (self) {
317 .one => "0x00000000",
318 .two => "0x40000000",
319 .four => "0x80000000",
320 .eight => "0xc0000000",
321 else => "0x00000000",
322 };
323 return Defines{
324 .s = s,
325 .uxt = uxt,
326 .b = b,
327 };
328 }
329
330 fn register(self: @This()) [:0]const u8 {
331 return if (@enumToInt(self) < 8) "w" else "x";
332 }
333
334 fn toBytes(self: @This()) [:0]const u8 {
335 return switch (self) {
336 .one => "1",
337 .two => "2",
338 .four => "4",
339 .eight => "8",
340 .sixteen => "16",
341 };
342 }
343
344 fn toBits(self: @This()) [:0]const u8 {
345 return switch (self) {
346 .one => "8",
347 .two => "16",
348 .four => "32",
349 .eight => "64",
350 .sixteen => "128",
351 };
352 }
353};
354
355const Ordering = enum {
356 relax,
357 acq,
358 rel,
359 acq_rel,
360
361 const Defines = struct {
362 suff: [:0]const u8,
363 a: [:0]const u8,
364 l: [:0]const u8,
365 m: [:0]const u8,
366 n: [:0]const u8,
367 };
368 fn defines(self: @This()) Defines {
369 const suff = switch (self) {
370 .relax => "_relax",
371 .acq => "_acq",
372 .rel => "_rel",
373 .acq_rel => "_acq_rel",
374 };
375 const a = switch (self) {
376 .relax => "",
377 .acq => "a",
378 .rel => "",
379 .acq_rel => "a",
380 };
381 const l = switch (self) {
382 .relax => "",
383 .acq => "",
384 .rel => "l",
385 .acq_rel => "l",
386 };
387 const m = switch (self) {
388 .relax => "0x000000",
389 .acq => "0x400000",
390 .rel => "0x008000",
391 .acq_rel => "0x408000",
392 };
393 const n = switch (self) {
394 .relax => "0x000000",
395 .acq => "0x800000",
396 .rel => "0x400000",
397 .acq_rel => "0xc00000",
398 };
399 return .{ .suff = suff, .a = a, .l = l, .m = m, .n = n };
400 }
401
402 fn capName(self: @This()) [:0]const u8 {
403 return switch (self) {
404 .relax => "Relax",
405 .acq => "Acq",
406 .rel => "Rel",
407 .acq_rel => "AcqRel",
408 };
409 }
410};
411
412const LdName = enum { ldadd, ldclr, ldeor, ldset };
413
414fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u8 {
415 const s_def = n.defines();
416 const o_def = order.defines();
417 var cas_buf = try alloc.create([200:0]u8);
418 var ldxr_buf = try alloc.create([200:0]u8);
419 var stxr_buf = try alloc.create([200:0]u8);
420 defer alloc.destroy(cas_buf);
421 defer alloc.destroy(ldxr_buf);
422 defer alloc.destroy(stxr_buf);
423 var instr_buf = try alloc.create([1000:0]u8);
424 errdefer alloc.destroy(instr_buf);
425
426 const reg = n.register();
427
428 if (@enumToInt(n) < 16) {
429 const cas = if (lse) blk: {
430 break :blk try std.fmt.bufPrintZ(cas_buf,
431 \\cas{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}1, [x2]
432 \\
433 , .{ .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg });
434 } else try std.fmt.bufPrintZ(cas_buf, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m });
435 const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s });
436 const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s });
437
438 return try std.fmt.bufPrintZ(instr_buf,
439 \\ cbz w16, 8f
440 \\ {[cas]s}
441 \\ cbz wzr, 1f
442 \\8:
443 \\ {[uxt]s} {[reg]s}16, {[reg]s}0
444 \\0:
445 \\ {[ldxr]s} {[reg]s}0, [x2]
446 \\ cmp {[reg]s}0, {[reg]s}16
447 \\ bne 1f
448 \\ {[stxr]s} w17, {[reg]s}1, [x2]
449 \\ cbnz w17, 0b
450 \\1:
451 , .{
452 .cas = cas,
453 .uxt = s_def.uxt,
454 .ldxr = ldxr,
455 .stxr = stxr,
456 .reg = reg,
457 });
458 } else {
459 const casp = if (lse)
460 try std.fmt.bufPrintZ(cas_buf, "casp{s}{s} x0, x1, x2, x3, [x4]\n", .{ o_def.a, o_def.l })
461 else
462 try std.fmt.bufPrintZ(cas_buf, ".inst 0x48207c82 + {s}\n", .{o_def.m});
463
464 const ldxp = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xp", .{o_def.a});
465 const stxp = try std.fmt.bufPrintZ(stxr_buf, "st{s}xp", .{o_def.l});
466
467 return try std.fmt.bufPrintZ(instr_buf,
468 \\ cbz w16, 8f
469 \\ {[casp]s}
470 \\ cbz wzr, 1f
471 \\8:
472 \\ mov x16, x0
473 \\ mov x17, x1
474 \\0:
475 \\ {[ldxp]s} x0, x1, [x4]
476 \\ cmp x0, x16
477 \\ ccmp x1, x17, #0, eq
478 \\ bne 1f
479 \\ {[stxp]s} w15, x2, x3, [x4]
480 \\ cbnz w15, 0b
481 \\1:
482 , .{
483 .casp = casp,
484 .ldxp = ldxp,
485 .stxp = stxp,
486 });
487 }
488}
489
490fn generateSwp(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u8 {
491 const s_def = n.defines();
492 const o_def = order.defines();
493
494 var swp_buf = try alloc.create([200:0]u8);
495 var ldxr_buf = try alloc.create([200:0]u8);
496 var stxr_buf = try alloc.create([200:0]u8);
497 defer alloc.destroy(swp_buf);
498 defer alloc.destroy(ldxr_buf);
499 defer alloc.destroy(stxr_buf);
500
501 const reg = n.register();
502
503 const swp = if (lse) blk: {
504 break :blk try std.fmt.bufPrintZ(swp_buf,
505 \\swp{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}0, [x1]
506 , .{ .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg });
507 } else std.fmt.bufPrintZ(swp_buf, ".inst 0x38208020 + {s} + {s}", .{ .b = s_def.b, .n = o_def.n });
508
509 const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s });
510 const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s });
511
512 var instr_buf = try alloc.create([1000:0]u8);
513 errdefer alloc.destroy(instr_buf);
514 return try std.fmt.bufPrintZ(instr_buf,
515 \\ cbz w16, 8f
516 \\ {[swp]s}
517 \\ cbz wzr, 1f
518 \\8:
519 \\ mov {[reg]s}16, {[reg]s}0
520 \\0:
521 \\ {[ldxr]s} {[reg]s}0, [x1]
522 \\ {[stxr]s} w17, {[reg]s}16, [x1]
523 \\ cbnz w17, 0b
524 \\1:
525 , .{
526 .swp = swp,
527 .ldxr = ldxr,
528 .stxr = stxr,
529 .reg = reg,
530 });
531}
532
533fn generateLd(alloc: *Allocator, n: N, order: Ordering, ld: LdName, lse: bool) ![:0]const u8 {
534 const s_def = n.defines();
535 const o_def = order.defines();
536 const ldname = @tagName(ld);
537 const op = switch (ld) {
538 .ldadd => "add",
539 .ldclr => "bic",
540 .ldeor => "eor",
541 .ldset => "orr",
542 };
543 const op_n = switch (ld) {
544 .ldadd => "0x0000",
545 .ldclr => "0x1000",
546 .ldeor => "0x2000",
547 .ldset => "0x3000",
548 };
549
550 var swp_buf = try alloc.create([200:0]u8);
551 var ldop_buf = try alloc.create([200:0]u8);
552 var ldxr_buf = try alloc.create([200:0]u8);
553 var stxr_buf = try alloc.create([200:0]u8);
554 defer alloc.destroy(swp_buf);
555 defer alloc.destroy(ldop_buf);
556 defer alloc.destroy(ldxr_buf);
557 defer alloc.destroy(stxr_buf);
558
559 const reg = n.register();
560
561 const ldop = if (lse)
562 std.fmt.bufPrintZ(ldop_buf,
563 \\{[ldnm]s}{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}0, [x1]
564 , .{ .ldnm = ldname, .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg })
565 else
566 std.fmt.bufPrintZ(ldop_buf,
567 \\.inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}
568 , .{ .op_n = op_n, .b = s_def.b, .n = o_def.n });
569
570 const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s });
571 const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s });
572
573 var instr_buf = try alloc.create([1000:0]u8);
574 errdefer alloc.destroy(instr_buf);
575 return try std.fmt.bufPrintZ(instr_buf,
576 \\ cbz w16, 8f
577 \\ {[ldop]s}
578 \\ cbz wzr, 1f
579 \\8:
580 \\ mov {[reg]s}16, {[reg]s}0
581 \\0:
582 \\ {[ldxr]s} {[reg]s}0, [x1]
583 \\ {[op]s} {[reg]s}17, {[reg]s}0, {[reg]s}16
584 \\ {[stxr]s} w15, {[reg]s}17, [x1]
585 \\ cbnz w15, 0b
586 \\1:
587 , .{
588 .ldop = ldop,
589 .ldxr = ldxr,
590 .stxr = stxr,
591 .op = op,
592 .reg = reg,
593 });
594}