1//! This file is generated by tools/gen_outline_atomics.zig.
2const builtin = @import("builtin");
3const std = @import("std");
4const compiler_rt = @import("../compiler_rt.zig");
5const symbol = compiler_rt.symbol;
6const always_has_lse = builtin.cpu.has(.aarch64, .lse);
7
8/// This default is overridden at runtime after inspecting CPU properties.
9/// It is intentionally not exported in order to make the machine code that
10/// uses it a statically predicted direct branch rather than using the PLT,
11/// which ARM is concerned would have too much overhead.
12var __aarch64_have_lse_atomics: u8 = @intFromBool(always_has_lse);
13
14fn __aarch64_cas1_relax() align(16) callconv(.naked) void {
15 @setRuntimeSafety(false);
16 asm volatile (
17 \\ cbz w16, 8f
18 \\ .inst 0x08a07c41 + 0x00000000 + 0x000000
19 \\ ret
20 \\8:
21 \\ uxtb w16, w0
22 \\0:
23 \\ ldxrb w0, [x2]
24 \\ cmp w0, w16
25 \\ bne 1f
26 \\ stxrb w17, w1, [x2]
27 \\ cbnz w17, 0b
28 \\1:
29 \\ ret
30 :
31 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
32 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
33 unreachable;
34}
35fn __aarch64_swp1_relax() align(16) callconv(.naked) void {
36 @setRuntimeSafety(false);
37 asm volatile (
38 \\ cbz w16, 8f
39 \\ .inst 0x38208020 + 0x00000000 + 0x000000
40 \\ ret
41 \\8:
42 \\ mov w16, w0
43 \\0:
44 \\ ldxrb w0, [x1]
45 \\ stxrb w17, w16, [x1]
46 \\ cbnz w17, 0b
47 \\1:
48 \\ ret
49 :
50 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
51 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
52 unreachable;
53}
54fn __aarch64_ldadd1_relax() align(16) callconv(.naked) void {
55 @setRuntimeSafety(false);
56 asm volatile (
57 \\ cbz w16, 8f
58 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
59 \\ ret
60 \\8:
61 \\ mov w16, w0
62 \\0:
63 \\ ldxrb w0, [x1]
64 \\ add w17, w0, w16
65 \\ stxrb w15, w17, [x1]
66 \\ cbnz w15, 0b
67 \\1:
68 \\ ret
69 :
70 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
71 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
72 unreachable;
73}
74fn __aarch64_ldclr1_relax() align(16) callconv(.naked) void {
75 @setRuntimeSafety(false);
76 asm volatile (
77 \\ cbz w16, 8f
78 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
79 \\ ret
80 \\8:
81 \\ mov w16, w0
82 \\0:
83 \\ ldxrb w0, [x1]
84 \\ bic w17, w0, w16
85 \\ stxrb w15, w17, [x1]
86 \\ cbnz w15, 0b
87 \\1:
88 \\ ret
89 :
90 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
91 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
92 unreachable;
93}
94fn __aarch64_ldeor1_relax() align(16) callconv(.naked) void {
95 @setRuntimeSafety(false);
96 asm volatile (
97 \\ cbz w16, 8f
98 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
99 \\ ret
100 \\8:
101 \\ mov w16, w0
102 \\0:
103 \\ ldxrb w0, [x1]
104 \\ eor w17, w0, w16
105 \\ stxrb w15, w17, [x1]
106 \\ cbnz w15, 0b
107 \\1:
108 \\ ret
109 :
110 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
111 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
112 unreachable;
113}
114fn __aarch64_ldset1_relax() align(16) callconv(.naked) void {
115 @setRuntimeSafety(false);
116 asm volatile (
117 \\ cbz w16, 8f
118 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
119 \\ ret
120 \\8:
121 \\ mov w16, w0
122 \\0:
123 \\ ldxrb w0, [x1]
124 \\ orr w17, w0, w16
125 \\ stxrb w15, w17, [x1]
126 \\ cbnz w15, 0b
127 \\1:
128 \\ ret
129 :
130 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
131 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
132 unreachable;
133}
134fn __aarch64_cas1_acq() align(16) callconv(.naked) void {
135 @setRuntimeSafety(false);
136 asm volatile (
137 \\ cbz w16, 8f
138 \\ .inst 0x08a07c41 + 0x00000000 + 0x400000
139 \\ ret
140 \\8:
141 \\ uxtb w16, w0
142 \\0:
143 \\ ldaxrb w0, [x2]
144 \\ cmp w0, w16
145 \\ bne 1f
146 \\ stxrb w17, w1, [x2]
147 \\ cbnz w17, 0b
148 \\1:
149 \\ ret
150 :
151 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
152 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
153 unreachable;
154}
155fn __aarch64_swp1_acq() align(16) callconv(.naked) void {
156 @setRuntimeSafety(false);
157 asm volatile (
158 \\ cbz w16, 8f
159 \\ .inst 0x38208020 + 0x00000000 + 0x800000
160 \\ ret
161 \\8:
162 \\ mov w16, w0
163 \\0:
164 \\ ldaxrb w0, [x1]
165 \\ stxrb w17, w16, [x1]
166 \\ cbnz w17, 0b
167 \\1:
168 \\ ret
169 :
170 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
171 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
172 unreachable;
173}
174fn __aarch64_ldadd1_acq() align(16) callconv(.naked) void {
175 @setRuntimeSafety(false);
176 asm volatile (
177 \\ cbz w16, 8f
178 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
179 \\ ret
180 \\8:
181 \\ mov w16, w0
182 \\0:
183 \\ ldaxrb w0, [x1]
184 \\ add w17, w0, w16
185 \\ stxrb w15, w17, [x1]
186 \\ cbnz w15, 0b
187 \\1:
188 \\ ret
189 :
190 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
191 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
192 unreachable;
193}
194fn __aarch64_ldclr1_acq() align(16) callconv(.naked) void {
195 @setRuntimeSafety(false);
196 asm volatile (
197 \\ cbz w16, 8f
198 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
199 \\ ret
200 \\8:
201 \\ mov w16, w0
202 \\0:
203 \\ ldaxrb w0, [x1]
204 \\ bic w17, w0, w16
205 \\ stxrb w15, w17, [x1]
206 \\ cbnz w15, 0b
207 \\1:
208 \\ ret
209 :
210 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
211 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
212 unreachable;
213}
214fn __aarch64_ldeor1_acq() align(16) callconv(.naked) void {
215 @setRuntimeSafety(false);
216 asm volatile (
217 \\ cbz w16, 8f
218 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
219 \\ ret
220 \\8:
221 \\ mov w16, w0
222 \\0:
223 \\ ldaxrb w0, [x1]
224 \\ eor w17, w0, w16
225 \\ stxrb w15, w17, [x1]
226 \\ cbnz w15, 0b
227 \\1:
228 \\ ret
229 :
230 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
231 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
232 unreachable;
233}
234fn __aarch64_ldset1_acq() align(16) callconv(.naked) void {
235 @setRuntimeSafety(false);
236 asm volatile (
237 \\ cbz w16, 8f
238 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
239 \\ ret
240 \\8:
241 \\ mov w16, w0
242 \\0:
243 \\ ldaxrb w0, [x1]
244 \\ orr w17, w0, w16
245 \\ stxrb w15, w17, [x1]
246 \\ cbnz w15, 0b
247 \\1:
248 \\ ret
249 :
250 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
251 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
252 unreachable;
253}
254fn __aarch64_cas1_rel() align(16) callconv(.naked) void {
255 @setRuntimeSafety(false);
256 asm volatile (
257 \\ cbz w16, 8f
258 \\ .inst 0x08a07c41 + 0x00000000 + 0x008000
259 \\ ret
260 \\8:
261 \\ uxtb w16, w0
262 \\0:
263 \\ ldxrb w0, [x2]
264 \\ cmp w0, w16
265 \\ bne 1f
266 \\ stlxrb w17, w1, [x2]
267 \\ cbnz w17, 0b
268 \\1:
269 \\ ret
270 :
271 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
272 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
273 unreachable;
274}
275fn __aarch64_swp1_rel() align(16) callconv(.naked) void {
276 @setRuntimeSafety(false);
277 asm volatile (
278 \\ cbz w16, 8f
279 \\ .inst 0x38208020 + 0x00000000 + 0x400000
280 \\ ret
281 \\8:
282 \\ mov w16, w0
283 \\0:
284 \\ ldxrb w0, [x1]
285 \\ stlxrb w17, w16, [x1]
286 \\ cbnz w17, 0b
287 \\1:
288 \\ ret
289 :
290 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
291 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
292 unreachable;
293}
294fn __aarch64_ldadd1_rel() align(16) callconv(.naked) void {
295 @setRuntimeSafety(false);
296 asm volatile (
297 \\ cbz w16, 8f
298 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
299 \\ ret
300 \\8:
301 \\ mov w16, w0
302 \\0:
303 \\ ldxrb w0, [x1]
304 \\ add w17, w0, w16
305 \\ stlxrb w15, w17, [x1]
306 \\ cbnz w15, 0b
307 \\1:
308 \\ ret
309 :
310 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
311 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
312 unreachable;
313}
314fn __aarch64_ldclr1_rel() align(16) callconv(.naked) void {
315 @setRuntimeSafety(false);
316 asm volatile (
317 \\ cbz w16, 8f
318 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
319 \\ ret
320 \\8:
321 \\ mov w16, w0
322 \\0:
323 \\ ldxrb w0, [x1]
324 \\ bic w17, w0, w16
325 \\ stlxrb w15, w17, [x1]
326 \\ cbnz w15, 0b
327 \\1:
328 \\ ret
329 :
330 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
331 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
332 unreachable;
333}
334fn __aarch64_ldeor1_rel() align(16) callconv(.naked) void {
335 @setRuntimeSafety(false);
336 asm volatile (
337 \\ cbz w16, 8f
338 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
339 \\ ret
340 \\8:
341 \\ mov w16, w0
342 \\0:
343 \\ ldxrb w0, [x1]
344 \\ eor w17, w0, w16
345 \\ stlxrb w15, w17, [x1]
346 \\ cbnz w15, 0b
347 \\1:
348 \\ ret
349 :
350 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
351 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
352 unreachable;
353}
354fn __aarch64_ldset1_rel() align(16) callconv(.naked) void {
355 @setRuntimeSafety(false);
356 asm volatile (
357 \\ cbz w16, 8f
358 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
359 \\ ret
360 \\8:
361 \\ mov w16, w0
362 \\0:
363 \\ ldxrb w0, [x1]
364 \\ orr w17, w0, w16
365 \\ stlxrb w15, w17, [x1]
366 \\ cbnz w15, 0b
367 \\1:
368 \\ ret
369 :
370 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
371 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
372 unreachable;
373}
374fn __aarch64_cas1_acq_rel() align(16) callconv(.naked) void {
375 @setRuntimeSafety(false);
376 asm volatile (
377 \\ cbz w16, 8f
378 \\ .inst 0x08a07c41 + 0x00000000 + 0x408000
379 \\ ret
380 \\8:
381 \\ uxtb w16, w0
382 \\0:
383 \\ ldaxrb w0, [x2]
384 \\ cmp w0, w16
385 \\ bne 1f
386 \\ stlxrb w17, w1, [x2]
387 \\ cbnz w17, 0b
388 \\1:
389 \\ ret
390 :
391 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
392 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
393 unreachable;
394}
395fn __aarch64_swp1_acq_rel() align(16) callconv(.naked) void {
396 @setRuntimeSafety(false);
397 asm volatile (
398 \\ cbz w16, 8f
399 \\ .inst 0x38208020 + 0x00000000 + 0xc00000
400 \\ ret
401 \\8:
402 \\ mov w16, w0
403 \\0:
404 \\ ldaxrb w0, [x1]
405 \\ stlxrb w17, w16, [x1]
406 \\ cbnz w17, 0b
407 \\1:
408 \\ ret
409 :
410 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
411 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
412 unreachable;
413}
414fn __aarch64_ldadd1_acq_rel() align(16) callconv(.naked) void {
415 @setRuntimeSafety(false);
416 asm volatile (
417 \\ cbz w16, 8f
418 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
419 \\ ret
420 \\8:
421 \\ mov w16, w0
422 \\0:
423 \\ ldaxrb w0, [x1]
424 \\ add w17, w0, w16
425 \\ stlxrb w15, w17, [x1]
426 \\ cbnz w15, 0b
427 \\1:
428 \\ ret
429 :
430 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
431 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
432 unreachable;
433}
434fn __aarch64_ldclr1_acq_rel() align(16) callconv(.naked) void {
435 @setRuntimeSafety(false);
436 asm volatile (
437 \\ cbz w16, 8f
438 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
439 \\ ret
440 \\8:
441 \\ mov w16, w0
442 \\0:
443 \\ ldaxrb w0, [x1]
444 \\ bic w17, w0, w16
445 \\ stlxrb w15, w17, [x1]
446 \\ cbnz w15, 0b
447 \\1:
448 \\ ret
449 :
450 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
451 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
452 unreachable;
453}
454fn __aarch64_ldeor1_acq_rel() align(16) callconv(.naked) void {
455 @setRuntimeSafety(false);
456 asm volatile (
457 \\ cbz w16, 8f
458 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
459 \\ ret
460 \\8:
461 \\ mov w16, w0
462 \\0:
463 \\ ldaxrb w0, [x1]
464 \\ eor w17, w0, w16
465 \\ stlxrb w15, w17, [x1]
466 \\ cbnz w15, 0b
467 \\1:
468 \\ ret
469 :
470 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
471 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
472 unreachable;
473}
474fn __aarch64_ldset1_acq_rel() align(16) callconv(.naked) void {
475 @setRuntimeSafety(false);
476 asm volatile (
477 \\ cbz w16, 8f
478 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
479 \\ ret
480 \\8:
481 \\ mov w16, w0
482 \\0:
483 \\ ldaxrb w0, [x1]
484 \\ orr w17, w0, w16
485 \\ stlxrb w15, w17, [x1]
486 \\ cbnz w15, 0b
487 \\1:
488 \\ ret
489 :
490 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
491 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
492 unreachable;
493}
494fn __aarch64_cas2_relax() align(16) callconv(.naked) void {
495 @setRuntimeSafety(false);
496 asm volatile (
497 \\ cbz w16, 8f
498 \\ .inst 0x08a07c41 + 0x40000000 + 0x000000
499 \\ ret
500 \\8:
501 \\ uxth w16, w0
502 \\0:
503 \\ ldxrh w0, [x2]
504 \\ cmp w0, w16
505 \\ bne 1f
506 \\ stxrh w17, w1, [x2]
507 \\ cbnz w17, 0b
508 \\1:
509 \\ ret
510 :
511 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
512 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
513 unreachable;
514}
515fn __aarch64_swp2_relax() align(16) callconv(.naked) void {
516 @setRuntimeSafety(false);
517 asm volatile (
518 \\ cbz w16, 8f
519 \\ .inst 0x38208020 + 0x40000000 + 0x000000
520 \\ ret
521 \\8:
522 \\ mov w16, w0
523 \\0:
524 \\ ldxrh w0, [x1]
525 \\ stxrh w17, w16, [x1]
526 \\ cbnz w17, 0b
527 \\1:
528 \\ ret
529 :
530 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
531 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
532 unreachable;
533}
534fn __aarch64_ldadd2_relax() align(16) callconv(.naked) void {
535 @setRuntimeSafety(false);
536 asm volatile (
537 \\ cbz w16, 8f
538 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
539 \\ ret
540 \\8:
541 \\ mov w16, w0
542 \\0:
543 \\ ldxrh w0, [x1]
544 \\ add w17, w0, w16
545 \\ stxrh w15, w17, [x1]
546 \\ cbnz w15, 0b
547 \\1:
548 \\ ret
549 :
550 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
551 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
552 unreachable;
553}
554fn __aarch64_ldclr2_relax() align(16) callconv(.naked) void {
555 @setRuntimeSafety(false);
556 asm volatile (
557 \\ cbz w16, 8f
558 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
559 \\ ret
560 \\8:
561 \\ mov w16, w0
562 \\0:
563 \\ ldxrh w0, [x1]
564 \\ bic w17, w0, w16
565 \\ stxrh w15, w17, [x1]
566 \\ cbnz w15, 0b
567 \\1:
568 \\ ret
569 :
570 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
571 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
572 unreachable;
573}
574fn __aarch64_ldeor2_relax() align(16) callconv(.naked) void {
575 @setRuntimeSafety(false);
576 asm volatile (
577 \\ cbz w16, 8f
578 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
579 \\ ret
580 \\8:
581 \\ mov w16, w0
582 \\0:
583 \\ ldxrh w0, [x1]
584 \\ eor w17, w0, w16
585 \\ stxrh w15, w17, [x1]
586 \\ cbnz w15, 0b
587 \\1:
588 \\ ret
589 :
590 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
591 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
592 unreachable;
593}
594fn __aarch64_ldset2_relax() align(16) callconv(.naked) void {
595 @setRuntimeSafety(false);
596 asm volatile (
597 \\ cbz w16, 8f
598 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
599 \\ ret
600 \\8:
601 \\ mov w16, w0
602 \\0:
603 \\ ldxrh w0, [x1]
604 \\ orr w17, w0, w16
605 \\ stxrh w15, w17, [x1]
606 \\ cbnz w15, 0b
607 \\1:
608 \\ ret
609 :
610 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
611 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
612 unreachable;
613}
614fn __aarch64_cas2_acq() align(16) callconv(.naked) void {
615 @setRuntimeSafety(false);
616 asm volatile (
617 \\ cbz w16, 8f
618 \\ .inst 0x08a07c41 + 0x40000000 + 0x400000
619 \\ ret
620 \\8:
621 \\ uxth w16, w0
622 \\0:
623 \\ ldaxrh w0, [x2]
624 \\ cmp w0, w16
625 \\ bne 1f
626 \\ stxrh w17, w1, [x2]
627 \\ cbnz w17, 0b
628 \\1:
629 \\ ret
630 :
631 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
632 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
633 unreachable;
634}
635fn __aarch64_swp2_acq() align(16) callconv(.naked) void {
636 @setRuntimeSafety(false);
637 asm volatile (
638 \\ cbz w16, 8f
639 \\ .inst 0x38208020 + 0x40000000 + 0x800000
640 \\ ret
641 \\8:
642 \\ mov w16, w0
643 \\0:
644 \\ ldaxrh w0, [x1]
645 \\ stxrh w17, w16, [x1]
646 \\ cbnz w17, 0b
647 \\1:
648 \\ ret
649 :
650 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
651 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
652 unreachable;
653}
654fn __aarch64_ldadd2_acq() align(16) callconv(.naked) void {
655 @setRuntimeSafety(false);
656 asm volatile (
657 \\ cbz w16, 8f
658 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
659 \\ ret
660 \\8:
661 \\ mov w16, w0
662 \\0:
663 \\ ldaxrh w0, [x1]
664 \\ add w17, w0, w16
665 \\ stxrh w15, w17, [x1]
666 \\ cbnz w15, 0b
667 \\1:
668 \\ ret
669 :
670 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
671 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
672 unreachable;
673}
674fn __aarch64_ldclr2_acq() align(16) callconv(.naked) void {
675 @setRuntimeSafety(false);
676 asm volatile (
677 \\ cbz w16, 8f
678 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
679 \\ ret
680 \\8:
681 \\ mov w16, w0
682 \\0:
683 \\ ldaxrh w0, [x1]
684 \\ bic w17, w0, w16
685 \\ stxrh w15, w17, [x1]
686 \\ cbnz w15, 0b
687 \\1:
688 \\ ret
689 :
690 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
691 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
692 unreachable;
693}
694fn __aarch64_ldeor2_acq() align(16) callconv(.naked) void {
695 @setRuntimeSafety(false);
696 asm volatile (
697 \\ cbz w16, 8f
698 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
699 \\ ret
700 \\8:
701 \\ mov w16, w0
702 \\0:
703 \\ ldaxrh w0, [x1]
704 \\ eor w17, w0, w16
705 \\ stxrh w15, w17, [x1]
706 \\ cbnz w15, 0b
707 \\1:
708 \\ ret
709 :
710 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
711 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
712 unreachable;
713}
714fn __aarch64_ldset2_acq() align(16) callconv(.naked) void {
715 @setRuntimeSafety(false);
716 asm volatile (
717 \\ cbz w16, 8f
718 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
719 \\ ret
720 \\8:
721 \\ mov w16, w0
722 \\0:
723 \\ ldaxrh w0, [x1]
724 \\ orr w17, w0, w16
725 \\ stxrh w15, w17, [x1]
726 \\ cbnz w15, 0b
727 \\1:
728 \\ ret
729 :
730 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
731 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
732 unreachable;
733}
734fn __aarch64_cas2_rel() align(16) callconv(.naked) void {
735 @setRuntimeSafety(false);
736 asm volatile (
737 \\ cbz w16, 8f
738 \\ .inst 0x08a07c41 + 0x40000000 + 0x008000
739 \\ ret
740 \\8:
741 \\ uxth w16, w0
742 \\0:
743 \\ ldxrh w0, [x2]
744 \\ cmp w0, w16
745 \\ bne 1f
746 \\ stlxrh w17, w1, [x2]
747 \\ cbnz w17, 0b
748 \\1:
749 \\ ret
750 :
751 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
752 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
753 unreachable;
754}
755fn __aarch64_swp2_rel() align(16) callconv(.naked) void {
756 @setRuntimeSafety(false);
757 asm volatile (
758 \\ cbz w16, 8f
759 \\ .inst 0x38208020 + 0x40000000 + 0x400000
760 \\ ret
761 \\8:
762 \\ mov w16, w0
763 \\0:
764 \\ ldxrh w0, [x1]
765 \\ stlxrh w17, w16, [x1]
766 \\ cbnz w17, 0b
767 \\1:
768 \\ ret
769 :
770 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
771 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
772 unreachable;
773}
774fn __aarch64_ldadd2_rel() align(16) callconv(.naked) void {
775 @setRuntimeSafety(false);
776 asm volatile (
777 \\ cbz w16, 8f
778 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
779 \\ ret
780 \\8:
781 \\ mov w16, w0
782 \\0:
783 \\ ldxrh w0, [x1]
784 \\ add w17, w0, w16
785 \\ stlxrh w15, w17, [x1]
786 \\ cbnz w15, 0b
787 \\1:
788 \\ ret
789 :
790 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
791 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
792 unreachable;
793}
794fn __aarch64_ldclr2_rel() align(16) callconv(.naked) void {
795 @setRuntimeSafety(false);
796 asm volatile (
797 \\ cbz w16, 8f
798 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
799 \\ ret
800 \\8:
801 \\ mov w16, w0
802 \\0:
803 \\ ldxrh w0, [x1]
804 \\ bic w17, w0, w16
805 \\ stlxrh w15, w17, [x1]
806 \\ cbnz w15, 0b
807 \\1:
808 \\ ret
809 :
810 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
811 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
812 unreachable;
813}
814fn __aarch64_ldeor2_rel() align(16) callconv(.naked) void {
815 @setRuntimeSafety(false);
816 asm volatile (
817 \\ cbz w16, 8f
818 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
819 \\ ret
820 \\8:
821 \\ mov w16, w0
822 \\0:
823 \\ ldxrh w0, [x1]
824 \\ eor w17, w0, w16
825 \\ stlxrh w15, w17, [x1]
826 \\ cbnz w15, 0b
827 \\1:
828 \\ ret
829 :
830 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
831 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
832 unreachable;
833}
834fn __aarch64_ldset2_rel() align(16) callconv(.naked) void {
835 @setRuntimeSafety(false);
836 asm volatile (
837 \\ cbz w16, 8f
838 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
839 \\ ret
840 \\8:
841 \\ mov w16, w0
842 \\0:
843 \\ ldxrh w0, [x1]
844 \\ orr w17, w0, w16
845 \\ stlxrh w15, w17, [x1]
846 \\ cbnz w15, 0b
847 \\1:
848 \\ ret
849 :
850 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
851 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
852 unreachable;
853}
854fn __aarch64_cas2_acq_rel() align(16) callconv(.naked) void {
855 @setRuntimeSafety(false);
856 asm volatile (
857 \\ cbz w16, 8f
858 \\ .inst 0x08a07c41 + 0x40000000 + 0x408000
859 \\ ret
860 \\8:
861 \\ uxth w16, w0
862 \\0:
863 \\ ldaxrh w0, [x2]
864 \\ cmp w0, w16
865 \\ bne 1f
866 \\ stlxrh w17, w1, [x2]
867 \\ cbnz w17, 0b
868 \\1:
869 \\ ret
870 :
871 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
872 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
873 unreachable;
874}
875fn __aarch64_swp2_acq_rel() align(16) callconv(.naked) void {
876 @setRuntimeSafety(false);
877 asm volatile (
878 \\ cbz w16, 8f
879 \\ .inst 0x38208020 + 0x40000000 + 0xc00000
880 \\ ret
881 \\8:
882 \\ mov w16, w0
883 \\0:
884 \\ ldaxrh w0, [x1]
885 \\ stlxrh w17, w16, [x1]
886 \\ cbnz w17, 0b
887 \\1:
888 \\ ret
889 :
890 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
891 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
892 unreachable;
893}
894fn __aarch64_ldadd2_acq_rel() align(16) callconv(.naked) void {
895 @setRuntimeSafety(false);
896 asm volatile (
897 \\ cbz w16, 8f
898 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
899 \\ ret
900 \\8:
901 \\ mov w16, w0
902 \\0:
903 \\ ldaxrh w0, [x1]
904 \\ add w17, w0, w16
905 \\ stlxrh w15, w17, [x1]
906 \\ cbnz w15, 0b
907 \\1:
908 \\ ret
909 :
910 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
911 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
912 unreachable;
913}
914fn __aarch64_ldclr2_acq_rel() align(16) callconv(.naked) void {
915 @setRuntimeSafety(false);
916 asm volatile (
917 \\ cbz w16, 8f
918 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
919 \\ ret
920 \\8:
921 \\ mov w16, w0
922 \\0:
923 \\ ldaxrh w0, [x1]
924 \\ bic w17, w0, w16
925 \\ stlxrh w15, w17, [x1]
926 \\ cbnz w15, 0b
927 \\1:
928 \\ ret
929 :
930 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
931 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
932 unreachable;
933}
934fn __aarch64_ldeor2_acq_rel() align(16) callconv(.naked) void {
935 @setRuntimeSafety(false);
936 asm volatile (
937 \\ cbz w16, 8f
938 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
939 \\ ret
940 \\8:
941 \\ mov w16, w0
942 \\0:
943 \\ ldaxrh w0, [x1]
944 \\ eor w17, w0, w16
945 \\ stlxrh w15, w17, [x1]
946 \\ cbnz w15, 0b
947 \\1:
948 \\ ret
949 :
950 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
951 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
952 unreachable;
953}
954fn __aarch64_ldset2_acq_rel() align(16) callconv(.naked) void {
955 @setRuntimeSafety(false);
956 asm volatile (
957 \\ cbz w16, 8f
958 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
959 \\ ret
960 \\8:
961 \\ mov w16, w0
962 \\0:
963 \\ ldaxrh w0, [x1]
964 \\ orr w17, w0, w16
965 \\ stlxrh w15, w17, [x1]
966 \\ cbnz w15, 0b
967 \\1:
968 \\ ret
969 :
970 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
971 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
972 unreachable;
973}
974fn __aarch64_cas4_relax() align(16) callconv(.naked) void {
975 @setRuntimeSafety(false);
976 asm volatile (
977 \\ cbz w16, 8f
978 \\ .inst 0x08a07c41 + 0x80000000 + 0x000000
979 \\ ret
980 \\8:
981 \\ mov w16, w0
982 \\0:
983 \\ ldxr w0, [x2]
984 \\ cmp w0, w16
985 \\ bne 1f
986 \\ stxr w17, w1, [x2]
987 \\ cbnz w17, 0b
988 \\1:
989 \\ ret
990 :
991 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
992 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
993 unreachable;
994}
995fn __aarch64_swp4_relax() align(16) callconv(.naked) void {
996 @setRuntimeSafety(false);
997 asm volatile (
998 \\ cbz w16, 8f
999 \\ .inst 0x38208020 + 0x80000000 + 0x000000
1000 \\ ret
1001 \\8:
1002 \\ mov w16, w0
1003 \\0:
1004 \\ ldxr w0, [x1]
1005 \\ stxr w17, w16, [x1]
1006 \\ cbnz w17, 0b
1007 \\1:
1008 \\ ret
1009 :
1010 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1011 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1012 unreachable;
1013}
1014fn __aarch64_ldadd4_relax() align(16) callconv(.naked) void {
1015 @setRuntimeSafety(false);
1016 asm volatile (
1017 \\ cbz w16, 8f
1018 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
1019 \\ ret
1020 \\8:
1021 \\ mov w16, w0
1022 \\0:
1023 \\ ldxr w0, [x1]
1024 \\ add w17, w0, w16
1025 \\ stxr w15, w17, [x1]
1026 \\ cbnz w15, 0b
1027 \\1:
1028 \\ ret
1029 :
1030 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1031 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1032 unreachable;
1033}
1034fn __aarch64_ldclr4_relax() align(16) callconv(.naked) void {
1035 @setRuntimeSafety(false);
1036 asm volatile (
1037 \\ cbz w16, 8f
1038 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
1039 \\ ret
1040 \\8:
1041 \\ mov w16, w0
1042 \\0:
1043 \\ ldxr w0, [x1]
1044 \\ bic w17, w0, w16
1045 \\ stxr w15, w17, [x1]
1046 \\ cbnz w15, 0b
1047 \\1:
1048 \\ ret
1049 :
1050 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1051 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1052 unreachable;
1053}
1054fn __aarch64_ldeor4_relax() align(16) callconv(.naked) void {
1055 @setRuntimeSafety(false);
1056 asm volatile (
1057 \\ cbz w16, 8f
1058 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
1059 \\ ret
1060 \\8:
1061 \\ mov w16, w0
1062 \\0:
1063 \\ ldxr w0, [x1]
1064 \\ eor w17, w0, w16
1065 \\ stxr w15, w17, [x1]
1066 \\ cbnz w15, 0b
1067 \\1:
1068 \\ ret
1069 :
1070 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1071 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1072 unreachable;
1073}
1074fn __aarch64_ldset4_relax() align(16) callconv(.naked) void {
1075 @setRuntimeSafety(false);
1076 asm volatile (
1077 \\ cbz w16, 8f
1078 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
1079 \\ ret
1080 \\8:
1081 \\ mov w16, w0
1082 \\0:
1083 \\ ldxr w0, [x1]
1084 \\ orr w17, w0, w16
1085 \\ stxr w15, w17, [x1]
1086 \\ cbnz w15, 0b
1087 \\1:
1088 \\ ret
1089 :
1090 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1091 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1092 unreachable;
1093}
1094fn __aarch64_cas4_acq() align(16) callconv(.naked) void {
1095 @setRuntimeSafety(false);
1096 asm volatile (
1097 \\ cbz w16, 8f
1098 \\ .inst 0x08a07c41 + 0x80000000 + 0x400000
1099 \\ ret
1100 \\8:
1101 \\ mov w16, w0
1102 \\0:
1103 \\ ldaxr w0, [x2]
1104 \\ cmp w0, w16
1105 \\ bne 1f
1106 \\ stxr w17, w1, [x2]
1107 \\ cbnz w17, 0b
1108 \\1:
1109 \\ ret
1110 :
1111 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1112 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1113 unreachable;
1114}
1115fn __aarch64_swp4_acq() align(16) callconv(.naked) void {
1116 @setRuntimeSafety(false);
1117 asm volatile (
1118 \\ cbz w16, 8f
1119 \\ .inst 0x38208020 + 0x80000000 + 0x800000
1120 \\ ret
1121 \\8:
1122 \\ mov w16, w0
1123 \\0:
1124 \\ ldaxr w0, [x1]
1125 \\ stxr w17, w16, [x1]
1126 \\ cbnz w17, 0b
1127 \\1:
1128 \\ ret
1129 :
1130 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1131 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1132 unreachable;
1133}
1134fn __aarch64_ldadd4_acq() align(16) callconv(.naked) void {
1135 @setRuntimeSafety(false);
1136 asm volatile (
1137 \\ cbz w16, 8f
1138 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
1139 \\ ret
1140 \\8:
1141 \\ mov w16, w0
1142 \\0:
1143 \\ ldaxr w0, [x1]
1144 \\ add w17, w0, w16
1145 \\ stxr w15, w17, [x1]
1146 \\ cbnz w15, 0b
1147 \\1:
1148 \\ ret
1149 :
1150 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1151 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1152 unreachable;
1153}
1154fn __aarch64_ldclr4_acq() align(16) callconv(.naked) void {
1155 @setRuntimeSafety(false);
1156 asm volatile (
1157 \\ cbz w16, 8f
1158 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
1159 \\ ret
1160 \\8:
1161 \\ mov w16, w0
1162 \\0:
1163 \\ ldaxr w0, [x1]
1164 \\ bic w17, w0, w16
1165 \\ stxr w15, w17, [x1]
1166 \\ cbnz w15, 0b
1167 \\1:
1168 \\ ret
1169 :
1170 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1171 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1172 unreachable;
1173}
1174fn __aarch64_ldeor4_acq() align(16) callconv(.naked) void {
1175 @setRuntimeSafety(false);
1176 asm volatile (
1177 \\ cbz w16, 8f
1178 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
1179 \\ ret
1180 \\8:
1181 \\ mov w16, w0
1182 \\0:
1183 \\ ldaxr w0, [x1]
1184 \\ eor w17, w0, w16
1185 \\ stxr w15, w17, [x1]
1186 \\ cbnz w15, 0b
1187 \\1:
1188 \\ ret
1189 :
1190 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1191 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1192 unreachable;
1193}
1194fn __aarch64_ldset4_acq() align(16) callconv(.naked) void {
1195 @setRuntimeSafety(false);
1196 asm volatile (
1197 \\ cbz w16, 8f
1198 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
1199 \\ ret
1200 \\8:
1201 \\ mov w16, w0
1202 \\0:
1203 \\ ldaxr w0, [x1]
1204 \\ orr w17, w0, w16
1205 \\ stxr w15, w17, [x1]
1206 \\ cbnz w15, 0b
1207 \\1:
1208 \\ ret
1209 :
1210 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1211 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1212 unreachable;
1213}
1214fn __aarch64_cas4_rel() align(16) callconv(.naked) void {
1215 @setRuntimeSafety(false);
1216 asm volatile (
1217 \\ cbz w16, 8f
1218 \\ .inst 0x08a07c41 + 0x80000000 + 0x008000
1219 \\ ret
1220 \\8:
1221 \\ mov w16, w0
1222 \\0:
1223 \\ ldxr w0, [x2]
1224 \\ cmp w0, w16
1225 \\ bne 1f
1226 \\ stlxr w17, w1, [x2]
1227 \\ cbnz w17, 0b
1228 \\1:
1229 \\ ret
1230 :
1231 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1232 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1233 unreachable;
1234}
1235fn __aarch64_swp4_rel() align(16) callconv(.naked) void {
1236 @setRuntimeSafety(false);
1237 asm volatile (
1238 \\ cbz w16, 8f
1239 \\ .inst 0x38208020 + 0x80000000 + 0x400000
1240 \\ ret
1241 \\8:
1242 \\ mov w16, w0
1243 \\0:
1244 \\ ldxr w0, [x1]
1245 \\ stlxr w17, w16, [x1]
1246 \\ cbnz w17, 0b
1247 \\1:
1248 \\ ret
1249 :
1250 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1251 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1252 unreachable;
1253}
1254fn __aarch64_ldadd4_rel() align(16) callconv(.naked) void {
1255 @setRuntimeSafety(false);
1256 asm volatile (
1257 \\ cbz w16, 8f
1258 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
1259 \\ ret
1260 \\8:
1261 \\ mov w16, w0
1262 \\0:
1263 \\ ldxr w0, [x1]
1264 \\ add w17, w0, w16
1265 \\ stlxr w15, w17, [x1]
1266 \\ cbnz w15, 0b
1267 \\1:
1268 \\ ret
1269 :
1270 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1271 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1272 unreachable;
1273}
1274fn __aarch64_ldclr4_rel() align(16) callconv(.naked) void {
1275 @setRuntimeSafety(false);
1276 asm volatile (
1277 \\ cbz w16, 8f
1278 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
1279 \\ ret
1280 \\8:
1281 \\ mov w16, w0
1282 \\0:
1283 \\ ldxr w0, [x1]
1284 \\ bic w17, w0, w16
1285 \\ stlxr w15, w17, [x1]
1286 \\ cbnz w15, 0b
1287 \\1:
1288 \\ ret
1289 :
1290 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1291 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1292 unreachable;
1293}
1294fn __aarch64_ldeor4_rel() align(16) callconv(.naked) void {
1295 @setRuntimeSafety(false);
1296 asm volatile (
1297 \\ cbz w16, 8f
1298 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
1299 \\ ret
1300 \\8:
1301 \\ mov w16, w0
1302 \\0:
1303 \\ ldxr w0, [x1]
1304 \\ eor w17, w0, w16
1305 \\ stlxr w15, w17, [x1]
1306 \\ cbnz w15, 0b
1307 \\1:
1308 \\ ret
1309 :
1310 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1311 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1312 unreachable;
1313}
1314fn __aarch64_ldset4_rel() align(16) callconv(.naked) void {
1315 @setRuntimeSafety(false);
1316 asm volatile (
1317 \\ cbz w16, 8f
1318 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
1319 \\ ret
1320 \\8:
1321 \\ mov w16, w0
1322 \\0:
1323 \\ ldxr w0, [x1]
1324 \\ orr w17, w0, w16
1325 \\ stlxr w15, w17, [x1]
1326 \\ cbnz w15, 0b
1327 \\1:
1328 \\ ret
1329 :
1330 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1331 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1332 unreachable;
1333}
1334fn __aarch64_cas4_acq_rel() align(16) callconv(.naked) void {
1335 @setRuntimeSafety(false);
1336 asm volatile (
1337 \\ cbz w16, 8f
1338 \\ .inst 0x08a07c41 + 0x80000000 + 0x408000
1339 \\ ret
1340 \\8:
1341 \\ mov w16, w0
1342 \\0:
1343 \\ ldaxr w0, [x2]
1344 \\ cmp w0, w16
1345 \\ bne 1f
1346 \\ stlxr w17, w1, [x2]
1347 \\ cbnz w17, 0b
1348 \\1:
1349 \\ ret
1350 :
1351 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1352 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1353 unreachable;
1354}
1355fn __aarch64_swp4_acq_rel() align(16) callconv(.naked) void {
1356 @setRuntimeSafety(false);
1357 asm volatile (
1358 \\ cbz w16, 8f
1359 \\ .inst 0x38208020 + 0x80000000 + 0xc00000
1360 \\ ret
1361 \\8:
1362 \\ mov w16, w0
1363 \\0:
1364 \\ ldaxr w0, [x1]
1365 \\ stlxr w17, w16, [x1]
1366 \\ cbnz w17, 0b
1367 \\1:
1368 \\ ret
1369 :
1370 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1371 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1372 unreachable;
1373}
1374fn __aarch64_ldadd4_acq_rel() align(16) callconv(.naked) void {
1375 @setRuntimeSafety(false);
1376 asm volatile (
1377 \\ cbz w16, 8f
1378 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
1379 \\ ret
1380 \\8:
1381 \\ mov w16, w0
1382 \\0:
1383 \\ ldaxr w0, [x1]
1384 \\ add w17, w0, w16
1385 \\ stlxr w15, w17, [x1]
1386 \\ cbnz w15, 0b
1387 \\1:
1388 \\ ret
1389 :
1390 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1391 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1392 unreachable;
1393}
1394fn __aarch64_ldclr4_acq_rel() align(16) callconv(.naked) void {
1395 @setRuntimeSafety(false);
1396 asm volatile (
1397 \\ cbz w16, 8f
1398 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
1399 \\ ret
1400 \\8:
1401 \\ mov w16, w0
1402 \\0:
1403 \\ ldaxr w0, [x1]
1404 \\ bic w17, w0, w16
1405 \\ stlxr w15, w17, [x1]
1406 \\ cbnz w15, 0b
1407 \\1:
1408 \\ ret
1409 :
1410 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1411 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1412 unreachable;
1413}
1414fn __aarch64_ldeor4_acq_rel() align(16) callconv(.naked) void {
1415 @setRuntimeSafety(false);
1416 asm volatile (
1417 \\ cbz w16, 8f
1418 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
1419 \\ ret
1420 \\8:
1421 \\ mov w16, w0
1422 \\0:
1423 \\ ldaxr w0, [x1]
1424 \\ eor w17, w0, w16
1425 \\ stlxr w15, w17, [x1]
1426 \\ cbnz w15, 0b
1427 \\1:
1428 \\ ret
1429 :
1430 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1431 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1432 unreachable;
1433}
1434fn __aarch64_ldset4_acq_rel() align(16) callconv(.naked) void {
1435 @setRuntimeSafety(false);
1436 asm volatile (
1437 \\ cbz w16, 8f
1438 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
1439 \\ ret
1440 \\8:
1441 \\ mov w16, w0
1442 \\0:
1443 \\ ldaxr w0, [x1]
1444 \\ orr w17, w0, w16
1445 \\ stlxr w15, w17, [x1]
1446 \\ cbnz w15, 0b
1447 \\1:
1448 \\ ret
1449 :
1450 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1451 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1452 unreachable;
1453}
1454fn __aarch64_cas8_relax() align(16) callconv(.naked) void {
1455 @setRuntimeSafety(false);
1456 asm volatile (
1457 \\ cbz w16, 8f
1458 \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
1459 \\ ret
1460 \\8:
1461 \\ mov x16, x0
1462 \\0:
1463 \\ ldxr x0, [x2]
1464 \\ cmp x0, x16
1465 \\ bne 1f
1466 \\ stxr w17, x1, [x2]
1467 \\ cbnz w17, 0b
1468 \\1:
1469 \\ ret
1470 :
1471 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1472 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1473 unreachable;
1474}
1475fn __aarch64_swp8_relax() align(16) callconv(.naked) void {
1476 @setRuntimeSafety(false);
1477 asm volatile (
1478 \\ cbz w16, 8f
1479 \\ .inst 0x38208020 + 0xc0000000 + 0x000000
1480 \\ ret
1481 \\8:
1482 \\ mov x16, x0
1483 \\0:
1484 \\ ldxr x0, [x1]
1485 \\ stxr w17, x16, [x1]
1486 \\ cbnz w17, 0b
1487 \\1:
1488 \\ ret
1489 :
1490 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1491 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1492 unreachable;
1493}
1494fn __aarch64_ldadd8_relax() align(16) callconv(.naked) void {
1495 @setRuntimeSafety(false);
1496 asm volatile (
1497 \\ cbz w16, 8f
1498 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
1499 \\ ret
1500 \\8:
1501 \\ mov x16, x0
1502 \\0:
1503 \\ ldxr x0, [x1]
1504 \\ add x17, x0, x16
1505 \\ stxr w15, x17, [x1]
1506 \\ cbnz w15, 0b
1507 \\1:
1508 \\ ret
1509 :
1510 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1511 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1512 unreachable;
1513}
1514fn __aarch64_ldclr8_relax() align(16) callconv(.naked) void {
1515 @setRuntimeSafety(false);
1516 asm volatile (
1517 \\ cbz w16, 8f
1518 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
1519 \\ ret
1520 \\8:
1521 \\ mov x16, x0
1522 \\0:
1523 \\ ldxr x0, [x1]
1524 \\ bic x17, x0, x16
1525 \\ stxr w15, x17, [x1]
1526 \\ cbnz w15, 0b
1527 \\1:
1528 \\ ret
1529 :
1530 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1531 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1532 unreachable;
1533}
1534fn __aarch64_ldeor8_relax() align(16) callconv(.naked) void {
1535 @setRuntimeSafety(false);
1536 asm volatile (
1537 \\ cbz w16, 8f
1538 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
1539 \\ ret
1540 \\8:
1541 \\ mov x16, x0
1542 \\0:
1543 \\ ldxr x0, [x1]
1544 \\ eor x17, x0, x16
1545 \\ stxr w15, x17, [x1]
1546 \\ cbnz w15, 0b
1547 \\1:
1548 \\ ret
1549 :
1550 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1551 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1552 unreachable;
1553}
1554fn __aarch64_ldset8_relax() align(16) callconv(.naked) void {
1555 @setRuntimeSafety(false);
1556 asm volatile (
1557 \\ cbz w16, 8f
1558 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
1559 \\ ret
1560 \\8:
1561 \\ mov x16, x0
1562 \\0:
1563 \\ ldxr x0, [x1]
1564 \\ orr x17, x0, x16
1565 \\ stxr w15, x17, [x1]
1566 \\ cbnz w15, 0b
1567 \\1:
1568 \\ ret
1569 :
1570 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1571 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1572 unreachable;
1573}
1574fn __aarch64_cas8_acq() align(16) callconv(.naked) void {
1575 @setRuntimeSafety(false);
1576 asm volatile (
1577 \\ cbz w16, 8f
1578 \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
1579 \\ ret
1580 \\8:
1581 \\ mov x16, x0
1582 \\0:
1583 \\ ldaxr x0, [x2]
1584 \\ cmp x0, x16
1585 \\ bne 1f
1586 \\ stxr w17, x1, [x2]
1587 \\ cbnz w17, 0b
1588 \\1:
1589 \\ ret
1590 :
1591 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1592 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1593 unreachable;
1594}
1595fn __aarch64_swp8_acq() align(16) callconv(.naked) void {
1596 @setRuntimeSafety(false);
1597 asm volatile (
1598 \\ cbz w16, 8f
1599 \\ .inst 0x38208020 + 0xc0000000 + 0x800000
1600 \\ ret
1601 \\8:
1602 \\ mov x16, x0
1603 \\0:
1604 \\ ldaxr x0, [x1]
1605 \\ stxr w17, x16, [x1]
1606 \\ cbnz w17, 0b
1607 \\1:
1608 \\ ret
1609 :
1610 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1611 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1612 unreachable;
1613}
1614fn __aarch64_ldadd8_acq() align(16) callconv(.naked) void {
1615 @setRuntimeSafety(false);
1616 asm volatile (
1617 \\ cbz w16, 8f
1618 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
1619 \\ ret
1620 \\8:
1621 \\ mov x16, x0
1622 \\0:
1623 \\ ldaxr x0, [x1]
1624 \\ add x17, x0, x16
1625 \\ stxr w15, x17, [x1]
1626 \\ cbnz w15, 0b
1627 \\1:
1628 \\ ret
1629 :
1630 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1631 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1632 unreachable;
1633}
1634fn __aarch64_ldclr8_acq() align(16) callconv(.naked) void {
1635 @setRuntimeSafety(false);
1636 asm volatile (
1637 \\ cbz w16, 8f
1638 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
1639 \\ ret
1640 \\8:
1641 \\ mov x16, x0
1642 \\0:
1643 \\ ldaxr x0, [x1]
1644 \\ bic x17, x0, x16
1645 \\ stxr w15, x17, [x1]
1646 \\ cbnz w15, 0b
1647 \\1:
1648 \\ ret
1649 :
1650 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1651 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1652 unreachable;
1653}
1654fn __aarch64_ldeor8_acq() align(16) callconv(.naked) void {
1655 @setRuntimeSafety(false);
1656 asm volatile (
1657 \\ cbz w16, 8f
1658 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
1659 \\ ret
1660 \\8:
1661 \\ mov x16, x0
1662 \\0:
1663 \\ ldaxr x0, [x1]
1664 \\ eor x17, x0, x16
1665 \\ stxr w15, x17, [x1]
1666 \\ cbnz w15, 0b
1667 \\1:
1668 \\ ret
1669 :
1670 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1671 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1672 unreachable;
1673}
1674fn __aarch64_ldset8_acq() align(16) callconv(.naked) void {
1675 @setRuntimeSafety(false);
1676 asm volatile (
1677 \\ cbz w16, 8f
1678 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
1679 \\ ret
1680 \\8:
1681 \\ mov x16, x0
1682 \\0:
1683 \\ ldaxr x0, [x1]
1684 \\ orr x17, x0, x16
1685 \\ stxr w15, x17, [x1]
1686 \\ cbnz w15, 0b
1687 \\1:
1688 \\ ret
1689 :
1690 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1691 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1692 unreachable;
1693}
1694fn __aarch64_cas8_rel() align(16) callconv(.naked) void {
1695 @setRuntimeSafety(false);
1696 asm volatile (
1697 \\ cbz w16, 8f
1698 \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
1699 \\ ret
1700 \\8:
1701 \\ mov x16, x0
1702 \\0:
1703 \\ ldxr x0, [x2]
1704 \\ cmp x0, x16
1705 \\ bne 1f
1706 \\ stlxr w17, x1, [x2]
1707 \\ cbnz w17, 0b
1708 \\1:
1709 \\ ret
1710 :
1711 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1712 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1713 unreachable;
1714}
1715fn __aarch64_swp8_rel() align(16) callconv(.naked) void {
1716 @setRuntimeSafety(false);
1717 asm volatile (
1718 \\ cbz w16, 8f
1719 \\ .inst 0x38208020 + 0xc0000000 + 0x400000
1720 \\ ret
1721 \\8:
1722 \\ mov x16, x0
1723 \\0:
1724 \\ ldxr x0, [x1]
1725 \\ stlxr w17, x16, [x1]
1726 \\ cbnz w17, 0b
1727 \\1:
1728 \\ ret
1729 :
1730 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1731 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1732 unreachable;
1733}
1734fn __aarch64_ldadd8_rel() align(16) callconv(.naked) void {
1735 @setRuntimeSafety(false);
1736 asm volatile (
1737 \\ cbz w16, 8f
1738 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
1739 \\ ret
1740 \\8:
1741 \\ mov x16, x0
1742 \\0:
1743 \\ ldxr x0, [x1]
1744 \\ add x17, x0, x16
1745 \\ stlxr w15, x17, [x1]
1746 \\ cbnz w15, 0b
1747 \\1:
1748 \\ ret
1749 :
1750 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1751 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1752 unreachable;
1753}
1754fn __aarch64_ldclr8_rel() align(16) callconv(.naked) void {
1755 @setRuntimeSafety(false);
1756 asm volatile (
1757 \\ cbz w16, 8f
1758 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
1759 \\ ret
1760 \\8:
1761 \\ mov x16, x0
1762 \\0:
1763 \\ ldxr x0, [x1]
1764 \\ bic x17, x0, x16
1765 \\ stlxr w15, x17, [x1]
1766 \\ cbnz w15, 0b
1767 \\1:
1768 \\ ret
1769 :
1770 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1771 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1772 unreachable;
1773}
1774fn __aarch64_ldeor8_rel() align(16) callconv(.naked) void {
1775 @setRuntimeSafety(false);
1776 asm volatile (
1777 \\ cbz w16, 8f
1778 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
1779 \\ ret
1780 \\8:
1781 \\ mov x16, x0
1782 \\0:
1783 \\ ldxr x0, [x1]
1784 \\ eor x17, x0, x16
1785 \\ stlxr w15, x17, [x1]
1786 \\ cbnz w15, 0b
1787 \\1:
1788 \\ ret
1789 :
1790 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1791 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1792 unreachable;
1793}
1794fn __aarch64_ldset8_rel() align(16) callconv(.naked) void {
1795 @setRuntimeSafety(false);
1796 asm volatile (
1797 \\ cbz w16, 8f
1798 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
1799 \\ ret
1800 \\8:
1801 \\ mov x16, x0
1802 \\0:
1803 \\ ldxr x0, [x1]
1804 \\ orr x17, x0, x16
1805 \\ stlxr w15, x17, [x1]
1806 \\ cbnz w15, 0b
1807 \\1:
1808 \\ ret
1809 :
1810 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1811 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1812 unreachable;
1813}
1814fn __aarch64_cas8_acq_rel() align(16) callconv(.naked) void {
1815 @setRuntimeSafety(false);
1816 asm volatile (
1817 \\ cbz w16, 8f
1818 \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
1819 \\ ret
1820 \\8:
1821 \\ mov x16, x0
1822 \\0:
1823 \\ ldaxr x0, [x2]
1824 \\ cmp x0, x16
1825 \\ bne 1f
1826 \\ stlxr w17, x1, [x2]
1827 \\ cbnz w17, 0b
1828 \\1:
1829 \\ ret
1830 :
1831 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1832 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1833 unreachable;
1834}
1835fn __aarch64_swp8_acq_rel() align(16) callconv(.naked) void {
1836 @setRuntimeSafety(false);
1837 asm volatile (
1838 \\ cbz w16, 8f
1839 \\ .inst 0x38208020 + 0xc0000000 + 0xc00000
1840 \\ ret
1841 \\8:
1842 \\ mov x16, x0
1843 \\0:
1844 \\ ldaxr x0, [x1]
1845 \\ stlxr w17, x16, [x1]
1846 \\ cbnz w17, 0b
1847 \\1:
1848 \\ ret
1849 :
1850 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1851 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1852 unreachable;
1853}
1854fn __aarch64_ldadd8_acq_rel() align(16) callconv(.naked) void {
1855 @setRuntimeSafety(false);
1856 asm volatile (
1857 \\ cbz w16, 8f
1858 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
1859 \\ ret
1860 \\8:
1861 \\ mov x16, x0
1862 \\0:
1863 \\ ldaxr x0, [x1]
1864 \\ add x17, x0, x16
1865 \\ stlxr w15, x17, [x1]
1866 \\ cbnz w15, 0b
1867 \\1:
1868 \\ ret
1869 :
1870 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1871 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1872 unreachable;
1873}
1874fn __aarch64_ldclr8_acq_rel() align(16) callconv(.naked) void {
1875 @setRuntimeSafety(false);
1876 asm volatile (
1877 \\ cbz w16, 8f
1878 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
1879 \\ ret
1880 \\8:
1881 \\ mov x16, x0
1882 \\0:
1883 \\ ldaxr x0, [x1]
1884 \\ bic x17, x0, x16
1885 \\ stlxr w15, x17, [x1]
1886 \\ cbnz w15, 0b
1887 \\1:
1888 \\ ret
1889 :
1890 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1891 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1892 unreachable;
1893}
1894fn __aarch64_ldeor8_acq_rel() align(16) callconv(.naked) void {
1895 @setRuntimeSafety(false);
1896 asm volatile (
1897 \\ cbz w16, 8f
1898 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
1899 \\ ret
1900 \\8:
1901 \\ mov x16, x0
1902 \\0:
1903 \\ ldaxr x0, [x1]
1904 \\ eor x17, x0, x16
1905 \\ stlxr w15, x17, [x1]
1906 \\ cbnz w15, 0b
1907 \\1:
1908 \\ ret
1909 :
1910 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1911 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1912 unreachable;
1913}
1914fn __aarch64_ldset8_acq_rel() align(16) callconv(.naked) void {
1915 @setRuntimeSafety(false);
1916 asm volatile (
1917 \\ cbz w16, 8f
1918 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
1919 \\ ret
1920 \\8:
1921 \\ mov x16, x0
1922 \\0:
1923 \\ ldaxr x0, [x1]
1924 \\ orr x17, x0, x16
1925 \\ stlxr w15, x17, [x1]
1926 \\ cbnz w15, 0b
1927 \\1:
1928 \\ ret
1929 :
1930 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1931 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1932 unreachable;
1933}
1934fn __aarch64_cas16_relax() align(16) callconv(.naked) void {
1935 @setRuntimeSafety(false);
1936 asm volatile (
1937 \\ cbz w16, 8f
1938 \\ .inst 0x48207c82 + 0x000000
1939 \\ ret
1940 \\8:
1941 \\ mov x16, x0
1942 \\ mov x17, x1
1943 \\0:
1944 \\ ldxp x0, x1, [x4]
1945 \\ cmp x0, x16
1946 \\ ccmp x1, x17, #0, eq
1947 \\ bne 1f
1948 \\ stxp w15, x2, x3, [x4]
1949 \\ cbnz w15, 0b
1950 \\1:
1951 \\ ret
1952 :
1953 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1954 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1955 unreachable;
1956}
1957fn __aarch64_cas16_acq() align(16) callconv(.naked) void {
1958 @setRuntimeSafety(false);
1959 asm volatile (
1960 \\ cbz w16, 8f
1961 \\ .inst 0x48207c82 + 0x400000
1962 \\ ret
1963 \\8:
1964 \\ mov x16, x0
1965 \\ mov x17, x1
1966 \\0:
1967 \\ ldaxp x0, x1, [x4]
1968 \\ cmp x0, x16
1969 \\ ccmp x1, x17, #0, eq
1970 \\ bne 1f
1971 \\ stxp w15, x2, x3, [x4]
1972 \\ cbnz w15, 0b
1973 \\1:
1974 \\ ret
1975 :
1976 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1977 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
1978 unreachable;
1979}
1980fn __aarch64_cas16_rel() align(16) callconv(.naked) void {
1981 @setRuntimeSafety(false);
1982 asm volatile (
1983 \\ cbz w16, 8f
1984 \\ .inst 0x48207c82 + 0x008000
1985 \\ ret
1986 \\8:
1987 \\ mov x16, x0
1988 \\ mov x17, x1
1989 \\0:
1990 \\ ldxp x0, x1, [x4]
1991 \\ cmp x0, x16
1992 \\ ccmp x1, x17, #0, eq
1993 \\ bne 1f
1994 \\ stlxp w15, x2, x3, [x4]
1995 \\ cbnz w15, 0b
1996 \\1:
1997 \\ ret
1998 :
1999 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2000 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
2001 unreachable;
2002}
2003fn __aarch64_cas16_acq_rel() align(16) callconv(.naked) void {
2004 @setRuntimeSafety(false);
2005 asm volatile (
2006 \\ cbz w16, 8f
2007 \\ .inst 0x48207c82 + 0x408000
2008 \\ ret
2009 \\8:
2010 \\ mov x16, x0
2011 \\ mov x17, x1
2012 \\0:
2013 \\ ldaxp x0, x1, [x4]
2014 \\ cmp x0, x16
2015 \\ ccmp x1, x17, #0, eq
2016 \\ bne 1f
2017 \\ stlxp w15, x2, x3, [x4]
2018 \\ cbnz w15, 0b
2019 \\1:
2020 \\ ret
2021 :
2022 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2023 : .{ .w15 = true, .w16 = true, .w17 = true, .memory = true });
2024 unreachable;
2025}
2026
2027comptime {
2028 symbol(&__aarch64_cas1_relax, "__aarch64_cas1_relax");
2029 symbol(&__aarch64_swp1_relax, "__aarch64_swp1_relax");
2030 symbol(&__aarch64_ldadd1_relax, "__aarch64_ldadd1_relax");
2031 symbol(&__aarch64_ldclr1_relax, "__aarch64_ldclr1_relax");
2032 symbol(&__aarch64_ldeor1_relax, "__aarch64_ldeor1_relax");
2033 symbol(&__aarch64_ldset1_relax, "__aarch64_ldset1_relax");
2034 symbol(&__aarch64_cas1_acq, "__aarch64_cas1_acq");
2035 symbol(&__aarch64_swp1_acq, "__aarch64_swp1_acq");
2036 symbol(&__aarch64_ldadd1_acq, "__aarch64_ldadd1_acq");
2037 symbol(&__aarch64_ldclr1_acq, "__aarch64_ldclr1_acq");
2038 symbol(&__aarch64_ldeor1_acq, "__aarch64_ldeor1_acq");
2039 symbol(&__aarch64_ldset1_acq, "__aarch64_ldset1_acq");
2040 symbol(&__aarch64_cas1_rel, "__aarch64_cas1_rel");
2041 symbol(&__aarch64_swp1_rel, "__aarch64_swp1_rel");
2042 symbol(&__aarch64_ldadd1_rel, "__aarch64_ldadd1_rel");
2043 symbol(&__aarch64_ldclr1_rel, "__aarch64_ldclr1_rel");
2044 symbol(&__aarch64_ldeor1_rel, "__aarch64_ldeor1_rel");
2045 symbol(&__aarch64_ldset1_rel, "__aarch64_ldset1_rel");
2046 symbol(&__aarch64_cas1_acq_rel, "__aarch64_cas1_acq_rel");
2047 symbol(&__aarch64_swp1_acq_rel, "__aarch64_swp1_acq_rel");
2048 symbol(&__aarch64_ldadd1_acq_rel, "__aarch64_ldadd1_acq_rel");
2049 symbol(&__aarch64_ldclr1_acq_rel, "__aarch64_ldclr1_acq_rel");
2050 symbol(&__aarch64_ldeor1_acq_rel, "__aarch64_ldeor1_acq_rel");
2051 symbol(&__aarch64_ldset1_acq_rel, "__aarch64_ldset1_acq_rel");
2052 symbol(&__aarch64_cas2_relax, "__aarch64_cas2_relax");
2053 symbol(&__aarch64_swp2_relax, "__aarch64_swp2_relax");
2054 symbol(&__aarch64_ldadd2_relax, "__aarch64_ldadd2_relax");
2055 symbol(&__aarch64_ldclr2_relax, "__aarch64_ldclr2_relax");
2056 symbol(&__aarch64_ldeor2_relax, "__aarch64_ldeor2_relax");
2057 symbol(&__aarch64_ldset2_relax, "__aarch64_ldset2_relax");
2058 symbol(&__aarch64_cas2_acq, "__aarch64_cas2_acq");
2059 symbol(&__aarch64_swp2_acq, "__aarch64_swp2_acq");
2060 symbol(&__aarch64_ldadd2_acq, "__aarch64_ldadd2_acq");
2061 symbol(&__aarch64_ldclr2_acq, "__aarch64_ldclr2_acq");
2062 symbol(&__aarch64_ldeor2_acq, "__aarch64_ldeor2_acq");
2063 symbol(&__aarch64_ldset2_acq, "__aarch64_ldset2_acq");
2064 symbol(&__aarch64_cas2_rel, "__aarch64_cas2_rel");
2065 symbol(&__aarch64_swp2_rel, "__aarch64_swp2_rel");
2066 symbol(&__aarch64_ldadd2_rel, "__aarch64_ldadd2_rel");
2067 symbol(&__aarch64_ldclr2_rel, "__aarch64_ldclr2_rel");
2068 symbol(&__aarch64_ldeor2_rel, "__aarch64_ldeor2_rel");
2069 symbol(&__aarch64_ldset2_rel, "__aarch64_ldset2_rel");
2070 symbol(&__aarch64_cas2_acq_rel, "__aarch64_cas2_acq_rel");
2071 symbol(&__aarch64_swp2_acq_rel, "__aarch64_swp2_acq_rel");
2072 symbol(&__aarch64_ldadd2_acq_rel, "__aarch64_ldadd2_acq_rel");
2073 symbol(&__aarch64_ldclr2_acq_rel, "__aarch64_ldclr2_acq_rel");
2074 symbol(&__aarch64_ldeor2_acq_rel, "__aarch64_ldeor2_acq_rel");
2075 symbol(&__aarch64_ldset2_acq_rel, "__aarch64_ldset2_acq_rel");
2076 symbol(&__aarch64_cas4_relax, "__aarch64_cas4_relax");
2077 symbol(&__aarch64_swp4_relax, "__aarch64_swp4_relax");
2078 symbol(&__aarch64_ldadd4_relax, "__aarch64_ldadd4_relax");
2079 symbol(&__aarch64_ldclr4_relax, "__aarch64_ldclr4_relax");
2080 symbol(&__aarch64_ldeor4_relax, "__aarch64_ldeor4_relax");
2081 symbol(&__aarch64_ldset4_relax, "__aarch64_ldset4_relax");
2082 symbol(&__aarch64_cas4_acq, "__aarch64_cas4_acq");
2083 symbol(&__aarch64_swp4_acq, "__aarch64_swp4_acq");
2084 symbol(&__aarch64_ldadd4_acq, "__aarch64_ldadd4_acq");
2085 symbol(&__aarch64_ldclr4_acq, "__aarch64_ldclr4_acq");
2086 symbol(&__aarch64_ldeor4_acq, "__aarch64_ldeor4_acq");
2087 symbol(&__aarch64_ldset4_acq, "__aarch64_ldset4_acq");
2088 symbol(&__aarch64_cas4_rel, "__aarch64_cas4_rel");
2089 symbol(&__aarch64_swp4_rel, "__aarch64_swp4_rel");
2090 symbol(&__aarch64_ldadd4_rel, "__aarch64_ldadd4_rel");
2091 symbol(&__aarch64_ldclr4_rel, "__aarch64_ldclr4_rel");
2092 symbol(&__aarch64_ldeor4_rel, "__aarch64_ldeor4_rel");
2093 symbol(&__aarch64_ldset4_rel, "__aarch64_ldset4_rel");
2094 symbol(&__aarch64_cas4_acq_rel, "__aarch64_cas4_acq_rel");
2095 symbol(&__aarch64_swp4_acq_rel, "__aarch64_swp4_acq_rel");
2096 symbol(&__aarch64_ldadd4_acq_rel, "__aarch64_ldadd4_acq_rel");
2097 symbol(&__aarch64_ldclr4_acq_rel, "__aarch64_ldclr4_acq_rel");
2098 symbol(&__aarch64_ldeor4_acq_rel, "__aarch64_ldeor4_acq_rel");
2099 symbol(&__aarch64_ldset4_acq_rel, "__aarch64_ldset4_acq_rel");
2100 symbol(&__aarch64_cas8_relax, "__aarch64_cas8_relax");
2101 symbol(&__aarch64_swp8_relax, "__aarch64_swp8_relax");
2102 symbol(&__aarch64_ldadd8_relax, "__aarch64_ldadd8_relax");
2103 symbol(&__aarch64_ldclr8_relax, "__aarch64_ldclr8_relax");
2104 symbol(&__aarch64_ldeor8_relax, "__aarch64_ldeor8_relax");
2105 symbol(&__aarch64_ldset8_relax, "__aarch64_ldset8_relax");
2106 symbol(&__aarch64_cas8_acq, "__aarch64_cas8_acq");
2107 symbol(&__aarch64_swp8_acq, "__aarch64_swp8_acq");
2108 symbol(&__aarch64_ldadd8_acq, "__aarch64_ldadd8_acq");
2109 symbol(&__aarch64_ldclr8_acq, "__aarch64_ldclr8_acq");
2110 symbol(&__aarch64_ldeor8_acq, "__aarch64_ldeor8_acq");
2111 symbol(&__aarch64_ldset8_acq, "__aarch64_ldset8_acq");
2112 symbol(&__aarch64_cas8_rel, "__aarch64_cas8_rel");
2113 symbol(&__aarch64_swp8_rel, "__aarch64_swp8_rel");
2114 symbol(&__aarch64_ldadd8_rel, "__aarch64_ldadd8_rel");
2115 symbol(&__aarch64_ldclr8_rel, "__aarch64_ldclr8_rel");
2116 symbol(&__aarch64_ldeor8_rel, "__aarch64_ldeor8_rel");
2117 symbol(&__aarch64_ldset8_rel, "__aarch64_ldset8_rel");
2118 symbol(&__aarch64_cas8_acq_rel, "__aarch64_cas8_acq_rel");
2119 symbol(&__aarch64_swp8_acq_rel, "__aarch64_swp8_acq_rel");
2120 symbol(&__aarch64_ldadd8_acq_rel, "__aarch64_ldadd8_acq_rel");
2121 symbol(&__aarch64_ldclr8_acq_rel, "__aarch64_ldclr8_acq_rel");
2122 symbol(&__aarch64_ldeor8_acq_rel, "__aarch64_ldeor8_acq_rel");
2123 symbol(&__aarch64_ldset8_acq_rel, "__aarch64_ldset8_acq_rel");
2124 symbol(&__aarch64_cas16_relax, "__aarch64_cas16_relax");
2125 symbol(&__aarch64_cas16_acq, "__aarch64_cas16_acq");
2126 symbol(&__aarch64_cas16_rel, "__aarch64_cas16_rel");
2127 symbol(&__aarch64_cas16_acq_rel, "__aarch64_cas16_acq_rel");
2128}