authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-21 12:43:20-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-21 17:17:02-07:00
log58430ae6d14a0fa266b9afc892b7057dda638aed
treee40ea602beeb77d01f331fc3cf63202a6ce56a5b
parent95ee8ab77da6b76729beee6fc6ffccb07c9a2167

outline atomics: ret instead of jump to ret

After this, the machine code generated by zig matches identically to gcc's after the differences in loading the have_lse flag.

2 files changed, 726 insertions(+), 793 deletions(-)

lib/compiler_rt/lse_atomics.zig+700-740
...@@ -10,13 +10,12 @@ const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .l...@@ -10,13 +10,12 @@ const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .l
10/// which ARM is concerned would have too much overhead.10/// which ARM is concerned would have too much overhead.
11var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);11var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);
1212
13fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {13fn __aarch64_cas1_relax() align(16) callconv(.Naked) void {
14 @setRuntimeSafety(false);14 @setRuntimeSafety(false);
15 return asm volatile (15 asm volatile (
16 \\ cbz w16, 8f16 \\ cbz w16, 8f
17 \\ .inst 0x08a07c41 + 0x00000000 + 0x00000017 \\ .inst 0x08a07c41 + 0x00000000 + 0x000000
18 \\18 \\ ret
19 \\ cbz wzr, 1f
20 \\8:19 \\8:
21 \\ uxtb w16, w020 \\ uxtb w16, w0
22 \\0:21 \\0:
...@@ -26,20 +25,19 @@ fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {...@@ -26,20 +25,19 @@ fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
26 \\ stxrb w17, w1, [x2]25 \\ stxrb w17, w1, [x2]
27 \\ cbnz w17, 0b26 \\ cbnz w17, 0b
28 \\1:27 \\1:
29 : [ret] "={w0}" (-> u8),28 \\ ret
30 : [expected] "{w0}" (expected),29 :
31 [desired] "{w1}" (desired),30 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
32 [ptr] "{x2}" (ptr),
33 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
34 : "w15", "w16", "w17", "memory"31 : "w15", "w16", "w17", "memory"
35 );32 );
33 unreachable;
36}34}
37fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 {35fn __aarch64_swp1_relax() align(16) callconv(.Naked) void {
38 @setRuntimeSafety(false);36 @setRuntimeSafety(false);
39 return asm volatile (37 asm volatile (
40 \\ cbz w16, 8f38 \\ cbz w16, 8f
41 \\ .inst 0x38208020 + 0x00000000 + 0x00000039 \\ .inst 0x38208020 + 0x00000000 + 0x000000
42 \\ cbz wzr, 1f40 \\ ret
43 \\8:41 \\8:
44 \\ mov w16, w042 \\ mov w16, w0
45 \\0:43 \\0:
...@@ -47,19 +45,19 @@ fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -47,19 +45,19 @@ fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
47 \\ stxrb w17, w16, [x1]45 \\ stxrb w17, w16, [x1]
48 \\ cbnz w17, 0b46 \\ cbnz w17, 0b
49 \\1:47 \\1:
50 : [ret] "={w0}" (-> u8),48 \\ ret
51 : [val] "{w0}" (val),49 :
52 [ptr] "{x1}" (ptr),50 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
53 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
54 : "w15", "w16", "w17", "memory"51 : "w15", "w16", "w17", "memory"
55 );52 );
53 unreachable;
56}54}
57fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 {55fn __aarch64_ldadd1_relax() align(16) callconv(.Naked) void {
58 @setRuntimeSafety(false);56 @setRuntimeSafety(false);
59 return asm volatile (57 asm volatile (
60 \\ cbz w16, 8f58 \\ cbz w16, 8f
61 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x00000059 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
62 \\ cbz wzr, 1f60 \\ ret
63 \\8:61 \\8:
64 \\ mov w16, w062 \\ mov w16, w0
65 \\0:63 \\0:
...@@ -68,19 +66,19 @@ fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -68,19 +66,19 @@ fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
68 \\ stxrb w15, w17, [x1]66 \\ stxrb w15, w17, [x1]
69 \\ cbnz w15, 0b67 \\ cbnz w15, 0b
70 \\1:68 \\1:
71 : [ret] "={w0}" (-> u8),69 \\ ret
72 : [val] "{w0}" (val),70 :
73 [ptr] "{x1}" (ptr),71 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
74 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
75 : "w15", "w16", "w17", "memory"72 : "w15", "w16", "w17", "memory"
76 );73 );
74 unreachable;
77}75}
78fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 {76fn __aarch64_ldclr1_relax() align(16) callconv(.Naked) void {
79 @setRuntimeSafety(false);77 @setRuntimeSafety(false);
80 return asm volatile (78 asm volatile (
81 \\ cbz w16, 8f79 \\ cbz w16, 8f
82 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x00000080 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
83 \\ cbz wzr, 1f81 \\ ret
84 \\8:82 \\8:
85 \\ mov w16, w083 \\ mov w16, w0
86 \\0:84 \\0:
...@@ -89,19 +87,19 @@ fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -89,19 +87,19 @@ fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
89 \\ stxrb w15, w17, [x1]87 \\ stxrb w15, w17, [x1]
90 \\ cbnz w15, 0b88 \\ cbnz w15, 0b
91 \\1:89 \\1:
92 : [ret] "={w0}" (-> u8),90 \\ ret
93 : [val] "{w0}" (val),91 :
94 [ptr] "{x1}" (ptr),92 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
95 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
96 : "w15", "w16", "w17", "memory"93 : "w15", "w16", "w17", "memory"
97 );94 );
95 unreachable;
98}96}
99fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 {97fn __aarch64_ldeor1_relax() align(16) callconv(.Naked) void {
100 @setRuntimeSafety(false);98 @setRuntimeSafety(false);
101 return asm volatile (99 asm volatile (
102 \\ cbz w16, 8f100 \\ cbz w16, 8f
103 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000101 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
104 \\ cbz wzr, 1f102 \\ ret
105 \\8:103 \\8:
106 \\ mov w16, w0104 \\ mov w16, w0
107 \\0:105 \\0:
...@@ -110,19 +108,19 @@ fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -110,19 +108,19 @@ fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
110 \\ stxrb w15, w17, [x1]108 \\ stxrb w15, w17, [x1]
111 \\ cbnz w15, 0b109 \\ cbnz w15, 0b
112 \\1:110 \\1:
113 : [ret] "={w0}" (-> u8),111 \\ ret
114 : [val] "{w0}" (val),112 :
115 [ptr] "{x1}" (ptr),113 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
116 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
117 : "w15", "w16", "w17", "memory"114 : "w15", "w16", "w17", "memory"
118 );115 );
116 unreachable;
119}117}
120fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 {118fn __aarch64_ldset1_relax() align(16) callconv(.Naked) void {
121 @setRuntimeSafety(false);119 @setRuntimeSafety(false);
122 return asm volatile (120 asm volatile (
123 \\ cbz w16, 8f121 \\ cbz w16, 8f
124 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000122 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
125 \\ cbz wzr, 1f123 \\ ret
126 \\8:124 \\8:
127 \\ mov w16, w0125 \\ mov w16, w0
128 \\0:126 \\0:
...@@ -131,20 +129,19 @@ fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -131,20 +129,19 @@ fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
131 \\ stxrb w15, w17, [x1]129 \\ stxrb w15, w17, [x1]
132 \\ cbnz w15, 0b130 \\ cbnz w15, 0b
133 \\1:131 \\1:
134 : [ret] "={w0}" (-> u8),132 \\ ret
135 : [val] "{w0}" (val),133 :
136 [ptr] "{x1}" (ptr),134 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
137 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
138 : "w15", "w16", "w17", "memory"135 : "w15", "w16", "w17", "memory"
139 );136 );
137 unreachable;
140}138}
141fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {139fn __aarch64_cas1_acq() align(16) callconv(.Naked) void {
142 @setRuntimeSafety(false);140 @setRuntimeSafety(false);
143 return asm volatile (141 asm volatile (
144 \\ cbz w16, 8f142 \\ cbz w16, 8f
145 \\ .inst 0x08a07c41 + 0x00000000 + 0x400000143 \\ .inst 0x08a07c41 + 0x00000000 + 0x400000
146 \\144 \\ ret
147 \\ cbz wzr, 1f
148 \\8:145 \\8:
149 \\ uxtb w16, w0146 \\ uxtb w16, w0
150 \\0:147 \\0:
...@@ -154,20 +151,19 @@ fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {...@@ -154,20 +151,19 @@ fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
154 \\ stxrb w17, w1, [x2]151 \\ stxrb w17, w1, [x2]
155 \\ cbnz w17, 0b152 \\ cbnz w17, 0b
156 \\1:153 \\1:
157 : [ret] "={w0}" (-> u8),154 \\ ret
158 : [expected] "{w0}" (expected),155 :
159 [desired] "{w1}" (desired),156 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
160 [ptr] "{x2}" (ptr),
161 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
162 : "w15", "w16", "w17", "memory"157 : "w15", "w16", "w17", "memory"
163 );158 );
159 unreachable;
164}160}
165fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 {161fn __aarch64_swp1_acq() align(16) callconv(.Naked) void {
166 @setRuntimeSafety(false);162 @setRuntimeSafety(false);
167 return asm volatile (163 asm volatile (
168 \\ cbz w16, 8f164 \\ cbz w16, 8f
169 \\ .inst 0x38208020 + 0x00000000 + 0x800000165 \\ .inst 0x38208020 + 0x00000000 + 0x800000
170 \\ cbz wzr, 1f166 \\ ret
171 \\8:167 \\8:
172 \\ mov w16, w0168 \\ mov w16, w0
173 \\0:169 \\0:
...@@ -175,19 +171,19 @@ fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -175,19 +171,19 @@ fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
175 \\ stxrb w17, w16, [x1]171 \\ stxrb w17, w16, [x1]
176 \\ cbnz w17, 0b172 \\ cbnz w17, 0b
177 \\1:173 \\1:
178 : [ret] "={w0}" (-> u8),174 \\ ret
179 : [val] "{w0}" (val),175 :
180 [ptr] "{x1}" (ptr),176 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
181 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
182 : "w15", "w16", "w17", "memory"177 : "w15", "w16", "w17", "memory"
183 );178 );
179 unreachable;
184}180}
185fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 {181fn __aarch64_ldadd1_acq() align(16) callconv(.Naked) void {
186 @setRuntimeSafety(false);182 @setRuntimeSafety(false);
187 return asm volatile (183 asm volatile (
188 \\ cbz w16, 8f184 \\ cbz w16, 8f
189 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000185 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
190 \\ cbz wzr, 1f186 \\ ret
191 \\8:187 \\8:
192 \\ mov w16, w0188 \\ mov w16, w0
193 \\0:189 \\0:
...@@ -196,19 +192,19 @@ fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -196,19 +192,19 @@ fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
196 \\ stxrb w15, w17, [x1]192 \\ stxrb w15, w17, [x1]
197 \\ cbnz w15, 0b193 \\ cbnz w15, 0b
198 \\1:194 \\1:
199 : [ret] "={w0}" (-> u8),195 \\ ret
200 : [val] "{w0}" (val),196 :
201 [ptr] "{x1}" (ptr),197 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
202 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
203 : "w15", "w16", "w17", "memory"198 : "w15", "w16", "w17", "memory"
204 );199 );
200 unreachable;
205}201}
206fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 {202fn __aarch64_ldclr1_acq() align(16) callconv(.Naked) void {
207 @setRuntimeSafety(false);203 @setRuntimeSafety(false);
208 return asm volatile (204 asm volatile (
209 \\ cbz w16, 8f205 \\ cbz w16, 8f
210 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000206 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
211 \\ cbz wzr, 1f207 \\ ret
212 \\8:208 \\8:
213 \\ mov w16, w0209 \\ mov w16, w0
214 \\0:210 \\0:
...@@ -217,19 +213,19 @@ fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -217,19 +213,19 @@ fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
217 \\ stxrb w15, w17, [x1]213 \\ stxrb w15, w17, [x1]
218 \\ cbnz w15, 0b214 \\ cbnz w15, 0b
219 \\1:215 \\1:
220 : [ret] "={w0}" (-> u8),216 \\ ret
221 : [val] "{w0}" (val),217 :
222 [ptr] "{x1}" (ptr),218 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
223 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
224 : "w15", "w16", "w17", "memory"219 : "w15", "w16", "w17", "memory"
225 );220 );
221 unreachable;
226}222}
227fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 {223fn __aarch64_ldeor1_acq() align(16) callconv(.Naked) void {
228 @setRuntimeSafety(false);224 @setRuntimeSafety(false);
229 return asm volatile (225 asm volatile (
230 \\ cbz w16, 8f226 \\ cbz w16, 8f
231 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000227 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
232 \\ cbz wzr, 1f228 \\ ret
233 \\8:229 \\8:
234 \\ mov w16, w0230 \\ mov w16, w0
235 \\0:231 \\0:
...@@ -238,19 +234,19 @@ fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -238,19 +234,19 @@ fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
238 \\ stxrb w15, w17, [x1]234 \\ stxrb w15, w17, [x1]
239 \\ cbnz w15, 0b235 \\ cbnz w15, 0b
240 \\1:236 \\1:
241 : [ret] "={w0}" (-> u8),237 \\ ret
242 : [val] "{w0}" (val),238 :
243 [ptr] "{x1}" (ptr),239 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
244 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
245 : "w15", "w16", "w17", "memory"240 : "w15", "w16", "w17", "memory"
246 );241 );
242 unreachable;
247}243}
248fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 {244fn __aarch64_ldset1_acq() align(16) callconv(.Naked) void {
249 @setRuntimeSafety(false);245 @setRuntimeSafety(false);
250 return asm volatile (246 asm volatile (
251 \\ cbz w16, 8f247 \\ cbz w16, 8f
252 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000248 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
253 \\ cbz wzr, 1f249 \\ ret
254 \\8:250 \\8:
255 \\ mov w16, w0251 \\ mov w16, w0
256 \\0:252 \\0:
...@@ -259,20 +255,19 @@ fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -259,20 +255,19 @@ fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
259 \\ stxrb w15, w17, [x1]255 \\ stxrb w15, w17, [x1]
260 \\ cbnz w15, 0b256 \\ cbnz w15, 0b
261 \\1:257 \\1:
262 : [ret] "={w0}" (-> u8),258 \\ ret
263 : [val] "{w0}" (val),259 :
264 [ptr] "{x1}" (ptr),260 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
265 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
266 : "w15", "w16", "w17", "memory"261 : "w15", "w16", "w17", "memory"
267 );262 );
263 unreachable;
268}264}
269fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {265fn __aarch64_cas1_rel() align(16) callconv(.Naked) void {
270 @setRuntimeSafety(false);266 @setRuntimeSafety(false);
271 return asm volatile (267 asm volatile (
272 \\ cbz w16, 8f268 \\ cbz w16, 8f
273 \\ .inst 0x08a07c41 + 0x00000000 + 0x008000269 \\ .inst 0x08a07c41 + 0x00000000 + 0x008000
274 \\270 \\ ret
275 \\ cbz wzr, 1f
276 \\8:271 \\8:
277 \\ uxtb w16, w0272 \\ uxtb w16, w0
278 \\0:273 \\0:
...@@ -282,20 +277,19 @@ fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {...@@ -282,20 +277,19 @@ fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
282 \\ stlxrb w17, w1, [x2]277 \\ stlxrb w17, w1, [x2]
283 \\ cbnz w17, 0b278 \\ cbnz w17, 0b
284 \\1:279 \\1:
285 : [ret] "={w0}" (-> u8),280 \\ ret
286 : [expected] "{w0}" (expected),281 :
287 [desired] "{w1}" (desired),282 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
288 [ptr] "{x2}" (ptr),
289 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
290 : "w15", "w16", "w17", "memory"283 : "w15", "w16", "w17", "memory"
291 );284 );
285 unreachable;
292}286}
293fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 {287fn __aarch64_swp1_rel() align(16) callconv(.Naked) void {
294 @setRuntimeSafety(false);288 @setRuntimeSafety(false);
295 return asm volatile (289 asm volatile (
296 \\ cbz w16, 8f290 \\ cbz w16, 8f
297 \\ .inst 0x38208020 + 0x00000000 + 0x400000291 \\ .inst 0x38208020 + 0x00000000 + 0x400000
298 \\ cbz wzr, 1f292 \\ ret
299 \\8:293 \\8:
300 \\ mov w16, w0294 \\ mov w16, w0
301 \\0:295 \\0:
...@@ -303,19 +297,19 @@ fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -303,19 +297,19 @@ fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
303 \\ stlxrb w17, w16, [x1]297 \\ stlxrb w17, w16, [x1]
304 \\ cbnz w17, 0b298 \\ cbnz w17, 0b
305 \\1:299 \\1:
306 : [ret] "={w0}" (-> u8),300 \\ ret
307 : [val] "{w0}" (val),301 :
308 [ptr] "{x1}" (ptr),302 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
309 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
310 : "w15", "w16", "w17", "memory"303 : "w15", "w16", "w17", "memory"
311 );304 );
305 unreachable;
312}306}
313fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 {307fn __aarch64_ldadd1_rel() align(16) callconv(.Naked) void {
314 @setRuntimeSafety(false);308 @setRuntimeSafety(false);
315 return asm volatile (309 asm volatile (
316 \\ cbz w16, 8f310 \\ cbz w16, 8f
317 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000311 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
318 \\ cbz wzr, 1f312 \\ ret
319 \\8:313 \\8:
320 \\ mov w16, w0314 \\ mov w16, w0
321 \\0:315 \\0:
...@@ -324,19 +318,19 @@ fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -324,19 +318,19 @@ fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
324 \\ stlxrb w15, w17, [x1]318 \\ stlxrb w15, w17, [x1]
325 \\ cbnz w15, 0b319 \\ cbnz w15, 0b
326 \\1:320 \\1:
327 : [ret] "={w0}" (-> u8),321 \\ ret
328 : [val] "{w0}" (val),322 :
329 [ptr] "{x1}" (ptr),323 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
330 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
331 : "w15", "w16", "w17", "memory"324 : "w15", "w16", "w17", "memory"
332 );325 );
326 unreachable;
333}327}
334fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 {328fn __aarch64_ldclr1_rel() align(16) callconv(.Naked) void {
335 @setRuntimeSafety(false);329 @setRuntimeSafety(false);
336 return asm volatile (330 asm volatile (
337 \\ cbz w16, 8f331 \\ cbz w16, 8f
338 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000332 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
339 \\ cbz wzr, 1f333 \\ ret
340 \\8:334 \\8:
341 \\ mov w16, w0335 \\ mov w16, w0
342 \\0:336 \\0:
...@@ -345,19 +339,19 @@ fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -345,19 +339,19 @@ fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
345 \\ stlxrb w15, w17, [x1]339 \\ stlxrb w15, w17, [x1]
346 \\ cbnz w15, 0b340 \\ cbnz w15, 0b
347 \\1:341 \\1:
348 : [ret] "={w0}" (-> u8),342 \\ ret
349 : [val] "{w0}" (val),343 :
350 [ptr] "{x1}" (ptr),344 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
351 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
352 : "w15", "w16", "w17", "memory"345 : "w15", "w16", "w17", "memory"
353 );346 );
347 unreachable;
354}348}
355fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 {349fn __aarch64_ldeor1_rel() align(16) callconv(.Naked) void {
356 @setRuntimeSafety(false);350 @setRuntimeSafety(false);
357 return asm volatile (351 asm volatile (
358 \\ cbz w16, 8f352 \\ cbz w16, 8f
359 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000353 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
360 \\ cbz wzr, 1f354 \\ ret
361 \\8:355 \\8:
362 \\ mov w16, w0356 \\ mov w16, w0
363 \\0:357 \\0:
...@@ -366,19 +360,19 @@ fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -366,19 +360,19 @@ fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
366 \\ stlxrb w15, w17, [x1]360 \\ stlxrb w15, w17, [x1]
367 \\ cbnz w15, 0b361 \\ cbnz w15, 0b
368 \\1:362 \\1:
369 : [ret] "={w0}" (-> u8),363 \\ ret
370 : [val] "{w0}" (val),364 :
371 [ptr] "{x1}" (ptr),365 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
372 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
373 : "w15", "w16", "w17", "memory"366 : "w15", "w16", "w17", "memory"
374 );367 );
368 unreachable;
375}369}
376fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 {370fn __aarch64_ldset1_rel() align(16) callconv(.Naked) void {
377 @setRuntimeSafety(false);371 @setRuntimeSafety(false);
378 return asm volatile (372 asm volatile (
379 \\ cbz w16, 8f373 \\ cbz w16, 8f
380 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000374 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
381 \\ cbz wzr, 1f375 \\ ret
382 \\8:376 \\8:
383 \\ mov w16, w0377 \\ mov w16, w0
384 \\0:378 \\0:
...@@ -387,20 +381,19 @@ fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -387,20 +381,19 @@ fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
387 \\ stlxrb w15, w17, [x1]381 \\ stlxrb w15, w17, [x1]
388 \\ cbnz w15, 0b382 \\ cbnz w15, 0b
389 \\1:383 \\1:
390 : [ret] "={w0}" (-> u8),384 \\ ret
391 : [val] "{w0}" (val),385 :
392 [ptr] "{x1}" (ptr),386 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
393 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
394 : "w15", "w16", "w17", "memory"387 : "w15", "w16", "w17", "memory"
395 );388 );
389 unreachable;
396}390}
397fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {391fn __aarch64_cas1_acq_rel() align(16) callconv(.Naked) void {
398 @setRuntimeSafety(false);392 @setRuntimeSafety(false);
399 return asm volatile (393 asm volatile (
400 \\ cbz w16, 8f394 \\ cbz w16, 8f
401 \\ .inst 0x08a07c41 + 0x00000000 + 0x408000395 \\ .inst 0x08a07c41 + 0x00000000 + 0x408000
402 \\396 \\ ret
403 \\ cbz wzr, 1f
404 \\8:397 \\8:
405 \\ uxtb w16, w0398 \\ uxtb w16, w0
406 \\0:399 \\0:
...@@ -410,20 +403,19 @@ fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {...@@ -410,20 +403,19 @@ fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
410 \\ stlxrb w17, w1, [x2]403 \\ stlxrb w17, w1, [x2]
411 \\ cbnz w17, 0b404 \\ cbnz w17, 0b
412 \\1:405 \\1:
413 : [ret] "={w0}" (-> u8),406 \\ ret
414 : [expected] "{w0}" (expected),407 :
415 [desired] "{w1}" (desired),408 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
416 [ptr] "{x2}" (ptr),
417 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
418 : "w15", "w16", "w17", "memory"409 : "w15", "w16", "w17", "memory"
419 );410 );
411 unreachable;
420}412}
421fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {413fn __aarch64_swp1_acq_rel() align(16) callconv(.Naked) void {
422 @setRuntimeSafety(false);414 @setRuntimeSafety(false);
423 return asm volatile (415 asm volatile (
424 \\ cbz w16, 8f416 \\ cbz w16, 8f
425 \\ .inst 0x38208020 + 0x00000000 + 0xc00000417 \\ .inst 0x38208020 + 0x00000000 + 0xc00000
426 \\ cbz wzr, 1f418 \\ ret
427 \\8:419 \\8:
428 \\ mov w16, w0420 \\ mov w16, w0
429 \\0:421 \\0:
...@@ -431,19 +423,19 @@ fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -431,19 +423,19 @@ fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
431 \\ stlxrb w17, w16, [x1]423 \\ stlxrb w17, w16, [x1]
432 \\ cbnz w17, 0b424 \\ cbnz w17, 0b
433 \\1:425 \\1:
434 : [ret] "={w0}" (-> u8),426 \\ ret
435 : [val] "{w0}" (val),427 :
436 [ptr] "{x1}" (ptr),428 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
437 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
438 : "w15", "w16", "w17", "memory"429 : "w15", "w16", "w17", "memory"
439 );430 );
431 unreachable;
440}432}
441fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {433fn __aarch64_ldadd1_acq_rel() align(16) callconv(.Naked) void {
442 @setRuntimeSafety(false);434 @setRuntimeSafety(false);
443 return asm volatile (435 asm volatile (
444 \\ cbz w16, 8f436 \\ cbz w16, 8f
445 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000437 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
446 \\ cbz wzr, 1f438 \\ ret
447 \\8:439 \\8:
448 \\ mov w16, w0440 \\ mov w16, w0
449 \\0:441 \\0:
...@@ -452,19 +444,19 @@ fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -452,19 +444,19 @@ fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
452 \\ stlxrb w15, w17, [x1]444 \\ stlxrb w15, w17, [x1]
453 \\ cbnz w15, 0b445 \\ cbnz w15, 0b
454 \\1:446 \\1:
455 : [ret] "={w0}" (-> u8),447 \\ ret
456 : [val] "{w0}" (val),448 :
457 [ptr] "{x1}" (ptr),449 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
458 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
459 : "w15", "w16", "w17", "memory"450 : "w15", "w16", "w17", "memory"
460 );451 );
452 unreachable;
461}453}
462fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {454fn __aarch64_ldclr1_acq_rel() align(16) callconv(.Naked) void {
463 @setRuntimeSafety(false);455 @setRuntimeSafety(false);
464 return asm volatile (456 asm volatile (
465 \\ cbz w16, 8f457 \\ cbz w16, 8f
466 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000458 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
467 \\ cbz wzr, 1f459 \\ ret
468 \\8:460 \\8:
469 \\ mov w16, w0461 \\ mov w16, w0
470 \\0:462 \\0:
...@@ -473,19 +465,19 @@ fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -473,19 +465,19 @@ fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
473 \\ stlxrb w15, w17, [x1]465 \\ stlxrb w15, w17, [x1]
474 \\ cbnz w15, 0b466 \\ cbnz w15, 0b
475 \\1:467 \\1:
476 : [ret] "={w0}" (-> u8),468 \\ ret
477 : [val] "{w0}" (val),469 :
478 [ptr] "{x1}" (ptr),470 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
479 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
480 : "w15", "w16", "w17", "memory"471 : "w15", "w16", "w17", "memory"
481 );472 );
473 unreachable;
482}474}
483fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {475fn __aarch64_ldeor1_acq_rel() align(16) callconv(.Naked) void {
484 @setRuntimeSafety(false);476 @setRuntimeSafety(false);
485 return asm volatile (477 asm volatile (
486 \\ cbz w16, 8f478 \\ cbz w16, 8f
487 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000479 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
488 \\ cbz wzr, 1f480 \\ ret
489 \\8:481 \\8:
490 \\ mov w16, w0482 \\ mov w16, w0
491 \\0:483 \\0:
...@@ -494,19 +486,19 @@ fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -494,19 +486,19 @@ fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
494 \\ stlxrb w15, w17, [x1]486 \\ stlxrb w15, w17, [x1]
495 \\ cbnz w15, 0b487 \\ cbnz w15, 0b
496 \\1:488 \\1:
497 : [ret] "={w0}" (-> u8),489 \\ ret
498 : [val] "{w0}" (val),490 :
499 [ptr] "{x1}" (ptr),491 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
500 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
501 : "w15", "w16", "w17", "memory"492 : "w15", "w16", "w17", "memory"
502 );493 );
494 unreachable;
503}495}
504fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {496fn __aarch64_ldset1_acq_rel() align(16) callconv(.Naked) void {
505 @setRuntimeSafety(false);497 @setRuntimeSafety(false);
506 return asm volatile (498 asm volatile (
507 \\ cbz w16, 8f499 \\ cbz w16, 8f
508 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000500 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
509 \\ cbz wzr, 1f501 \\ ret
510 \\8:502 \\8:
511 \\ mov w16, w0503 \\ mov w16, w0
512 \\0:504 \\0:
...@@ -515,20 +507,19 @@ fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {...@@ -515,20 +507,19 @@ fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
515 \\ stlxrb w15, w17, [x1]507 \\ stlxrb w15, w17, [x1]
516 \\ cbnz w15, 0b508 \\ cbnz w15, 0b
517 \\1:509 \\1:
518 : [ret] "={w0}" (-> u8),510 \\ ret
519 : [val] "{w0}" (val),511 :
520 [ptr] "{x1}" (ptr),512 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
521 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
522 : "w15", "w16", "w17", "memory"513 : "w15", "w16", "w17", "memory"
523 );514 );
515 unreachable;
524}516}
525fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {517fn __aarch64_cas2_relax() align(16) callconv(.Naked) void {
526 @setRuntimeSafety(false);518 @setRuntimeSafety(false);
527 return asm volatile (519 asm volatile (
528 \\ cbz w16, 8f520 \\ cbz w16, 8f
529 \\ .inst 0x08a07c41 + 0x40000000 + 0x000000521 \\ .inst 0x08a07c41 + 0x40000000 + 0x000000
530 \\522 \\ ret
531 \\ cbz wzr, 1f
532 \\8:523 \\8:
533 \\ uxth w16, w0524 \\ uxth w16, w0
534 \\0:525 \\0:
...@@ -538,20 +529,19 @@ fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16...@@ -538,20 +529,19 @@ fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16
538 \\ stxrh w17, w1, [x2]529 \\ stxrh w17, w1, [x2]
539 \\ cbnz w17, 0b530 \\ cbnz w17, 0b
540 \\1:531 \\1:
541 : [ret] "={w0}" (-> u16),532 \\ ret
542 : [expected] "{w0}" (expected),533 :
543 [desired] "{w1}" (desired),534 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
544 [ptr] "{x2}" (ptr),
545 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
546 : "w15", "w16", "w17", "memory"535 : "w15", "w16", "w17", "memory"
547 );536 );
537 unreachable;
548}538}
549fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 {539fn __aarch64_swp2_relax() align(16) callconv(.Naked) void {
550 @setRuntimeSafety(false);540 @setRuntimeSafety(false);
551 return asm volatile (541 asm volatile (
552 \\ cbz w16, 8f542 \\ cbz w16, 8f
553 \\ .inst 0x38208020 + 0x40000000 + 0x000000543 \\ .inst 0x38208020 + 0x40000000 + 0x000000
554 \\ cbz wzr, 1f544 \\ ret
555 \\8:545 \\8:
556 \\ mov w16, w0546 \\ mov w16, w0
557 \\0:547 \\0:
...@@ -559,19 +549,19 @@ fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -559,19 +549,19 @@ fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
559 \\ stxrh w17, w16, [x1]549 \\ stxrh w17, w16, [x1]
560 \\ cbnz w17, 0b550 \\ cbnz w17, 0b
561 \\1:551 \\1:
562 : [ret] "={w0}" (-> u16),552 \\ ret
563 : [val] "{w0}" (val),553 :
564 [ptr] "{x1}" (ptr),554 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
565 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
566 : "w15", "w16", "w17", "memory"555 : "w15", "w16", "w17", "memory"
567 );556 );
557 unreachable;
568}558}
569fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 {559fn __aarch64_ldadd2_relax() align(16) callconv(.Naked) void {
570 @setRuntimeSafety(false);560 @setRuntimeSafety(false);
571 return asm volatile (561 asm volatile (
572 \\ cbz w16, 8f562 \\ cbz w16, 8f
573 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000563 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
574 \\ cbz wzr, 1f564 \\ ret
575 \\8:565 \\8:
576 \\ mov w16, w0566 \\ mov w16, w0
577 \\0:567 \\0:
...@@ -580,19 +570,19 @@ fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -580,19 +570,19 @@ fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
580 \\ stxrh w15, w17, [x1]570 \\ stxrh w15, w17, [x1]
581 \\ cbnz w15, 0b571 \\ cbnz w15, 0b
582 \\1:572 \\1:
583 : [ret] "={w0}" (-> u16),573 \\ ret
584 : [val] "{w0}" (val),574 :
585 [ptr] "{x1}" (ptr),575 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
586 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
587 : "w15", "w16", "w17", "memory"576 : "w15", "w16", "w17", "memory"
588 );577 );
578 unreachable;
589}579}
590fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 {580fn __aarch64_ldclr2_relax() align(16) callconv(.Naked) void {
591 @setRuntimeSafety(false);581 @setRuntimeSafety(false);
592 return asm volatile (582 asm volatile (
593 \\ cbz w16, 8f583 \\ cbz w16, 8f
594 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000584 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
595 \\ cbz wzr, 1f585 \\ ret
596 \\8:586 \\8:
597 \\ mov w16, w0587 \\ mov w16, w0
598 \\0:588 \\0:
...@@ -601,19 +591,19 @@ fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -601,19 +591,19 @@ fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
601 \\ stxrh w15, w17, [x1]591 \\ stxrh w15, w17, [x1]
602 \\ cbnz w15, 0b592 \\ cbnz w15, 0b
603 \\1:593 \\1:
604 : [ret] "={w0}" (-> u16),594 \\ ret
605 : [val] "{w0}" (val),595 :
606 [ptr] "{x1}" (ptr),596 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
607 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
608 : "w15", "w16", "w17", "memory"597 : "w15", "w16", "w17", "memory"
609 );598 );
599 unreachable;
610}600}
611fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 {601fn __aarch64_ldeor2_relax() align(16) callconv(.Naked) void {
612 @setRuntimeSafety(false);602 @setRuntimeSafety(false);
613 return asm volatile (603 asm volatile (
614 \\ cbz w16, 8f604 \\ cbz w16, 8f
615 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000605 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
616 \\ cbz wzr, 1f606 \\ ret
617 \\8:607 \\8:
618 \\ mov w16, w0608 \\ mov w16, w0
619 \\0:609 \\0:
...@@ -622,19 +612,19 @@ fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -622,19 +612,19 @@ fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
622 \\ stxrh w15, w17, [x1]612 \\ stxrh w15, w17, [x1]
623 \\ cbnz w15, 0b613 \\ cbnz w15, 0b
624 \\1:614 \\1:
625 : [ret] "={w0}" (-> u16),615 \\ ret
626 : [val] "{w0}" (val),616 :
627 [ptr] "{x1}" (ptr),617 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
628 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
629 : "w15", "w16", "w17", "memory"618 : "w15", "w16", "w17", "memory"
630 );619 );
620 unreachable;
631}621}
632fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 {622fn __aarch64_ldset2_relax() align(16) callconv(.Naked) void {
633 @setRuntimeSafety(false);623 @setRuntimeSafety(false);
634 return asm volatile (624 asm volatile (
635 \\ cbz w16, 8f625 \\ cbz w16, 8f
636 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000626 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
637 \\ cbz wzr, 1f627 \\ ret
638 \\8:628 \\8:
639 \\ mov w16, w0629 \\ mov w16, w0
640 \\0:630 \\0:
...@@ -643,20 +633,19 @@ fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -643,20 +633,19 @@ fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
643 \\ stxrh w15, w17, [x1]633 \\ stxrh w15, w17, [x1]
644 \\ cbnz w15, 0b634 \\ cbnz w15, 0b
645 \\1:635 \\1:
646 : [ret] "={w0}" (-> u16),636 \\ ret
647 : [val] "{w0}" (val),637 :
648 [ptr] "{x1}" (ptr),638 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
649 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
650 : "w15", "w16", "w17", "memory"639 : "w15", "w16", "w17", "memory"
651 );640 );
641 unreachable;
652}642}
653fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {643fn __aarch64_cas2_acq() align(16) callconv(.Naked) void {
654 @setRuntimeSafety(false);644 @setRuntimeSafety(false);
655 return asm volatile (645 asm volatile (
656 \\ cbz w16, 8f646 \\ cbz w16, 8f
657 \\ .inst 0x08a07c41 + 0x40000000 + 0x400000647 \\ .inst 0x08a07c41 + 0x40000000 + 0x400000
658 \\648 \\ ret
659 \\ cbz wzr, 1f
660 \\8:649 \\8:
661 \\ uxth w16, w0650 \\ uxth w16, w0
662 \\0:651 \\0:
...@@ -666,20 +655,19 @@ fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {...@@ -666,20 +655,19 @@ fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
666 \\ stxrh w17, w1, [x2]655 \\ stxrh w17, w1, [x2]
667 \\ cbnz w17, 0b656 \\ cbnz w17, 0b
668 \\1:657 \\1:
669 : [ret] "={w0}" (-> u16),658 \\ ret
670 : [expected] "{w0}" (expected),659 :
671 [desired] "{w1}" (desired),660 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
672 [ptr] "{x2}" (ptr),
673 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
674 : "w15", "w16", "w17", "memory"661 : "w15", "w16", "w17", "memory"
675 );662 );
663 unreachable;
676}664}
677fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 {665fn __aarch64_swp2_acq() align(16) callconv(.Naked) void {
678 @setRuntimeSafety(false);666 @setRuntimeSafety(false);
679 return asm volatile (667 asm volatile (
680 \\ cbz w16, 8f668 \\ cbz w16, 8f
681 \\ .inst 0x38208020 + 0x40000000 + 0x800000669 \\ .inst 0x38208020 + 0x40000000 + 0x800000
682 \\ cbz wzr, 1f670 \\ ret
683 \\8:671 \\8:
684 \\ mov w16, w0672 \\ mov w16, w0
685 \\0:673 \\0:
...@@ -687,19 +675,19 @@ fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -687,19 +675,19 @@ fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
687 \\ stxrh w17, w16, [x1]675 \\ stxrh w17, w16, [x1]
688 \\ cbnz w17, 0b676 \\ cbnz w17, 0b
689 \\1:677 \\1:
690 : [ret] "={w0}" (-> u16),678 \\ ret
691 : [val] "{w0}" (val),679 :
692 [ptr] "{x1}" (ptr),680 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
693 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
694 : "w15", "w16", "w17", "memory"681 : "w15", "w16", "w17", "memory"
695 );682 );
683 unreachable;
696}684}
697fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 {685fn __aarch64_ldadd2_acq() align(16) callconv(.Naked) void {
698 @setRuntimeSafety(false);686 @setRuntimeSafety(false);
699 return asm volatile (687 asm volatile (
700 \\ cbz w16, 8f688 \\ cbz w16, 8f
701 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000689 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
702 \\ cbz wzr, 1f690 \\ ret
703 \\8:691 \\8:
704 \\ mov w16, w0692 \\ mov w16, w0
705 \\0:693 \\0:
...@@ -708,19 +696,19 @@ fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -708,19 +696,19 @@ fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
708 \\ stxrh w15, w17, [x1]696 \\ stxrh w15, w17, [x1]
709 \\ cbnz w15, 0b697 \\ cbnz w15, 0b
710 \\1:698 \\1:
711 : [ret] "={w0}" (-> u16),699 \\ ret
712 : [val] "{w0}" (val),700 :
713 [ptr] "{x1}" (ptr),701 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
714 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
715 : "w15", "w16", "w17", "memory"702 : "w15", "w16", "w17", "memory"
716 );703 );
704 unreachable;
717}705}
718fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 {706fn __aarch64_ldclr2_acq() align(16) callconv(.Naked) void {
719 @setRuntimeSafety(false);707 @setRuntimeSafety(false);
720 return asm volatile (708 asm volatile (
721 \\ cbz w16, 8f709 \\ cbz w16, 8f
722 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000710 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
723 \\ cbz wzr, 1f711 \\ ret
724 \\8:712 \\8:
725 \\ mov w16, w0713 \\ mov w16, w0
726 \\0:714 \\0:
...@@ -729,19 +717,19 @@ fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -729,19 +717,19 @@ fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
729 \\ stxrh w15, w17, [x1]717 \\ stxrh w15, w17, [x1]
730 \\ cbnz w15, 0b718 \\ cbnz w15, 0b
731 \\1:719 \\1:
732 : [ret] "={w0}" (-> u16),720 \\ ret
733 : [val] "{w0}" (val),721 :
734 [ptr] "{x1}" (ptr),722 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
735 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
736 : "w15", "w16", "w17", "memory"723 : "w15", "w16", "w17", "memory"
737 );724 );
725 unreachable;
738}726}
739fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 {727fn __aarch64_ldeor2_acq() align(16) callconv(.Naked) void {
740 @setRuntimeSafety(false);728 @setRuntimeSafety(false);
741 return asm volatile (729 asm volatile (
742 \\ cbz w16, 8f730 \\ cbz w16, 8f
743 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000731 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
744 \\ cbz wzr, 1f732 \\ ret
745 \\8:733 \\8:
746 \\ mov w16, w0734 \\ mov w16, w0
747 \\0:735 \\0:
...@@ -750,19 +738,19 @@ fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -750,19 +738,19 @@ fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
750 \\ stxrh w15, w17, [x1]738 \\ stxrh w15, w17, [x1]
751 \\ cbnz w15, 0b739 \\ cbnz w15, 0b
752 \\1:740 \\1:
753 : [ret] "={w0}" (-> u16),741 \\ ret
754 : [val] "{w0}" (val),742 :
755 [ptr] "{x1}" (ptr),743 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
756 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
757 : "w15", "w16", "w17", "memory"744 : "w15", "w16", "w17", "memory"
758 );745 );
746 unreachable;
759}747}
760fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 {748fn __aarch64_ldset2_acq() align(16) callconv(.Naked) void {
761 @setRuntimeSafety(false);749 @setRuntimeSafety(false);
762 return asm volatile (750 asm volatile (
763 \\ cbz w16, 8f751 \\ cbz w16, 8f
764 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000752 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
765 \\ cbz wzr, 1f753 \\ ret
766 \\8:754 \\8:
767 \\ mov w16, w0755 \\ mov w16, w0
768 \\0:756 \\0:
...@@ -771,20 +759,19 @@ fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -771,20 +759,19 @@ fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
771 \\ stxrh w15, w17, [x1]759 \\ stxrh w15, w17, [x1]
772 \\ cbnz w15, 0b760 \\ cbnz w15, 0b
773 \\1:761 \\1:
774 : [ret] "={w0}" (-> u16),762 \\ ret
775 : [val] "{w0}" (val),763 :
776 [ptr] "{x1}" (ptr),764 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
777 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
778 : "w15", "w16", "w17", "memory"765 : "w15", "w16", "w17", "memory"
779 );766 );
767 unreachable;
780}768}
781fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {769fn __aarch64_cas2_rel() align(16) callconv(.Naked) void {
782 @setRuntimeSafety(false);770 @setRuntimeSafety(false);
783 return asm volatile (771 asm volatile (
784 \\ cbz w16, 8f772 \\ cbz w16, 8f
785 \\ .inst 0x08a07c41 + 0x40000000 + 0x008000773 \\ .inst 0x08a07c41 + 0x40000000 + 0x008000
786 \\774 \\ ret
787 \\ cbz wzr, 1f
788 \\8:775 \\8:
789 \\ uxth w16, w0776 \\ uxth w16, w0
790 \\0:777 \\0:
...@@ -794,20 +781,19 @@ fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {...@@ -794,20 +781,19 @@ fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
794 \\ stlxrh w17, w1, [x2]781 \\ stlxrh w17, w1, [x2]
795 \\ cbnz w17, 0b782 \\ cbnz w17, 0b
796 \\1:783 \\1:
797 : [ret] "={w0}" (-> u16),784 \\ ret
798 : [expected] "{w0}" (expected),785 :
799 [desired] "{w1}" (desired),786 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
800 [ptr] "{x2}" (ptr),
801 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
802 : "w15", "w16", "w17", "memory"787 : "w15", "w16", "w17", "memory"
803 );788 );
789 unreachable;
804}790}
805fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 {791fn __aarch64_swp2_rel() align(16) callconv(.Naked) void {
806 @setRuntimeSafety(false);792 @setRuntimeSafety(false);
807 return asm volatile (793 asm volatile (
808 \\ cbz w16, 8f794 \\ cbz w16, 8f
809 \\ .inst 0x38208020 + 0x40000000 + 0x400000795 \\ .inst 0x38208020 + 0x40000000 + 0x400000
810 \\ cbz wzr, 1f796 \\ ret
811 \\8:797 \\8:
812 \\ mov w16, w0798 \\ mov w16, w0
813 \\0:799 \\0:
...@@ -815,19 +801,19 @@ fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -815,19 +801,19 @@ fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
815 \\ stlxrh w17, w16, [x1]801 \\ stlxrh w17, w16, [x1]
816 \\ cbnz w17, 0b802 \\ cbnz w17, 0b
817 \\1:803 \\1:
818 : [ret] "={w0}" (-> u16),804 \\ ret
819 : [val] "{w0}" (val),805 :
820 [ptr] "{x1}" (ptr),806 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
821 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
822 : "w15", "w16", "w17", "memory"807 : "w15", "w16", "w17", "memory"
823 );808 );
809 unreachable;
824}810}
825fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 {811fn __aarch64_ldadd2_rel() align(16) callconv(.Naked) void {
826 @setRuntimeSafety(false);812 @setRuntimeSafety(false);
827 return asm volatile (813 asm volatile (
828 \\ cbz w16, 8f814 \\ cbz w16, 8f
829 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000815 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
830 \\ cbz wzr, 1f816 \\ ret
831 \\8:817 \\8:
832 \\ mov w16, w0818 \\ mov w16, w0
833 \\0:819 \\0:
...@@ -836,19 +822,19 @@ fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -836,19 +822,19 @@ fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
836 \\ stlxrh w15, w17, [x1]822 \\ stlxrh w15, w17, [x1]
837 \\ cbnz w15, 0b823 \\ cbnz w15, 0b
838 \\1:824 \\1:
839 : [ret] "={w0}" (-> u16),825 \\ ret
840 : [val] "{w0}" (val),826 :
841 [ptr] "{x1}" (ptr),827 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
842 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
843 : "w15", "w16", "w17", "memory"828 : "w15", "w16", "w17", "memory"
844 );829 );
830 unreachable;
845}831}
846fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 {832fn __aarch64_ldclr2_rel() align(16) callconv(.Naked) void {
847 @setRuntimeSafety(false);833 @setRuntimeSafety(false);
848 return asm volatile (834 asm volatile (
849 \\ cbz w16, 8f835 \\ cbz w16, 8f
850 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000836 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
851 \\ cbz wzr, 1f837 \\ ret
852 \\8:838 \\8:
853 \\ mov w16, w0839 \\ mov w16, w0
854 \\0:840 \\0:
...@@ -857,19 +843,19 @@ fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -857,19 +843,19 @@ fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
857 \\ stlxrh w15, w17, [x1]843 \\ stlxrh w15, w17, [x1]
858 \\ cbnz w15, 0b844 \\ cbnz w15, 0b
859 \\1:845 \\1:
860 : [ret] "={w0}" (-> u16),846 \\ ret
861 : [val] "{w0}" (val),847 :
862 [ptr] "{x1}" (ptr),848 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
863 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
864 : "w15", "w16", "w17", "memory"849 : "w15", "w16", "w17", "memory"
865 );850 );
851 unreachable;
866}852}
867fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 {853fn __aarch64_ldeor2_rel() align(16) callconv(.Naked) void {
868 @setRuntimeSafety(false);854 @setRuntimeSafety(false);
869 return asm volatile (855 asm volatile (
870 \\ cbz w16, 8f856 \\ cbz w16, 8f
871 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000857 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
872 \\ cbz wzr, 1f858 \\ ret
873 \\8:859 \\8:
874 \\ mov w16, w0860 \\ mov w16, w0
875 \\0:861 \\0:
...@@ -878,19 +864,19 @@ fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -878,19 +864,19 @@ fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
878 \\ stlxrh w15, w17, [x1]864 \\ stlxrh w15, w17, [x1]
879 \\ cbnz w15, 0b865 \\ cbnz w15, 0b
880 \\1:866 \\1:
881 : [ret] "={w0}" (-> u16),867 \\ ret
882 : [val] "{w0}" (val),868 :
883 [ptr] "{x1}" (ptr),869 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
884 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
885 : "w15", "w16", "w17", "memory"870 : "w15", "w16", "w17", "memory"
886 );871 );
872 unreachable;
887}873}
888fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 {874fn __aarch64_ldset2_rel() align(16) callconv(.Naked) void {
889 @setRuntimeSafety(false);875 @setRuntimeSafety(false);
890 return asm volatile (876 asm volatile (
891 \\ cbz w16, 8f877 \\ cbz w16, 8f
892 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000878 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
893 \\ cbz wzr, 1f879 \\ ret
894 \\8:880 \\8:
895 \\ mov w16, w0881 \\ mov w16, w0
896 \\0:882 \\0:
...@@ -899,20 +885,19 @@ fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -899,20 +885,19 @@ fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
899 \\ stlxrh w15, w17, [x1]885 \\ stlxrh w15, w17, [x1]
900 \\ cbnz w15, 0b886 \\ cbnz w15, 0b
901 \\1:887 \\1:
902 : [ret] "={w0}" (-> u16),888 \\ ret
903 : [val] "{w0}" (val),889 :
904 [ptr] "{x1}" (ptr),890 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
905 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
906 : "w15", "w16", "w17", "memory"891 : "w15", "w16", "w17", "memory"
907 );892 );
893 unreachable;
908}894}
909fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {895fn __aarch64_cas2_acq_rel() align(16) callconv(.Naked) void {
910 @setRuntimeSafety(false);896 @setRuntimeSafety(false);
911 return asm volatile (897 asm volatile (
912 \\ cbz w16, 8f898 \\ cbz w16, 8f
913 \\ .inst 0x08a07c41 + 0x40000000 + 0x408000899 \\ .inst 0x08a07c41 + 0x40000000 + 0x408000
914 \\900 \\ ret
915 \\ cbz wzr, 1f
916 \\8:901 \\8:
917 \\ uxth w16, w0902 \\ uxth w16, w0
918 \\0:903 \\0:
...@@ -922,20 +907,19 @@ fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u...@@ -922,20 +907,19 @@ fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u
922 \\ stlxrh w17, w1, [x2]907 \\ stlxrh w17, w1, [x2]
923 \\ cbnz w17, 0b908 \\ cbnz w17, 0b
924 \\1:909 \\1:
925 : [ret] "={w0}" (-> u16),910 \\ ret
926 : [expected] "{w0}" (expected),911 :
927 [desired] "{w1}" (desired),912 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
928 [ptr] "{x2}" (ptr),
929 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
930 : "w15", "w16", "w17", "memory"913 : "w15", "w16", "w17", "memory"
931 );914 );
915 unreachable;
932}916}
933fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {917fn __aarch64_swp2_acq_rel() align(16) callconv(.Naked) void {
934 @setRuntimeSafety(false);918 @setRuntimeSafety(false);
935 return asm volatile (919 asm volatile (
936 \\ cbz w16, 8f920 \\ cbz w16, 8f
937 \\ .inst 0x38208020 + 0x40000000 + 0xc00000921 \\ .inst 0x38208020 + 0x40000000 + 0xc00000
938 \\ cbz wzr, 1f922 \\ ret
939 \\8:923 \\8:
940 \\ mov w16, w0924 \\ mov w16, w0
941 \\0:925 \\0:
...@@ -943,19 +927,19 @@ fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -943,19 +927,19 @@ fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
943 \\ stlxrh w17, w16, [x1]927 \\ stlxrh w17, w16, [x1]
944 \\ cbnz w17, 0b928 \\ cbnz w17, 0b
945 \\1:929 \\1:
946 : [ret] "={w0}" (-> u16),930 \\ ret
947 : [val] "{w0}" (val),931 :
948 [ptr] "{x1}" (ptr),932 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
949 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
950 : "w15", "w16", "w17", "memory"933 : "w15", "w16", "w17", "memory"
951 );934 );
935 unreachable;
952}936}
953fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {937fn __aarch64_ldadd2_acq_rel() align(16) callconv(.Naked) void {
954 @setRuntimeSafety(false);938 @setRuntimeSafety(false);
955 return asm volatile (939 asm volatile (
956 \\ cbz w16, 8f940 \\ cbz w16, 8f
957 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000941 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
958 \\ cbz wzr, 1f942 \\ ret
959 \\8:943 \\8:
960 \\ mov w16, w0944 \\ mov w16, w0
961 \\0:945 \\0:
...@@ -964,19 +948,19 @@ fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -964,19 +948,19 @@ fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
964 \\ stlxrh w15, w17, [x1]948 \\ stlxrh w15, w17, [x1]
965 \\ cbnz w15, 0b949 \\ cbnz w15, 0b
966 \\1:950 \\1:
967 : [ret] "={w0}" (-> u16),951 \\ ret
968 : [val] "{w0}" (val),952 :
969 [ptr] "{x1}" (ptr),953 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
970 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
971 : "w15", "w16", "w17", "memory"954 : "w15", "w16", "w17", "memory"
972 );955 );
956 unreachable;
973}957}
974fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {958fn __aarch64_ldclr2_acq_rel() align(16) callconv(.Naked) void {
975 @setRuntimeSafety(false);959 @setRuntimeSafety(false);
976 return asm volatile (960 asm volatile (
977 \\ cbz w16, 8f961 \\ cbz w16, 8f
978 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000962 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
979 \\ cbz wzr, 1f963 \\ ret
980 \\8:964 \\8:
981 \\ mov w16, w0965 \\ mov w16, w0
982 \\0:966 \\0:
...@@ -985,19 +969,19 @@ fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -985,19 +969,19 @@ fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
985 \\ stlxrh w15, w17, [x1]969 \\ stlxrh w15, w17, [x1]
986 \\ cbnz w15, 0b970 \\ cbnz w15, 0b
987 \\1:971 \\1:
988 : [ret] "={w0}" (-> u16),972 \\ ret
989 : [val] "{w0}" (val),973 :
990 [ptr] "{x1}" (ptr),974 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
991 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
992 : "w15", "w16", "w17", "memory"975 : "w15", "w16", "w17", "memory"
993 );976 );
977 unreachable;
994}978}
995fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {979fn __aarch64_ldeor2_acq_rel() align(16) callconv(.Naked) void {
996 @setRuntimeSafety(false);980 @setRuntimeSafety(false);
997 return asm volatile (981 asm volatile (
998 \\ cbz w16, 8f982 \\ cbz w16, 8f
999 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000983 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
1000 \\ cbz wzr, 1f984 \\ ret
1001 \\8:985 \\8:
1002 \\ mov w16, w0986 \\ mov w16, w0
1003 \\0:987 \\0:
...@@ -1006,19 +990,19 @@ fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -1006,19 +990,19 @@ fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1006 \\ stlxrh w15, w17, [x1]990 \\ stlxrh w15, w17, [x1]
1007 \\ cbnz w15, 0b991 \\ cbnz w15, 0b
1008 \\1:992 \\1:
1009 : [ret] "={w0}" (-> u16),993 \\ ret
1010 : [val] "{w0}" (val),994 :
1011 [ptr] "{x1}" (ptr),995 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1012 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1013 : "w15", "w16", "w17", "memory"996 : "w15", "w16", "w17", "memory"
1014 );997 );
998 unreachable;
1015}999}
1016fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {1000fn __aarch64_ldset2_acq_rel() align(16) callconv(.Naked) void {
1017 @setRuntimeSafety(false);1001 @setRuntimeSafety(false);
1018 return asm volatile (1002 asm volatile (
1019 \\ cbz w16, 8f1003 \\ cbz w16, 8f
1020 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc000001004 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
1021 \\ cbz wzr, 1f1005 \\ ret
1022 \\8:1006 \\8:
1023 \\ mov w16, w01007 \\ mov w16, w0
1024 \\0:1008 \\0:
...@@ -1027,20 +1011,19 @@ fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {...@@ -1027,20 +1011,19 @@ fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1027 \\ stlxrh w15, w17, [x1]1011 \\ stlxrh w15, w17, [x1]
1028 \\ cbnz w15, 0b1012 \\ cbnz w15, 0b
1029 \\1:1013 \\1:
1030 : [ret] "={w0}" (-> u16),1014 \\ ret
1031 : [val] "{w0}" (val),1015 :
1032 [ptr] "{x1}" (ptr),1016 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1033 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1034 : "w15", "w16", "w17", "memory"1017 : "w15", "w16", "w17", "memory"
1035 );1018 );
1019 unreachable;
1036}1020}
1037fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {1021fn __aarch64_cas4_relax() align(16) callconv(.Naked) void {
1038 @setRuntimeSafety(false);1022 @setRuntimeSafety(false);
1039 return asm volatile (1023 asm volatile (
1040 \\ cbz w16, 8f1024 \\ cbz w16, 8f
1041 \\ .inst 0x08a07c41 + 0x80000000 + 0x0000001025 \\ .inst 0x08a07c41 + 0x80000000 + 0x000000
1042 \\1026 \\ ret
1043 \\ cbz wzr, 1f
1044 \\8:1027 \\8:
1045 \\ mov w16, w01028 \\ mov w16, w0
1046 \\0:1029 \\0:
...@@ -1050,20 +1033,19 @@ fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32...@@ -1050,20 +1033,19 @@ fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32
1050 \\ stxr w17, w1, [x2]1033 \\ stxr w17, w1, [x2]
1051 \\ cbnz w17, 0b1034 \\ cbnz w17, 0b
1052 \\1:1035 \\1:
1053 : [ret] "={w0}" (-> u32),1036 \\ ret
1054 : [expected] "{w0}" (expected),1037 :
1055 [desired] "{w1}" (desired),1038 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1056 [ptr] "{x2}" (ptr),
1057 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1058 : "w15", "w16", "w17", "memory"1039 : "w15", "w16", "w17", "memory"
1059 );1040 );
1041 unreachable;
1060}1042}
1061fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 {1043fn __aarch64_swp4_relax() align(16) callconv(.Naked) void {
1062 @setRuntimeSafety(false);1044 @setRuntimeSafety(false);
1063 return asm volatile (1045 asm volatile (
1064 \\ cbz w16, 8f1046 \\ cbz w16, 8f
1065 \\ .inst 0x38208020 + 0x80000000 + 0x0000001047 \\ .inst 0x38208020 + 0x80000000 + 0x000000
1066 \\ cbz wzr, 1f1048 \\ ret
1067 \\8:1049 \\8:
1068 \\ mov w16, w01050 \\ mov w16, w0
1069 \\0:1051 \\0:
...@@ -1071,19 +1053,19 @@ fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1071,19 +1053,19 @@ fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1071 \\ stxr w17, w16, [x1]1053 \\ stxr w17, w16, [x1]
1072 \\ cbnz w17, 0b1054 \\ cbnz w17, 0b
1073 \\1:1055 \\1:
1074 : [ret] "={w0}" (-> u32),1056 \\ ret
1075 : [val] "{w0}" (val),1057 :
1076 [ptr] "{x1}" (ptr),1058 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1077 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1078 : "w15", "w16", "w17", "memory"1059 : "w15", "w16", "w17", "memory"
1079 );1060 );
1061 unreachable;
1080}1062}
1081fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 {1063fn __aarch64_ldadd4_relax() align(16) callconv(.Naked) void {
1082 @setRuntimeSafety(false);1064 @setRuntimeSafety(false);
1083 return asm volatile (1065 asm volatile (
1084 \\ cbz w16, 8f1066 \\ cbz w16, 8f
1085 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x0000001067 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
1086 \\ cbz wzr, 1f1068 \\ ret
1087 \\8:1069 \\8:
1088 \\ mov w16, w01070 \\ mov w16, w0
1089 \\0:1071 \\0:
...@@ -1092,19 +1074,19 @@ fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1092,19 +1074,19 @@ fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1092 \\ stxr w15, w17, [x1]1074 \\ stxr w15, w17, [x1]
1093 \\ cbnz w15, 0b1075 \\ cbnz w15, 0b
1094 \\1:1076 \\1:
1095 : [ret] "={w0}" (-> u32),1077 \\ ret
1096 : [val] "{w0}" (val),1078 :
1097 [ptr] "{x1}" (ptr),1079 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1098 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1099 : "w15", "w16", "w17", "memory"1080 : "w15", "w16", "w17", "memory"
1100 );1081 );
1082 unreachable;
1101}1083}
1102fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 {1084fn __aarch64_ldclr4_relax() align(16) callconv(.Naked) void {
1103 @setRuntimeSafety(false);1085 @setRuntimeSafety(false);
1104 return asm volatile (1086 asm volatile (
1105 \\ cbz w16, 8f1087 \\ cbz w16, 8f
1106 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x0000001088 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
1107 \\ cbz wzr, 1f1089 \\ ret
1108 \\8:1090 \\8:
1109 \\ mov w16, w01091 \\ mov w16, w0
1110 \\0:1092 \\0:
...@@ -1113,19 +1095,19 @@ fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1113,19 +1095,19 @@ fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1113 \\ stxr w15, w17, [x1]1095 \\ stxr w15, w17, [x1]
1114 \\ cbnz w15, 0b1096 \\ cbnz w15, 0b
1115 \\1:1097 \\1:
1116 : [ret] "={w0}" (-> u32),1098 \\ ret
1117 : [val] "{w0}" (val),1099 :
1118 [ptr] "{x1}" (ptr),1100 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1119 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1120 : "w15", "w16", "w17", "memory"1101 : "w15", "w16", "w17", "memory"
1121 );1102 );
1103 unreachable;
1122}1104}
1123fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 {1105fn __aarch64_ldeor4_relax() align(16) callconv(.Naked) void {
1124 @setRuntimeSafety(false);1106 @setRuntimeSafety(false);
1125 return asm volatile (1107 asm volatile (
1126 \\ cbz w16, 8f1108 \\ cbz w16, 8f
1127 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x0000001109 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
1128 \\ cbz wzr, 1f1110 \\ ret
1129 \\8:1111 \\8:
1130 \\ mov w16, w01112 \\ mov w16, w0
1131 \\0:1113 \\0:
...@@ -1134,19 +1116,19 @@ fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1134,19 +1116,19 @@ fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1134 \\ stxr w15, w17, [x1]1116 \\ stxr w15, w17, [x1]
1135 \\ cbnz w15, 0b1117 \\ cbnz w15, 0b
1136 \\1:1118 \\1:
1137 : [ret] "={w0}" (-> u32),1119 \\ ret
1138 : [val] "{w0}" (val),1120 :
1139 [ptr] "{x1}" (ptr),1121 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1140 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1141 : "w15", "w16", "w17", "memory"1122 : "w15", "w16", "w17", "memory"
1142 );1123 );
1124 unreachable;
1143}1125}
1144fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 {1126fn __aarch64_ldset4_relax() align(16) callconv(.Naked) void {
1145 @setRuntimeSafety(false);1127 @setRuntimeSafety(false);
1146 return asm volatile (1128 asm volatile (
1147 \\ cbz w16, 8f1129 \\ cbz w16, 8f
1148 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x0000001130 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
1149 \\ cbz wzr, 1f1131 \\ ret
1150 \\8:1132 \\8:
1151 \\ mov w16, w01133 \\ mov w16, w0
1152 \\0:1134 \\0:
...@@ -1155,20 +1137,19 @@ fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1155,20 +1137,19 @@ fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1155 \\ stxr w15, w17, [x1]1137 \\ stxr w15, w17, [x1]
1156 \\ cbnz w15, 0b1138 \\ cbnz w15, 0b
1157 \\1:1139 \\1:
1158 : [ret] "={w0}" (-> u32),1140 \\ ret
1159 : [val] "{w0}" (val),1141 :
1160 [ptr] "{x1}" (ptr),1142 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1161 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1162 : "w15", "w16", "w17", "memory"1143 : "w15", "w16", "w17", "memory"
1163 );1144 );
1145 unreachable;
1164}1146}
1165fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {1147fn __aarch64_cas4_acq() align(16) callconv(.Naked) void {
1166 @setRuntimeSafety(false);1148 @setRuntimeSafety(false);
1167 return asm volatile (1149 asm volatile (
1168 \\ cbz w16, 8f1150 \\ cbz w16, 8f
1169 \\ .inst 0x08a07c41 + 0x80000000 + 0x4000001151 \\ .inst 0x08a07c41 + 0x80000000 + 0x400000
1170 \\1152 \\ ret
1171 \\ cbz wzr, 1f
1172 \\8:1153 \\8:
1173 \\ mov w16, w01154 \\ mov w16, w0
1174 \\0:1155 \\0:
...@@ -1178,20 +1159,19 @@ fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {...@@ -1178,20 +1159,19 @@ fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1178 \\ stxr w17, w1, [x2]1159 \\ stxr w17, w1, [x2]
1179 \\ cbnz w17, 0b1160 \\ cbnz w17, 0b
1180 \\1:1161 \\1:
1181 : [ret] "={w0}" (-> u32),1162 \\ ret
1182 : [expected] "{w0}" (expected),1163 :
1183 [desired] "{w1}" (desired),1164 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1184 [ptr] "{x2}" (ptr),
1185 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1186 : "w15", "w16", "w17", "memory"1165 : "w15", "w16", "w17", "memory"
1187 );1166 );
1167 unreachable;
1188}1168}
1189fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 {1169fn __aarch64_swp4_acq() align(16) callconv(.Naked) void {
1190 @setRuntimeSafety(false);1170 @setRuntimeSafety(false);
1191 return asm volatile (1171 asm volatile (
1192 \\ cbz w16, 8f1172 \\ cbz w16, 8f
1193 \\ .inst 0x38208020 + 0x80000000 + 0x8000001173 \\ .inst 0x38208020 + 0x80000000 + 0x800000
1194 \\ cbz wzr, 1f1174 \\ ret
1195 \\8:1175 \\8:
1196 \\ mov w16, w01176 \\ mov w16, w0
1197 \\0:1177 \\0:
...@@ -1199,19 +1179,19 @@ fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1199,19 +1179,19 @@ fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1199 \\ stxr w17, w16, [x1]1179 \\ stxr w17, w16, [x1]
1200 \\ cbnz w17, 0b1180 \\ cbnz w17, 0b
1201 \\1:1181 \\1:
1202 : [ret] "={w0}" (-> u32),1182 \\ ret
1203 : [val] "{w0}" (val),1183 :
1204 [ptr] "{x1}" (ptr),1184 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1205 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1206 : "w15", "w16", "w17", "memory"1185 : "w15", "w16", "w17", "memory"
1207 );1186 );
1187 unreachable;
1208}1188}
1209fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 {1189fn __aarch64_ldadd4_acq() align(16) callconv(.Naked) void {
1210 @setRuntimeSafety(false);1190 @setRuntimeSafety(false);
1211 return asm volatile (1191 asm volatile (
1212 \\ cbz w16, 8f1192 \\ cbz w16, 8f
1213 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x8000001193 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
1214 \\ cbz wzr, 1f1194 \\ ret
1215 \\8:1195 \\8:
1216 \\ mov w16, w01196 \\ mov w16, w0
1217 \\0:1197 \\0:
...@@ -1220,19 +1200,19 @@ fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1220,19 +1200,19 @@ fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1220 \\ stxr w15, w17, [x1]1200 \\ stxr w15, w17, [x1]
1221 \\ cbnz w15, 0b1201 \\ cbnz w15, 0b
1222 \\1:1202 \\1:
1223 : [ret] "={w0}" (-> u32),1203 \\ ret
1224 : [val] "{w0}" (val),1204 :
1225 [ptr] "{x1}" (ptr),1205 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1226 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1227 : "w15", "w16", "w17", "memory"1206 : "w15", "w16", "w17", "memory"
1228 );1207 );
1208 unreachable;
1229}1209}
1230fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 {1210fn __aarch64_ldclr4_acq() align(16) callconv(.Naked) void {
1231 @setRuntimeSafety(false);1211 @setRuntimeSafety(false);
1232 return asm volatile (1212 asm volatile (
1233 \\ cbz w16, 8f1213 \\ cbz w16, 8f
1234 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x8000001214 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
1235 \\ cbz wzr, 1f1215 \\ ret
1236 \\8:1216 \\8:
1237 \\ mov w16, w01217 \\ mov w16, w0
1238 \\0:1218 \\0:
...@@ -1241,19 +1221,19 @@ fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1241,19 +1221,19 @@ fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1241 \\ stxr w15, w17, [x1]1221 \\ stxr w15, w17, [x1]
1242 \\ cbnz w15, 0b1222 \\ cbnz w15, 0b
1243 \\1:1223 \\1:
1244 : [ret] "={w0}" (-> u32),1224 \\ ret
1245 : [val] "{w0}" (val),1225 :
1246 [ptr] "{x1}" (ptr),1226 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1247 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1248 : "w15", "w16", "w17", "memory"1227 : "w15", "w16", "w17", "memory"
1249 );1228 );
1229 unreachable;
1250}1230}
1251fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 {1231fn __aarch64_ldeor4_acq() align(16) callconv(.Naked) void {
1252 @setRuntimeSafety(false);1232 @setRuntimeSafety(false);
1253 return asm volatile (1233 asm volatile (
1254 \\ cbz w16, 8f1234 \\ cbz w16, 8f
1255 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x8000001235 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
1256 \\ cbz wzr, 1f1236 \\ ret
1257 \\8:1237 \\8:
1258 \\ mov w16, w01238 \\ mov w16, w0
1259 \\0:1239 \\0:
...@@ -1262,19 +1242,19 @@ fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1262,19 +1242,19 @@ fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1262 \\ stxr w15, w17, [x1]1242 \\ stxr w15, w17, [x1]
1263 \\ cbnz w15, 0b1243 \\ cbnz w15, 0b
1264 \\1:1244 \\1:
1265 : [ret] "={w0}" (-> u32),1245 \\ ret
1266 : [val] "{w0}" (val),1246 :
1267 [ptr] "{x1}" (ptr),1247 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1268 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1269 : "w15", "w16", "w17", "memory"1248 : "w15", "w16", "w17", "memory"
1270 );1249 );
1250 unreachable;
1271}1251}
1272fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 {1252fn __aarch64_ldset4_acq() align(16) callconv(.Naked) void {
1273 @setRuntimeSafety(false);1253 @setRuntimeSafety(false);
1274 return asm volatile (1254 asm volatile (
1275 \\ cbz w16, 8f1255 \\ cbz w16, 8f
1276 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x8000001256 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
1277 \\ cbz wzr, 1f1257 \\ ret
1278 \\8:1258 \\8:
1279 \\ mov w16, w01259 \\ mov w16, w0
1280 \\0:1260 \\0:
...@@ -1283,20 +1263,19 @@ fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1283,20 +1263,19 @@ fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1283 \\ stxr w15, w17, [x1]1263 \\ stxr w15, w17, [x1]
1284 \\ cbnz w15, 0b1264 \\ cbnz w15, 0b
1285 \\1:1265 \\1:
1286 : [ret] "={w0}" (-> u32),1266 \\ ret
1287 : [val] "{w0}" (val),1267 :
1288 [ptr] "{x1}" (ptr),1268 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1289 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1290 : "w15", "w16", "w17", "memory"1269 : "w15", "w16", "w17", "memory"
1291 );1270 );
1271 unreachable;
1292}1272}
1293fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {1273fn __aarch64_cas4_rel() align(16) callconv(.Naked) void {
1294 @setRuntimeSafety(false);1274 @setRuntimeSafety(false);
1295 return asm volatile (1275 asm volatile (
1296 \\ cbz w16, 8f1276 \\ cbz w16, 8f
1297 \\ .inst 0x08a07c41 + 0x80000000 + 0x0080001277 \\ .inst 0x08a07c41 + 0x80000000 + 0x008000
1298 \\1278 \\ ret
1299 \\ cbz wzr, 1f
1300 \\8:1279 \\8:
1301 \\ mov w16, w01280 \\ mov w16, w0
1302 \\0:1281 \\0:
...@@ -1306,20 +1285,19 @@ fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {...@@ -1306,20 +1285,19 @@ fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1306 \\ stlxr w17, w1, [x2]1285 \\ stlxr w17, w1, [x2]
1307 \\ cbnz w17, 0b1286 \\ cbnz w17, 0b
1308 \\1:1287 \\1:
1309 : [ret] "={w0}" (-> u32),1288 \\ ret
1310 : [expected] "{w0}" (expected),1289 :
1311 [desired] "{w1}" (desired),1290 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1312 [ptr] "{x2}" (ptr),
1313 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1314 : "w15", "w16", "w17", "memory"1291 : "w15", "w16", "w17", "memory"
1315 );1292 );
1293 unreachable;
1316}1294}
1317fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 {1295fn __aarch64_swp4_rel() align(16) callconv(.Naked) void {
1318 @setRuntimeSafety(false);1296 @setRuntimeSafety(false);
1319 return asm volatile (1297 asm volatile (
1320 \\ cbz w16, 8f1298 \\ cbz w16, 8f
1321 \\ .inst 0x38208020 + 0x80000000 + 0x4000001299 \\ .inst 0x38208020 + 0x80000000 + 0x400000
1322 \\ cbz wzr, 1f1300 \\ ret
1323 \\8:1301 \\8:
1324 \\ mov w16, w01302 \\ mov w16, w0
1325 \\0:1303 \\0:
...@@ -1327,19 +1305,19 @@ fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1327,19 +1305,19 @@ fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1327 \\ stlxr w17, w16, [x1]1305 \\ stlxr w17, w16, [x1]
1328 \\ cbnz w17, 0b1306 \\ cbnz w17, 0b
1329 \\1:1307 \\1:
1330 : [ret] "={w0}" (-> u32),1308 \\ ret
1331 : [val] "{w0}" (val),1309 :
1332 [ptr] "{x1}" (ptr),1310 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1333 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1334 : "w15", "w16", "w17", "memory"1311 : "w15", "w16", "w17", "memory"
1335 );1312 );
1313 unreachable;
1336}1314}
1337fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 {1315fn __aarch64_ldadd4_rel() align(16) callconv(.Naked) void {
1338 @setRuntimeSafety(false);1316 @setRuntimeSafety(false);
1339 return asm volatile (1317 asm volatile (
1340 \\ cbz w16, 8f1318 \\ cbz w16, 8f
1341 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x4000001319 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
1342 \\ cbz wzr, 1f1320 \\ ret
1343 \\8:1321 \\8:
1344 \\ mov w16, w01322 \\ mov w16, w0
1345 \\0:1323 \\0:
...@@ -1348,19 +1326,19 @@ fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1348,19 +1326,19 @@ fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1348 \\ stlxr w15, w17, [x1]1326 \\ stlxr w15, w17, [x1]
1349 \\ cbnz w15, 0b1327 \\ cbnz w15, 0b
1350 \\1:1328 \\1:
1351 : [ret] "={w0}" (-> u32),1329 \\ ret
1352 : [val] "{w0}" (val),1330 :
1353 [ptr] "{x1}" (ptr),1331 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1354 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1355 : "w15", "w16", "w17", "memory"1332 : "w15", "w16", "w17", "memory"
1356 );1333 );
1334 unreachable;
1357}1335}
1358fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 {1336fn __aarch64_ldclr4_rel() align(16) callconv(.Naked) void {
1359 @setRuntimeSafety(false);1337 @setRuntimeSafety(false);
1360 return asm volatile (1338 asm volatile (
1361 \\ cbz w16, 8f1339 \\ cbz w16, 8f
1362 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x4000001340 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
1363 \\ cbz wzr, 1f1341 \\ ret
1364 \\8:1342 \\8:
1365 \\ mov w16, w01343 \\ mov w16, w0
1366 \\0:1344 \\0:
...@@ -1369,19 +1347,19 @@ fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1369,19 +1347,19 @@ fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1369 \\ stlxr w15, w17, [x1]1347 \\ stlxr w15, w17, [x1]
1370 \\ cbnz w15, 0b1348 \\ cbnz w15, 0b
1371 \\1:1349 \\1:
1372 : [ret] "={w0}" (-> u32),1350 \\ ret
1373 : [val] "{w0}" (val),1351 :
1374 [ptr] "{x1}" (ptr),1352 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1375 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1376 : "w15", "w16", "w17", "memory"1353 : "w15", "w16", "w17", "memory"
1377 );1354 );
1355 unreachable;
1378}1356}
1379fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 {1357fn __aarch64_ldeor4_rel() align(16) callconv(.Naked) void {
1380 @setRuntimeSafety(false);1358 @setRuntimeSafety(false);
1381 return asm volatile (1359 asm volatile (
1382 \\ cbz w16, 8f1360 \\ cbz w16, 8f
1383 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x4000001361 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
1384 \\ cbz wzr, 1f1362 \\ ret
1385 \\8:1363 \\8:
1386 \\ mov w16, w01364 \\ mov w16, w0
1387 \\0:1365 \\0:
...@@ -1390,19 +1368,19 @@ fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1390,19 +1368,19 @@ fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1390 \\ stlxr w15, w17, [x1]1368 \\ stlxr w15, w17, [x1]
1391 \\ cbnz w15, 0b1369 \\ cbnz w15, 0b
1392 \\1:1370 \\1:
1393 : [ret] "={w0}" (-> u32),1371 \\ ret
1394 : [val] "{w0}" (val),1372 :
1395 [ptr] "{x1}" (ptr),1373 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1396 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1397 : "w15", "w16", "w17", "memory"1374 : "w15", "w16", "w17", "memory"
1398 );1375 );
1376 unreachable;
1399}1377}
1400fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 {1378fn __aarch64_ldset4_rel() align(16) callconv(.Naked) void {
1401 @setRuntimeSafety(false);1379 @setRuntimeSafety(false);
1402 return asm volatile (1380 asm volatile (
1403 \\ cbz w16, 8f1381 \\ cbz w16, 8f
1404 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x4000001382 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
1405 \\ cbz wzr, 1f1383 \\ ret
1406 \\8:1384 \\8:
1407 \\ mov w16, w01385 \\ mov w16, w0
1408 \\0:1386 \\0:
...@@ -1411,20 +1389,19 @@ fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1411,20 +1389,19 @@ fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1411 \\ stlxr w15, w17, [x1]1389 \\ stlxr w15, w17, [x1]
1412 \\ cbnz w15, 0b1390 \\ cbnz w15, 0b
1413 \\1:1391 \\1:
1414 : [ret] "={w0}" (-> u32),1392 \\ ret
1415 : [val] "{w0}" (val),1393 :
1416 [ptr] "{x1}" (ptr),1394 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1417 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1418 : "w15", "w16", "w17", "memory"1395 : "w15", "w16", "w17", "memory"
1419 );1396 );
1397 unreachable;
1420}1398}
1421fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {1399fn __aarch64_cas4_acq_rel() align(16) callconv(.Naked) void {
1422 @setRuntimeSafety(false);1400 @setRuntimeSafety(false);
1423 return asm volatile (1401 asm volatile (
1424 \\ cbz w16, 8f1402 \\ cbz w16, 8f
1425 \\ .inst 0x08a07c41 + 0x80000000 + 0x4080001403 \\ .inst 0x08a07c41 + 0x80000000 + 0x408000
1426 \\1404 \\ ret
1427 \\ cbz wzr, 1f
1428 \\8:1405 \\8:
1429 \\ mov w16, w01406 \\ mov w16, w0
1430 \\0:1407 \\0:
...@@ -1434,20 +1411,19 @@ fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u...@@ -1434,20 +1411,19 @@ fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u
1434 \\ stlxr w17, w1, [x2]1411 \\ stlxr w17, w1, [x2]
1435 \\ cbnz w17, 0b1412 \\ cbnz w17, 0b
1436 \\1:1413 \\1:
1437 : [ret] "={w0}" (-> u32),1414 \\ ret
1438 : [expected] "{w0}" (expected),1415 :
1439 [desired] "{w1}" (desired),1416 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1440 [ptr] "{x2}" (ptr),
1441 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1442 : "w15", "w16", "w17", "memory"1417 : "w15", "w16", "w17", "memory"
1443 );1418 );
1419 unreachable;
1444}1420}
1445fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {1421fn __aarch64_swp4_acq_rel() align(16) callconv(.Naked) void {
1446 @setRuntimeSafety(false);1422 @setRuntimeSafety(false);
1447 return asm volatile (1423 asm volatile (
1448 \\ cbz w16, 8f1424 \\ cbz w16, 8f
1449 \\ .inst 0x38208020 + 0x80000000 + 0xc000001425 \\ .inst 0x38208020 + 0x80000000 + 0xc00000
1450 \\ cbz wzr, 1f1426 \\ ret
1451 \\8:1427 \\8:
1452 \\ mov w16, w01428 \\ mov w16, w0
1453 \\0:1429 \\0:
...@@ -1455,19 +1431,19 @@ fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1455,19 +1431,19 @@ fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1455 \\ stlxr w17, w16, [x1]1431 \\ stlxr w17, w16, [x1]
1456 \\ cbnz w17, 0b1432 \\ cbnz w17, 0b
1457 \\1:1433 \\1:
1458 : [ret] "={w0}" (-> u32),1434 \\ ret
1459 : [val] "{w0}" (val),1435 :
1460 [ptr] "{x1}" (ptr),1436 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1461 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1462 : "w15", "w16", "w17", "memory"1437 : "w15", "w16", "w17", "memory"
1463 );1438 );
1439 unreachable;
1464}1440}
1465fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {1441fn __aarch64_ldadd4_acq_rel() align(16) callconv(.Naked) void {
1466 @setRuntimeSafety(false);1442 @setRuntimeSafety(false);
1467 return asm volatile (1443 asm volatile (
1468 \\ cbz w16, 8f1444 \\ cbz w16, 8f
1469 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc000001445 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
1470 \\ cbz wzr, 1f1446 \\ ret
1471 \\8:1447 \\8:
1472 \\ mov w16, w01448 \\ mov w16, w0
1473 \\0:1449 \\0:
...@@ -1476,19 +1452,19 @@ fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1476,19 +1452,19 @@ fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1476 \\ stlxr w15, w17, [x1]1452 \\ stlxr w15, w17, [x1]
1477 \\ cbnz w15, 0b1453 \\ cbnz w15, 0b
1478 \\1:1454 \\1:
1479 : [ret] "={w0}" (-> u32),1455 \\ ret
1480 : [val] "{w0}" (val),1456 :
1481 [ptr] "{x1}" (ptr),1457 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1482 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1483 : "w15", "w16", "w17", "memory"1458 : "w15", "w16", "w17", "memory"
1484 );1459 );
1460 unreachable;
1485}1461}
1486fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {1462fn __aarch64_ldclr4_acq_rel() align(16) callconv(.Naked) void {
1487 @setRuntimeSafety(false);1463 @setRuntimeSafety(false);
1488 return asm volatile (1464 asm volatile (
1489 \\ cbz w16, 8f1465 \\ cbz w16, 8f
1490 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc000001466 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
1491 \\ cbz wzr, 1f1467 \\ ret
1492 \\8:1468 \\8:
1493 \\ mov w16, w01469 \\ mov w16, w0
1494 \\0:1470 \\0:
...@@ -1497,19 +1473,19 @@ fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1497,19 +1473,19 @@ fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1497 \\ stlxr w15, w17, [x1]1473 \\ stlxr w15, w17, [x1]
1498 \\ cbnz w15, 0b1474 \\ cbnz w15, 0b
1499 \\1:1475 \\1:
1500 : [ret] "={w0}" (-> u32),1476 \\ ret
1501 : [val] "{w0}" (val),1477 :
1502 [ptr] "{x1}" (ptr),1478 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1503 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1504 : "w15", "w16", "w17", "memory"1479 : "w15", "w16", "w17", "memory"
1505 );1480 );
1481 unreachable;
1506}1482}
1507fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {1483fn __aarch64_ldeor4_acq_rel() align(16) callconv(.Naked) void {
1508 @setRuntimeSafety(false);1484 @setRuntimeSafety(false);
1509 return asm volatile (1485 asm volatile (
1510 \\ cbz w16, 8f1486 \\ cbz w16, 8f
1511 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc000001487 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
1512 \\ cbz wzr, 1f1488 \\ ret
1513 \\8:1489 \\8:
1514 \\ mov w16, w01490 \\ mov w16, w0
1515 \\0:1491 \\0:
...@@ -1518,19 +1494,19 @@ fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1518,19 +1494,19 @@ fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1518 \\ stlxr w15, w17, [x1]1494 \\ stlxr w15, w17, [x1]
1519 \\ cbnz w15, 0b1495 \\ cbnz w15, 0b
1520 \\1:1496 \\1:
1521 : [ret] "={w0}" (-> u32),1497 \\ ret
1522 : [val] "{w0}" (val),1498 :
1523 [ptr] "{x1}" (ptr),1499 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1524 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1525 : "w15", "w16", "w17", "memory"1500 : "w15", "w16", "w17", "memory"
1526 );1501 );
1502 unreachable;
1527}1503}
1528fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {1504fn __aarch64_ldset4_acq_rel() align(16) callconv(.Naked) void {
1529 @setRuntimeSafety(false);1505 @setRuntimeSafety(false);
1530 return asm volatile (1506 asm volatile (
1531 \\ cbz w16, 8f1507 \\ cbz w16, 8f
1532 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc000001508 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
1533 \\ cbz wzr, 1f1509 \\ ret
1534 \\8:1510 \\8:
1535 \\ mov w16, w01511 \\ mov w16, w0
1536 \\0:1512 \\0:
...@@ -1539,20 +1515,19 @@ fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {...@@ -1539,20 +1515,19 @@ fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1539 \\ stlxr w15, w17, [x1]1515 \\ stlxr w15, w17, [x1]
1540 \\ cbnz w15, 0b1516 \\ cbnz w15, 0b
1541 \\1:1517 \\1:
1542 : [ret] "={w0}" (-> u32),1518 \\ ret
1543 : [val] "{w0}" (val),1519 :
1544 [ptr] "{x1}" (ptr),1520 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1545 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1546 : "w15", "w16", "w17", "memory"1521 : "w15", "w16", "w17", "memory"
1547 );1522 );
1523 unreachable;
1548}1524}
1549fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {1525fn __aarch64_cas8_relax() align(16) callconv(.Naked) void {
1550 @setRuntimeSafety(false);1526 @setRuntimeSafety(false);
1551 return asm volatile (1527 asm volatile (
1552 \\ cbz w16, 8f1528 \\ cbz w16, 8f
1553 \\ .inst 0x08a07c41 + 0xc0000000 + 0x0000001529 \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
1554 \\1530 \\ ret
1555 \\ cbz wzr, 1f
1556 \\8:1531 \\8:
1557 \\ mov x16, x01532 \\ mov x16, x0
1558 \\0:1533 \\0:
...@@ -1562,20 +1537,19 @@ fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64...@@ -1562,20 +1537,19 @@ fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64
1562 \\ stxr w17, x1, [x2]1537 \\ stxr w17, x1, [x2]
1563 \\ cbnz w17, 0b1538 \\ cbnz w17, 0b
1564 \\1:1539 \\1:
1565 : [ret] "={x0}" (-> u64),1540 \\ ret
1566 : [expected] "{x0}" (expected),1541 :
1567 [desired] "{x1}" (desired),1542 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1568 [ptr] "{x2}" (ptr),
1569 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1570 : "w15", "w16", "w17", "memory"1543 : "w15", "w16", "w17", "memory"
1571 );1544 );
1545 unreachable;
1572}1546}
1573fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 {1547fn __aarch64_swp8_relax() align(16) callconv(.Naked) void {
1574 @setRuntimeSafety(false);1548 @setRuntimeSafety(false);
1575 return asm volatile (1549 asm volatile (
1576 \\ cbz w16, 8f1550 \\ cbz w16, 8f
1577 \\ .inst 0x38208020 + 0xc0000000 + 0x0000001551 \\ .inst 0x38208020 + 0xc0000000 + 0x000000
1578 \\ cbz wzr, 1f1552 \\ ret
1579 \\8:1553 \\8:
1580 \\ mov x16, x01554 \\ mov x16, x0
1581 \\0:1555 \\0:
...@@ -1583,19 +1557,19 @@ fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1583,19 +1557,19 @@ fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1583 \\ stxr w17, x16, [x1]1557 \\ stxr w17, x16, [x1]
1584 \\ cbnz w17, 0b1558 \\ cbnz w17, 0b
1585 \\1:1559 \\1:
1586 : [ret] "={x0}" (-> u64),1560 \\ ret
1587 : [val] "{x0}" (val),1561 :
1588 [ptr] "{x1}" (ptr),1562 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1589 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1590 : "w15", "w16", "w17", "memory"1563 : "w15", "w16", "w17", "memory"
1591 );1564 );
1565 unreachable;
1592}1566}
1593fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 {1567fn __aarch64_ldadd8_relax() align(16) callconv(.Naked) void {
1594 @setRuntimeSafety(false);1568 @setRuntimeSafety(false);
1595 return asm volatile (1569 asm volatile (
1596 \\ cbz w16, 8f1570 \\ cbz w16, 8f
1597 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x0000001571 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
1598 \\ cbz wzr, 1f1572 \\ ret
1599 \\8:1573 \\8:
1600 \\ mov x16, x01574 \\ mov x16, x0
1601 \\0:1575 \\0:
...@@ -1604,19 +1578,19 @@ fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1604,19 +1578,19 @@ fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1604 \\ stxr w15, x17, [x1]1578 \\ stxr w15, x17, [x1]
1605 \\ cbnz w15, 0b1579 \\ cbnz w15, 0b
1606 \\1:1580 \\1:
1607 : [ret] "={x0}" (-> u64),1581 \\ ret
1608 : [val] "{x0}" (val),1582 :
1609 [ptr] "{x1}" (ptr),1583 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1610 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1611 : "w15", "w16", "w17", "memory"1584 : "w15", "w16", "w17", "memory"
1612 );1585 );
1586 unreachable;
1613}1587}
1614fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 {1588fn __aarch64_ldclr8_relax() align(16) callconv(.Naked) void {
1615 @setRuntimeSafety(false);1589 @setRuntimeSafety(false);
1616 return asm volatile (1590 asm volatile (
1617 \\ cbz w16, 8f1591 \\ cbz w16, 8f
1618 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x0000001592 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
1619 \\ cbz wzr, 1f1593 \\ ret
1620 \\8:1594 \\8:
1621 \\ mov x16, x01595 \\ mov x16, x0
1622 \\0:1596 \\0:
...@@ -1625,19 +1599,19 @@ fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1625,19 +1599,19 @@ fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1625 \\ stxr w15, x17, [x1]1599 \\ stxr w15, x17, [x1]
1626 \\ cbnz w15, 0b1600 \\ cbnz w15, 0b
1627 \\1:1601 \\1:
1628 : [ret] "={x0}" (-> u64),1602 \\ ret
1629 : [val] "{x0}" (val),1603 :
1630 [ptr] "{x1}" (ptr),1604 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1631 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1632 : "w15", "w16", "w17", "memory"1605 : "w15", "w16", "w17", "memory"
1633 );1606 );
1607 unreachable;
1634}1608}
1635fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 {1609fn __aarch64_ldeor8_relax() align(16) callconv(.Naked) void {
1636 @setRuntimeSafety(false);1610 @setRuntimeSafety(false);
1637 return asm volatile (1611 asm volatile (
1638 \\ cbz w16, 8f1612 \\ cbz w16, 8f
1639 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x0000001613 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
1640 \\ cbz wzr, 1f1614 \\ ret
1641 \\8:1615 \\8:
1642 \\ mov x16, x01616 \\ mov x16, x0
1643 \\0:1617 \\0:
...@@ -1646,19 +1620,19 @@ fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1646,19 +1620,19 @@ fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1646 \\ stxr w15, x17, [x1]1620 \\ stxr w15, x17, [x1]
1647 \\ cbnz w15, 0b1621 \\ cbnz w15, 0b
1648 \\1:1622 \\1:
1649 : [ret] "={x0}" (-> u64),1623 \\ ret
1650 : [val] "{x0}" (val),1624 :
1651 [ptr] "{x1}" (ptr),1625 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1652 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1653 : "w15", "w16", "w17", "memory"1626 : "w15", "w16", "w17", "memory"
1654 );1627 );
1628 unreachable;
1655}1629}
1656fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 {1630fn __aarch64_ldset8_relax() align(16) callconv(.Naked) void {
1657 @setRuntimeSafety(false);1631 @setRuntimeSafety(false);
1658 return asm volatile (1632 asm volatile (
1659 \\ cbz w16, 8f1633 \\ cbz w16, 8f
1660 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x0000001634 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
1661 \\ cbz wzr, 1f1635 \\ ret
1662 \\8:1636 \\8:
1663 \\ mov x16, x01637 \\ mov x16, x0
1664 \\0:1638 \\0:
...@@ -1667,20 +1641,19 @@ fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1667,20 +1641,19 @@ fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1667 \\ stxr w15, x17, [x1]1641 \\ stxr w15, x17, [x1]
1668 \\ cbnz w15, 0b1642 \\ cbnz w15, 0b
1669 \\1:1643 \\1:
1670 : [ret] "={x0}" (-> u64),1644 \\ ret
1671 : [val] "{x0}" (val),1645 :
1672 [ptr] "{x1}" (ptr),1646 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1673 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1674 : "w15", "w16", "w17", "memory"1647 : "w15", "w16", "w17", "memory"
1675 );1648 );
1649 unreachable;
1676}1650}
1677fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {1651fn __aarch64_cas8_acq() align(16) callconv(.Naked) void {
1678 @setRuntimeSafety(false);1652 @setRuntimeSafety(false);
1679 return asm volatile (1653 asm volatile (
1680 \\ cbz w16, 8f1654 \\ cbz w16, 8f
1681 \\ .inst 0x08a07c41 + 0xc0000000 + 0x4000001655 \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
1682 \\1656 \\ ret
1683 \\ cbz wzr, 1f
1684 \\8:1657 \\8:
1685 \\ mov x16, x01658 \\ mov x16, x0
1686 \\0:1659 \\0:
...@@ -1690,20 +1663,19 @@ fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {...@@ -1690,20 +1663,19 @@ fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
1690 \\ stxr w17, x1, [x2]1663 \\ stxr w17, x1, [x2]
1691 \\ cbnz w17, 0b1664 \\ cbnz w17, 0b
1692 \\1:1665 \\1:
1693 : [ret] "={x0}" (-> u64),1666 \\ ret
1694 : [expected] "{x0}" (expected),1667 :
1695 [desired] "{x1}" (desired),1668 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1696 [ptr] "{x2}" (ptr),
1697 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1698 : "w15", "w16", "w17", "memory"1669 : "w15", "w16", "w17", "memory"
1699 );1670 );
1671 unreachable;
1700}1672}
1701fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 {1673fn __aarch64_swp8_acq() align(16) callconv(.Naked) void {
1702 @setRuntimeSafety(false);1674 @setRuntimeSafety(false);
1703 return asm volatile (1675 asm volatile (
1704 \\ cbz w16, 8f1676 \\ cbz w16, 8f
1705 \\ .inst 0x38208020 + 0xc0000000 + 0x8000001677 \\ .inst 0x38208020 + 0xc0000000 + 0x800000
1706 \\ cbz wzr, 1f1678 \\ ret
1707 \\8:1679 \\8:
1708 \\ mov x16, x01680 \\ mov x16, x0
1709 \\0:1681 \\0:
...@@ -1711,19 +1683,19 @@ fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1711,19 +1683,19 @@ fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1711 \\ stxr w17, x16, [x1]1683 \\ stxr w17, x16, [x1]
1712 \\ cbnz w17, 0b1684 \\ cbnz w17, 0b
1713 \\1:1685 \\1:
1714 : [ret] "={x0}" (-> u64),1686 \\ ret
1715 : [val] "{x0}" (val),1687 :
1716 [ptr] "{x1}" (ptr),1688 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1717 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1718 : "w15", "w16", "w17", "memory"1689 : "w15", "w16", "w17", "memory"
1719 );1690 );
1691 unreachable;
1720}1692}
1721fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 {1693fn __aarch64_ldadd8_acq() align(16) callconv(.Naked) void {
1722 @setRuntimeSafety(false);1694 @setRuntimeSafety(false);
1723 return asm volatile (1695 asm volatile (
1724 \\ cbz w16, 8f1696 \\ cbz w16, 8f
1725 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x8000001697 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
1726 \\ cbz wzr, 1f1698 \\ ret
1727 \\8:1699 \\8:
1728 \\ mov x16, x01700 \\ mov x16, x0
1729 \\0:1701 \\0:
...@@ -1732,19 +1704,19 @@ fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1732,19 +1704,19 @@ fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1732 \\ stxr w15, x17, [x1]1704 \\ stxr w15, x17, [x1]
1733 \\ cbnz w15, 0b1705 \\ cbnz w15, 0b
1734 \\1:1706 \\1:
1735 : [ret] "={x0}" (-> u64),1707 \\ ret
1736 : [val] "{x0}" (val),1708 :
1737 [ptr] "{x1}" (ptr),1709 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1738 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1739 : "w15", "w16", "w17", "memory"1710 : "w15", "w16", "w17", "memory"
1740 );1711 );
1712 unreachable;
1741}1713}
1742fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 {1714fn __aarch64_ldclr8_acq() align(16) callconv(.Naked) void {
1743 @setRuntimeSafety(false);1715 @setRuntimeSafety(false);
1744 return asm volatile (1716 asm volatile (
1745 \\ cbz w16, 8f1717 \\ cbz w16, 8f
1746 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x8000001718 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
1747 \\ cbz wzr, 1f1719 \\ ret
1748 \\8:1720 \\8:
1749 \\ mov x16, x01721 \\ mov x16, x0
1750 \\0:1722 \\0:
...@@ -1753,19 +1725,19 @@ fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1753,19 +1725,19 @@ fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1753 \\ stxr w15, x17, [x1]1725 \\ stxr w15, x17, [x1]
1754 \\ cbnz w15, 0b1726 \\ cbnz w15, 0b
1755 \\1:1727 \\1:
1756 : [ret] "={x0}" (-> u64),1728 \\ ret
1757 : [val] "{x0}" (val),1729 :
1758 [ptr] "{x1}" (ptr),1730 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1759 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1760 : "w15", "w16", "w17", "memory"1731 : "w15", "w16", "w17", "memory"
1761 );1732 );
1733 unreachable;
1762}1734}
1763fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 {1735fn __aarch64_ldeor8_acq() align(16) callconv(.Naked) void {
1764 @setRuntimeSafety(false);1736 @setRuntimeSafety(false);
1765 return asm volatile (1737 asm volatile (
1766 \\ cbz w16, 8f1738 \\ cbz w16, 8f
1767 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x8000001739 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
1768 \\ cbz wzr, 1f1740 \\ ret
1769 \\8:1741 \\8:
1770 \\ mov x16, x01742 \\ mov x16, x0
1771 \\0:1743 \\0:
...@@ -1774,19 +1746,19 @@ fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1774,19 +1746,19 @@ fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1774 \\ stxr w15, x17, [x1]1746 \\ stxr w15, x17, [x1]
1775 \\ cbnz w15, 0b1747 \\ cbnz w15, 0b
1776 \\1:1748 \\1:
1777 : [ret] "={x0}" (-> u64),1749 \\ ret
1778 : [val] "{x0}" (val),1750 :
1779 [ptr] "{x1}" (ptr),1751 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1780 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1781 : "w15", "w16", "w17", "memory"1752 : "w15", "w16", "w17", "memory"
1782 );1753 );
1754 unreachable;
1783}1755}
1784fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 {1756fn __aarch64_ldset8_acq() align(16) callconv(.Naked) void {
1785 @setRuntimeSafety(false);1757 @setRuntimeSafety(false);
1786 return asm volatile (1758 asm volatile (
1787 \\ cbz w16, 8f1759 \\ cbz w16, 8f
1788 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x8000001760 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
1789 \\ cbz wzr, 1f1761 \\ ret
1790 \\8:1762 \\8:
1791 \\ mov x16, x01763 \\ mov x16, x0
1792 \\0:1764 \\0:
...@@ -1795,20 +1767,19 @@ fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1795,20 +1767,19 @@ fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1795 \\ stxr w15, x17, [x1]1767 \\ stxr w15, x17, [x1]
1796 \\ cbnz w15, 0b1768 \\ cbnz w15, 0b
1797 \\1:1769 \\1:
1798 : [ret] "={x0}" (-> u64),1770 \\ ret
1799 : [val] "{x0}" (val),1771 :
1800 [ptr] "{x1}" (ptr),1772 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1801 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1802 : "w15", "w16", "w17", "memory"1773 : "w15", "w16", "w17", "memory"
1803 );1774 );
1775 unreachable;
1804}1776}
1805fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {1777fn __aarch64_cas8_rel() align(16) callconv(.Naked) void {
1806 @setRuntimeSafety(false);1778 @setRuntimeSafety(false);
1807 return asm volatile (1779 asm volatile (
1808 \\ cbz w16, 8f1780 \\ cbz w16, 8f
1809 \\ .inst 0x08a07c41 + 0xc0000000 + 0x0080001781 \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
1810 \\1782 \\ ret
1811 \\ cbz wzr, 1f
1812 \\8:1783 \\8:
1813 \\ mov x16, x01784 \\ mov x16, x0
1814 \\0:1785 \\0:
...@@ -1818,20 +1789,19 @@ fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {...@@ -1818,20 +1789,19 @@ fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
1818 \\ stlxr w17, x1, [x2]1789 \\ stlxr w17, x1, [x2]
1819 \\ cbnz w17, 0b1790 \\ cbnz w17, 0b
1820 \\1:1791 \\1:
1821 : [ret] "={x0}" (-> u64),1792 \\ ret
1822 : [expected] "{x0}" (expected),1793 :
1823 [desired] "{x1}" (desired),1794 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1824 [ptr] "{x2}" (ptr),
1825 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1826 : "w15", "w16", "w17", "memory"1795 : "w15", "w16", "w17", "memory"
1827 );1796 );
1797 unreachable;
1828}1798}
1829fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 {1799fn __aarch64_swp8_rel() align(16) callconv(.Naked) void {
1830 @setRuntimeSafety(false);1800 @setRuntimeSafety(false);
1831 return asm volatile (1801 asm volatile (
1832 \\ cbz w16, 8f1802 \\ cbz w16, 8f
1833 \\ .inst 0x38208020 + 0xc0000000 + 0x4000001803 \\ .inst 0x38208020 + 0xc0000000 + 0x400000
1834 \\ cbz wzr, 1f1804 \\ ret
1835 \\8:1805 \\8:
1836 \\ mov x16, x01806 \\ mov x16, x0
1837 \\0:1807 \\0:
...@@ -1839,19 +1809,19 @@ fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1839,19 +1809,19 @@ fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1839 \\ stlxr w17, x16, [x1]1809 \\ stlxr w17, x16, [x1]
1840 \\ cbnz w17, 0b1810 \\ cbnz w17, 0b
1841 \\1:1811 \\1:
1842 : [ret] "={x0}" (-> u64),1812 \\ ret
1843 : [val] "{x0}" (val),1813 :
1844 [ptr] "{x1}" (ptr),1814 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1845 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1846 : "w15", "w16", "w17", "memory"1815 : "w15", "w16", "w17", "memory"
1847 );1816 );
1817 unreachable;
1848}1818}
1849fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 {1819fn __aarch64_ldadd8_rel() align(16) callconv(.Naked) void {
1850 @setRuntimeSafety(false);1820 @setRuntimeSafety(false);
1851 return asm volatile (1821 asm volatile (
1852 \\ cbz w16, 8f1822 \\ cbz w16, 8f
1853 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x4000001823 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
1854 \\ cbz wzr, 1f1824 \\ ret
1855 \\8:1825 \\8:
1856 \\ mov x16, x01826 \\ mov x16, x0
1857 \\0:1827 \\0:
...@@ -1860,19 +1830,19 @@ fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1860,19 +1830,19 @@ fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1860 \\ stlxr w15, x17, [x1]1830 \\ stlxr w15, x17, [x1]
1861 \\ cbnz w15, 0b1831 \\ cbnz w15, 0b
1862 \\1:1832 \\1:
1863 : [ret] "={x0}" (-> u64),1833 \\ ret
1864 : [val] "{x0}" (val),1834 :
1865 [ptr] "{x1}" (ptr),1835 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1866 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1867 : "w15", "w16", "w17", "memory"1836 : "w15", "w16", "w17", "memory"
1868 );1837 );
1838 unreachable;
1869}1839}
1870fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 {1840fn __aarch64_ldclr8_rel() align(16) callconv(.Naked) void {
1871 @setRuntimeSafety(false);1841 @setRuntimeSafety(false);
1872 return asm volatile (1842 asm volatile (
1873 \\ cbz w16, 8f1843 \\ cbz w16, 8f
1874 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x4000001844 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
1875 \\ cbz wzr, 1f1845 \\ ret
1876 \\8:1846 \\8:
1877 \\ mov x16, x01847 \\ mov x16, x0
1878 \\0:1848 \\0:
...@@ -1881,19 +1851,19 @@ fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1881,19 +1851,19 @@ fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1881 \\ stlxr w15, x17, [x1]1851 \\ stlxr w15, x17, [x1]
1882 \\ cbnz w15, 0b1852 \\ cbnz w15, 0b
1883 \\1:1853 \\1:
1884 : [ret] "={x0}" (-> u64),1854 \\ ret
1885 : [val] "{x0}" (val),1855 :
1886 [ptr] "{x1}" (ptr),1856 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1887 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1888 : "w15", "w16", "w17", "memory"1857 : "w15", "w16", "w17", "memory"
1889 );1858 );
1859 unreachable;
1890}1860}
1891fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 {1861fn __aarch64_ldeor8_rel() align(16) callconv(.Naked) void {
1892 @setRuntimeSafety(false);1862 @setRuntimeSafety(false);
1893 return asm volatile (1863 asm volatile (
1894 \\ cbz w16, 8f1864 \\ cbz w16, 8f
1895 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x4000001865 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
1896 \\ cbz wzr, 1f1866 \\ ret
1897 \\8:1867 \\8:
1898 \\ mov x16, x01868 \\ mov x16, x0
1899 \\0:1869 \\0:
...@@ -1902,19 +1872,19 @@ fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1902,19 +1872,19 @@ fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1902 \\ stlxr w15, x17, [x1]1872 \\ stlxr w15, x17, [x1]
1903 \\ cbnz w15, 0b1873 \\ cbnz w15, 0b
1904 \\1:1874 \\1:
1905 : [ret] "={x0}" (-> u64),1875 \\ ret
1906 : [val] "{x0}" (val),1876 :
1907 [ptr] "{x1}" (ptr),1877 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1908 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1909 : "w15", "w16", "w17", "memory"1878 : "w15", "w16", "w17", "memory"
1910 );1879 );
1880 unreachable;
1911}1881}
1912fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 {1882fn __aarch64_ldset8_rel() align(16) callconv(.Naked) void {
1913 @setRuntimeSafety(false);1883 @setRuntimeSafety(false);
1914 return asm volatile (1884 asm volatile (
1915 \\ cbz w16, 8f1885 \\ cbz w16, 8f
1916 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x4000001886 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
1917 \\ cbz wzr, 1f1887 \\ ret
1918 \\8:1888 \\8:
1919 \\ mov x16, x01889 \\ mov x16, x0
1920 \\0:1890 \\0:
...@@ -1923,20 +1893,19 @@ fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1923,20 +1893,19 @@ fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1923 \\ stlxr w15, x17, [x1]1893 \\ stlxr w15, x17, [x1]
1924 \\ cbnz w15, 0b1894 \\ cbnz w15, 0b
1925 \\1:1895 \\1:
1926 : [ret] "={x0}" (-> u64),1896 \\ ret
1927 : [val] "{x0}" (val),1897 :
1928 [ptr] "{x1}" (ptr),1898 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1929 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1930 : "w15", "w16", "w17", "memory"1899 : "w15", "w16", "w17", "memory"
1931 );1900 );
1901 unreachable;
1932}1902}
1933fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {1903fn __aarch64_cas8_acq_rel() align(16) callconv(.Naked) void {
1934 @setRuntimeSafety(false);1904 @setRuntimeSafety(false);
1935 return asm volatile (1905 asm volatile (
1936 \\ cbz w16, 8f1906 \\ cbz w16, 8f
1937 \\ .inst 0x08a07c41 + 0xc0000000 + 0x4080001907 \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
1938 \\1908 \\ ret
1939 \\ cbz wzr, 1f
1940 \\8:1909 \\8:
1941 \\ mov x16, x01910 \\ mov x16, x0
1942 \\0:1911 \\0:
...@@ -1946,20 +1915,19 @@ fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u...@@ -1946,20 +1915,19 @@ fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u
1946 \\ stlxr w17, x1, [x2]1915 \\ stlxr w17, x1, [x2]
1947 \\ cbnz w17, 0b1916 \\ cbnz w17, 0b
1948 \\1:1917 \\1:
1949 : [ret] "={x0}" (-> u64),1918 \\ ret
1950 : [expected] "{x0}" (expected),1919 :
1951 [desired] "{x1}" (desired),1920 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1952 [ptr] "{x2}" (ptr),
1953 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1954 : "w15", "w16", "w17", "memory"1921 : "w15", "w16", "w17", "memory"
1955 );1922 );
1923 unreachable;
1956}1924}
1957fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {1925fn __aarch64_swp8_acq_rel() align(16) callconv(.Naked) void {
1958 @setRuntimeSafety(false);1926 @setRuntimeSafety(false);
1959 return asm volatile (1927 asm volatile (
1960 \\ cbz w16, 8f1928 \\ cbz w16, 8f
1961 \\ .inst 0x38208020 + 0xc0000000 + 0xc000001929 \\ .inst 0x38208020 + 0xc0000000 + 0xc00000
1962 \\ cbz wzr, 1f1930 \\ ret
1963 \\8:1931 \\8:
1964 \\ mov x16, x01932 \\ mov x16, x0
1965 \\0:1933 \\0:
...@@ -1967,19 +1935,19 @@ fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1967,19 +1935,19 @@ fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1967 \\ stlxr w17, x16, [x1]1935 \\ stlxr w17, x16, [x1]
1968 \\ cbnz w17, 0b1936 \\ cbnz w17, 0b
1969 \\1:1937 \\1:
1970 : [ret] "={x0}" (-> u64),1938 \\ ret
1971 : [val] "{x0}" (val),1939 :
1972 [ptr] "{x1}" (ptr),1940 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1973 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1974 : "w15", "w16", "w17", "memory"1941 : "w15", "w16", "w17", "memory"
1975 );1942 );
1943 unreachable;
1976}1944}
1977fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {1945fn __aarch64_ldadd8_acq_rel() align(16) callconv(.Naked) void {
1978 @setRuntimeSafety(false);1946 @setRuntimeSafety(false);
1979 return asm volatile (1947 asm volatile (
1980 \\ cbz w16, 8f1948 \\ cbz w16, 8f
1981 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc000001949 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
1982 \\ cbz wzr, 1f1950 \\ ret
1983 \\8:1951 \\8:
1984 \\ mov x16, x01952 \\ mov x16, x0
1985 \\0:1953 \\0:
...@@ -1988,19 +1956,19 @@ fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -1988,19 +1956,19 @@ fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1988 \\ stlxr w15, x17, [x1]1956 \\ stlxr w15, x17, [x1]
1989 \\ cbnz w15, 0b1957 \\ cbnz w15, 0b
1990 \\1:1958 \\1:
1991 : [ret] "={x0}" (-> u64),1959 \\ ret
1992 : [val] "{x0}" (val),1960 :
1993 [ptr] "{x1}" (ptr),1961 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1994 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1995 : "w15", "w16", "w17", "memory"1962 : "w15", "w16", "w17", "memory"
1996 );1963 );
1964 unreachable;
1997}1965}
1998fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {1966fn __aarch64_ldclr8_acq_rel() align(16) callconv(.Naked) void {
1999 @setRuntimeSafety(false);1967 @setRuntimeSafety(false);
2000 return asm volatile (1968 asm volatile (
2001 \\ cbz w16, 8f1969 \\ cbz w16, 8f
2002 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc000001970 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
2003 \\ cbz wzr, 1f1971 \\ ret
2004 \\8:1972 \\8:
2005 \\ mov x16, x01973 \\ mov x16, x0
2006 \\0:1974 \\0:
...@@ -2009,19 +1977,19 @@ fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -2009,19 +1977,19 @@ fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
2009 \\ stlxr w15, x17, [x1]1977 \\ stlxr w15, x17, [x1]
2010 \\ cbnz w15, 0b1978 \\ cbnz w15, 0b
2011 \\1:1979 \\1:
2012 : [ret] "={x0}" (-> u64),1980 \\ ret
2013 : [val] "{x0}" (val),1981 :
2014 [ptr] "{x1}" (ptr),1982 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2015 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2016 : "w15", "w16", "w17", "memory"1983 : "w15", "w16", "w17", "memory"
2017 );1984 );
1985 unreachable;
2018}1986}
2019fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {1987fn __aarch64_ldeor8_acq_rel() align(16) callconv(.Naked) void {
2020 @setRuntimeSafety(false);1988 @setRuntimeSafety(false);
2021 return asm volatile (1989 asm volatile (
2022 \\ cbz w16, 8f1990 \\ cbz w16, 8f
2023 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc000001991 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
2024 \\ cbz wzr, 1f1992 \\ ret
2025 \\8:1993 \\8:
2026 \\ mov x16, x01994 \\ mov x16, x0
2027 \\0:1995 \\0:
...@@ -2030,19 +1998,19 @@ fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -2030,19 +1998,19 @@ fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
2030 \\ stlxr w15, x17, [x1]1998 \\ stlxr w15, x17, [x1]
2031 \\ cbnz w15, 0b1999 \\ cbnz w15, 0b
2032 \\1:2000 \\1:
2033 : [ret] "={x0}" (-> u64),2001 \\ ret
2034 : [val] "{x0}" (val),2002 :
2035 [ptr] "{x1}" (ptr),2003 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2036 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2037 : "w15", "w16", "w17", "memory"2004 : "w15", "w16", "w17", "memory"
2038 );2005 );
2006 unreachable;
2039}2007}
2040fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {2008fn __aarch64_ldset8_acq_rel() align(16) callconv(.Naked) void {
2041 @setRuntimeSafety(false);2009 @setRuntimeSafety(false);
2042 return asm volatile (2010 asm volatile (
2043 \\ cbz w16, 8f2011 \\ cbz w16, 8f
2044 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc000002012 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
2045 \\ cbz wzr, 1f2013 \\ ret
2046 \\8:2014 \\8:
2047 \\ mov x16, x02015 \\ mov x16, x0
2048 \\0:2016 \\0:
...@@ -2051,20 +2019,19 @@ fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {...@@ -2051,20 +2019,19 @@ fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
2051 \\ stlxr w15, x17, [x1]2019 \\ stlxr w15, x17, [x1]
2052 \\ cbnz w15, 0b2020 \\ cbnz w15, 0b
2053 \\1:2021 \\1:
2054 : [ret] "={x0}" (-> u64),2022 \\ ret
2055 : [val] "{x0}" (val),2023 :
2056 [ptr] "{x1}" (ptr),2024 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2057 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2058 : "w15", "w16", "w17", "memory"2025 : "w15", "w16", "w17", "memory"
2059 );2026 );
2027 unreachable;
2060}2028}
2061fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {2029fn __aarch64_cas16_relax() align(16) callconv(.Naked) void {
2062 @setRuntimeSafety(false);2030 @setRuntimeSafety(false);
2063 return asm volatile (2031 asm volatile (
2064 \\ cbz w16, 8f2032 \\ cbz w16, 8f
2065 \\ .inst 0x48207c82 + 0x0000002033 \\ .inst 0x48207c82 + 0x000000
2066 \\2034 \\ ret
2067 \\ cbz wzr, 1f
2068 \\8:2035 \\8:
2069 \\ mov x16, x02036 \\ mov x16, x0
2070 \\ mov x17, x12037 \\ mov x17, x1
...@@ -2076,21 +2043,19 @@ fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C)...@@ -2076,21 +2043,19 @@ fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C)
2076 \\ stxp w15, x2, x3, [x4]2043 \\ stxp w15, x2, x3, [x4]
2077 \\ cbnz w15, 0b2044 \\ cbnz w15, 0b
2078 \\1:2045 \\1:
2079 : [ret] "={x0}" (-> u128),2046 \\ ret
2080 : [expected] "{x0}" (expected),2047 :
2081 [desired] "{x1}" (desired),2048 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2082 [ptr] "{x2}" (ptr),
2083 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2084 : "w15", "w16", "w17", "memory"2049 : "w15", "w16", "w17", "memory"
2085 );2050 );
2051 unreachable;
2086}2052}
2087fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {2053fn __aarch64_cas16_acq() align(16) callconv(.Naked) void {
2088 @setRuntimeSafety(false);2054 @setRuntimeSafety(false);
2089 return asm volatile (2055 asm volatile (
2090 \\ cbz w16, 8f2056 \\ cbz w16, 8f
2091 \\ .inst 0x48207c82 + 0x4000002057 \\ .inst 0x48207c82 + 0x400000
2092 \\2058 \\ ret
2093 \\ cbz wzr, 1f
2094 \\8:2059 \\8:
2095 \\ mov x16, x02060 \\ mov x16, x0
2096 \\ mov x17, x12061 \\ mov x17, x1
...@@ -2102,21 +2067,19 @@ fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u...@@ -2102,21 +2067,19 @@ fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u
2102 \\ stxp w15, x2, x3, [x4]2067 \\ stxp w15, x2, x3, [x4]
2103 \\ cbnz w15, 0b2068 \\ cbnz w15, 0b
2104 \\1:2069 \\1:
2105 : [ret] "={x0}" (-> u128),2070 \\ ret
2106 : [expected] "{x0}" (expected),2071 :
2107 [desired] "{x1}" (desired),2072 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2108 [ptr] "{x2}" (ptr),
2109 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2110 : "w15", "w16", "w17", "memory"2073 : "w15", "w16", "w17", "memory"
2111 );2074 );
2075 unreachable;
2112}2076}
2113fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {2077fn __aarch64_cas16_rel() align(16) callconv(.Naked) void {
2114 @setRuntimeSafety(false);2078 @setRuntimeSafety(false);
2115 return asm volatile (2079 asm volatile (
2116 \\ cbz w16, 8f2080 \\ cbz w16, 8f
2117 \\ .inst 0x48207c82 + 0x0080002081 \\ .inst 0x48207c82 + 0x008000
2118 \\2082 \\ ret
2119 \\ cbz wzr, 1f
2120 \\8:2083 \\8:
2121 \\ mov x16, x02084 \\ mov x16, x0
2122 \\ mov x17, x12085 \\ mov x17, x1
...@@ -2128,21 +2091,19 @@ fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u...@@ -2128,21 +2091,19 @@ fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u
2128 \\ stlxp w15, x2, x3, [x4]2091 \\ stlxp w15, x2, x3, [x4]
2129 \\ cbnz w15, 0b2092 \\ cbnz w15, 0b
2130 \\1:2093 \\1:
2131 : [ret] "={x0}" (-> u128),2094 \\ ret
2132 : [expected] "{x0}" (expected),2095 :
2133 [desired] "{x1}" (desired),2096 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2134 [ptr] "{x2}" (ptr),
2135 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2136 : "w15", "w16", "w17", "memory"2097 : "w15", "w16", "w17", "memory"
2137 );2098 );
2099 unreachable;
2138}2100}
2139fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {2101fn __aarch64_cas16_acq_rel() align(16) callconv(.Naked) void {
2140 @setRuntimeSafety(false);2102 @setRuntimeSafety(false);
2141 return asm volatile (2103 asm volatile (
2142 \\ cbz w16, 8f2104 \\ cbz w16, 8f
2143 \\ .inst 0x48207c82 + 0x4080002105 \\ .inst 0x48207c82 + 0x408000
2144 \\2106 \\ ret
2145 \\ cbz wzr, 1f
2146 \\8:2107 \\8:
2147 \\ mov x16, x02108 \\ mov x16, x0
2148 \\ mov x17, x12109 \\ mov x17, x1
...@@ -2154,13 +2115,12 @@ fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(....@@ -2154,13 +2115,12 @@ fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(.
2154 \\ stlxp w15, x2, x3, [x4]2115 \\ stlxp w15, x2, x3, [x4]
2155 \\ cbnz w15, 0b2116 \\ cbnz w15, 0b
2156 \\1:2117 \\1:
2157 : [ret] "={x0}" (-> u128),2118 \\ ret
2158 : [expected] "{x0}" (expected),2119 :
2159 [desired] "{x1}" (desired),2120 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2160 [ptr] "{x2}" (ptr),
2161 [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2162 : "w15", "w16", "w17", "memory"2121 : "w15", "w16", "w17", "memory"
2163 );2122 );
2123 unreachable;
2164}2124}
21652125
2166comptime {2126comptime {
tools/gen_outline_atomics.zig+26-53
...@@ -76,21 +76,16 @@ fn writeFunction(...@@ -76,21 +76,16 @@ fn writeFunction(
76 .ldeor => try generateLd(arena, n, order, .ldeor),76 .ldeor => try generateLd(arena, n, order, .ldeor),
77 .ldset => try generateLd(arena, n, order, .ldset),77 .ldset => try generateLd(arena, n, order, .ldset),
78 };78 };
79 const fn_sig = if (op != .cas)79 const fn_sig = try std.fmt.allocPrint(
80 try std.fmt.allocPrint(arena, "fn {[name]s}(val: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{80 arena,
81 .name = name,81 "fn {[name]s}() align(16) callconv(.Naked) void {{",
82 .n = n.toBits(),82 .{ .name = name },
83 })83 );
84 else
85 try std.fmt.allocPrint(arena, "fn {[name]s}(expected: u{[n]d}, desired: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{
86 .name = name,
87 .n = n.toBits(),
88 });
89 try w.writeAll(fn_sig);84 try w.writeAll(fn_sig);
90 try w.writeAll(85 try w.writeAll(
91 \\86 \\
92 \\ @setRuntimeSafety(false);87 \\ @setRuntimeSafety(false);
93 \\ return asm volatile (88 \\ asm volatile (
94 \\89 \\
95 );90 );
96 var iter = std.mem.split(u8, body, "\n");91 var iter = std.mem.split(u8, body, "\n");
...@@ -99,32 +94,15 @@ fn writeFunction(...@@ -99,32 +94,15 @@ fn writeFunction(
99 try w.writeAll(line);94 try w.writeAll(line);
100 try w.writeAll("\n");95 try w.writeAll("\n");
101 }96 }
102 const constraints = if (op != .cas)
103 try std.fmt.allocPrint(arena,
104 \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}),
105 \\ : [val] "{{{[reg]s}0}}" (val),
106 \\ [ptr] "{{x1}}" (ptr),
107 \\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
108 \\ : "w15", "w16", "w17", "memory"
109 \\
110 , .{ .reg = n.register(), .ty = n.toBits() })
111 else
112 try std.fmt.allocPrint(arena,
113 \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}),
114 \\ : [expected] "{{{[reg]s}0}}" (expected),
115 \\ [desired] "{{{[reg]s}1}}" (desired),
116 \\ [ptr] "{{x2}}" (ptr),
117 \\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
118 \\ : "w15", "w16", "w17", "memory"
119 \\
120 , .{ .reg = n.register(), .ty = n.toBits() });
121
122 try w.writeAll(constraints);
123 try w.writeAll(97 try w.writeAll(
98 \\ :
99 \\ : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
100 \\ : "w15", "w16", "w17", "memory"
124 \\ );101 \\ );
102 \\ unreachable;
103 \\}
125 \\104 \\
126 );105 );
127 try w.writeAll("}\n");
128}106}
129107
130const N = enum(u8) {108const N = enum(u8) {
...@@ -185,11 +163,11 @@ const Ordering = enum {...@@ -185,11 +163,11 @@ const Ordering = enum {
185 acq_rel,163 acq_rel,
186164
187 const Defines = struct {165 const Defines = struct {
188 suff: [:0]const u8,166 suff: []const u8,
189 a: [:0]const u8,167 a: []const u8,
190 l: [:0]const u8,168 l: []const u8,
191 m: [:0]const u8,169 m: []const u8,
192 n: [:0]const u8,170 n: []const u8,
193 };171 };
194 fn defines(self: @This()) Defines {172 fn defines(self: @This()) Defines {
195 const suff = switch (self) {173 const suff = switch (self) {
...@@ -224,15 +202,6 @@ const Ordering = enum {...@@ -224,15 +202,6 @@ const Ordering = enum {
224 };202 };
225 return .{ .suff = suff, .a = a, .l = l, .m = m, .n = n };203 return .{ .suff = suff, .a = a, .l = l, .m = m, .n = n };
226 }204 }
227
228 fn capName(self: @This()) [:0]const u8 {
229 return switch (self) {
230 .relax => "Relax",
231 .acq => "Acq",
232 .rel => "Rel",
233 .acq_rel => "AcqRel",
234 };
235 }
236};205};
237206
238const LdName = enum { ldadd, ldclr, ldeor, ldset };207const LdName = enum { ldadd, ldclr, ldeor, ldset };
...@@ -244,14 +213,14 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {...@@ -244,14 +213,14 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
244 const reg = n.register();213 const reg = n.register();
245214
246 if (@enumToInt(n) < 16) {215 if (@enumToInt(n) < 16) {
247 const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m });216 const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}", .{ s_def.b, o_def.m });
248 const ldxr = try std.fmt.allocPrint(arena, "ld{s}xr{s}", .{ o_def.a, s_def.s });217 const ldxr = try std.fmt.allocPrint(arena, "ld{s}xr{s}", .{ o_def.a, s_def.s });
249 const stxr = try std.fmt.allocPrint(arena, "st{s}xr{s}", .{ o_def.l, s_def.s });218 const stxr = try std.fmt.allocPrint(arena, "st{s}xr{s}", .{ o_def.l, s_def.s });
250219
251 return try std.fmt.allocPrint(arena,220 return try std.fmt.allocPrint(arena,
252 \\ cbz w16, 8f221 \\ cbz w16, 8f
253 \\ {[cas]s}222 \\ {[cas]s}
254 \\ cbz wzr, 1f223 \\ ret
255 \\8:224 \\8:
256 \\ {[uxt]s} {[reg]s}16, {[reg]s}0225 \\ {[uxt]s} {[reg]s}16, {[reg]s}0
257 \\0:226 \\0:
...@@ -261,6 +230,7 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {...@@ -261,6 +230,7 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
261 \\ {[stxr]s} w17, {[reg]s}1, [x2]230 \\ {[stxr]s} w17, {[reg]s}1, [x2]
262 \\ cbnz w17, 0b231 \\ cbnz w17, 0b
263 \\1:232 \\1:
233 \\ ret
264 , .{234 , .{
265 .cas = cas,235 .cas = cas,
266 .uxt = s_def.uxt,236 .uxt = s_def.uxt,
...@@ -269,14 +239,14 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {...@@ -269,14 +239,14 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
269 .reg = reg,239 .reg = reg,
270 });240 });
271 } else {241 } else {
272 const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}\n", .{o_def.m});242 const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}", .{o_def.m});
273 const ldxp = try std.fmt.allocPrint(arena, "ld{s}xp", .{o_def.a});243 const ldxp = try std.fmt.allocPrint(arena, "ld{s}xp", .{o_def.a});
274 const stxp = try std.fmt.allocPrint(arena, "st{s}xp", .{o_def.l});244 const stxp = try std.fmt.allocPrint(arena, "st{s}xp", .{o_def.l});
275245
276 return try std.fmt.allocPrint(arena,246 return try std.fmt.allocPrint(arena,
277 \\ cbz w16, 8f247 \\ cbz w16, 8f
278 \\ {[casp]s}248 \\ {[casp]s}
279 \\ cbz wzr, 1f249 \\ ret
280 \\8:250 \\8:
281 \\ mov x16, x0251 \\ mov x16, x0
282 \\ mov x17, x1252 \\ mov x17, x1
...@@ -288,6 +258,7 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {...@@ -288,6 +258,7 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
288 \\ {[stxp]s} w15, x2, x3, [x4]258 \\ {[stxp]s} w15, x2, x3, [x4]
289 \\ cbnz w15, 0b259 \\ cbnz w15, 0b
290 \\1:260 \\1:
261 \\ ret
291 , .{262 , .{
292 .casp = casp,263 .casp = casp,
293 .ldxp = ldxp,264 .ldxp = ldxp,
...@@ -304,7 +275,7 @@ fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {...@@ -304,7 +275,7 @@ fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {
304 return try std.fmt.allocPrint(arena,275 return try std.fmt.allocPrint(arena,
305 \\ cbz w16, 8f276 \\ cbz w16, 8f
306 \\ .inst 0x38208020 + {[b]s} + {[n]s}277 \\ .inst 0x38208020 + {[b]s} + {[n]s}
307 \\ cbz wzr, 1f278 \\ ret
308 \\8:279 \\8:
309 \\ mov {[reg]s}16, {[reg]s}0280 \\ mov {[reg]s}16, {[reg]s}0
310 \\0:281 \\0:
...@@ -312,6 +283,7 @@ fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {...@@ -312,6 +283,7 @@ fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {
312 \\ st{[l]s}xr{[s]s} w17, {[reg]s}16, [x1]283 \\ st{[l]s}xr{[s]s} w17, {[reg]s}16, [x1]
313 \\ cbnz w17, 0b284 \\ cbnz w17, 0b
314 \\1:285 \\1:
286 \\ ret
315 , .{287 , .{
316 .b = s_def.b,288 .b = s_def.b,
317 .n = o_def.n,289 .n = o_def.n,
...@@ -343,7 +315,7 @@ fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {...@@ -343,7 +315,7 @@ fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {
343 return try std.fmt.allocPrint(arena,315 return try std.fmt.allocPrint(arena,
344 \\ cbz w16, 8f316 \\ cbz w16, 8f
345 \\ .inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}317 \\ .inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}
346 \\ cbz wzr, 1f318 \\ ret
347 \\8:319 \\8:
348 \\ mov {[reg]s}16, {[reg]s}0320 \\ mov {[reg]s}16, {[reg]s}0
349 \\0:321 \\0:
...@@ -352,6 +324,7 @@ fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {...@@ -352,6 +324,7 @@ fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {
352 \\ st{[l]s}xr{[s]s} w15, {[reg]s}17, [x1]324 \\ st{[l]s}xr{[s]s} w15, {[reg]s}17, [x1]
353 \\ cbnz w15, 0b325 \\ cbnz w15, 0b
354 \\1:326 \\1:
327 \\ ret
355 , .{328 , .{
356 .op_n = op_n,329 .op_n = op_n,
357 .b = s_def.b,330 .b = s_def.b,