Commit 58430ae6d14a0fa266b9afc892b7057dda638aed | zig

author	Andrew Kelley <andrew@ziglang.org> 2022-11-21 12:43:20-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2022-11-21 17:17:02-07:00
log	58430ae6d14a0fa266b9afc892b7057dda638aed
tree	e40ea602beeb77d01f331fc3cf63202a6ce56a5b
parent	95ee8ab77da6b76729beee6fc6ffccb07c9a2167

outline atomics: ret instead of jump to ret

After this, the machine code generated by zig matches identically to
gcc's after the differences in loading the have_lse flag.

2 files changed, 726 insertions(+), 793 deletions(-)

lib
- compiler_rt
  - lse_atomics.zig
tools
- gen_outline_atomics.zig

lib/compiler_rt/lse_atomics.zig+700-740

...	...	@@ -10,13 +10,12 @@ const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .l
10	10	/// which ARM is concerned would have too much overhead.
11	11	var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);
12	12
13		fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	13	fn __aarch64_cas1_relax() align(16) callconv(.Naked) void {
14	14	@setRuntimeSafety(false);
15		return asm volatile (
	15	asm volatile (
16	16	\\ cbz w16, 8f
17	17	\\ .inst 0x08a07c41 + 0x00000000 + 0x000000
18		\\
19		\\ cbz wzr, 1f
	18	\\ ret
20	19	\\8:
21	20	\\ uxtb w16, w0
22	21	\\0:
...	...	@@ -26,20 +25,19 @@ fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
26	25	\\ stxrb w17, w1, [x2]
27	26	\\ cbnz w17, 0b
28	27	\\1:
29		: [ret] "={w0}" (-> u8),
30		: [expected] "{w0}" (expected),
31		[desired] "{w1}" (desired),
32		[ptr] "{x2}" (ptr),
33		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	28	\\ ret
	29	:
	30	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
34	31	: "w15", "w16", "w17", "memory"
35	32	);
	33	unreachable;
36	34	}
37		fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
	35	fn __aarch64_swp1_relax() align(16) callconv(.Naked) void {
38	36	@setRuntimeSafety(false);
39		return asm volatile (
	37	asm volatile (
40	38	\\ cbz w16, 8f
41	39	\\ .inst 0x38208020 + 0x00000000 + 0x000000
42		\\ cbz wzr, 1f
	40	\\ ret
43	41	\\8:
44	42	\\ mov w16, w0
45	43	\\0:
...	...	@@ -47,19 +45,19 @@ fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
47	45	\\ stxrb w17, w16, [x1]
48	46	\\ cbnz w17, 0b
49	47	\\1:
50		: [ret] "={w0}" (-> u8),
51		: [val] "{w0}" (val),
52		[ptr] "{x1}" (ptr),
53		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	48	\\ ret
	49	:
	50	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
54	51	: "w15", "w16", "w17", "memory"
55	52	);
	53	unreachable;
56	54	}
57		fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
	55	fn __aarch64_ldadd1_relax() align(16) callconv(.Naked) void {
58	56	@setRuntimeSafety(false);
59		return asm volatile (
	57	asm volatile (
60	58	\\ cbz w16, 8f
61	59	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
62		\\ cbz wzr, 1f
	60	\\ ret
63	61	\\8:
64	62	\\ mov w16, w0
65	63	\\0:
...	...	@@ -68,19 +66,19 @@ fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
68	66	\\ stxrb w15, w17, [x1]
69	67	\\ cbnz w15, 0b
70	68	\\1:
71		: [ret] "={w0}" (-> u8),
72		: [val] "{w0}" (val),
73		[ptr] "{x1}" (ptr),
74		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	69	\\ ret
	70	:
	71	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
75	72	: "w15", "w16", "w17", "memory"
76	73	);
	74	unreachable;
77	75	}
78		fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
	76	fn __aarch64_ldclr1_relax() align(16) callconv(.Naked) void {
79	77	@setRuntimeSafety(false);
80		return asm volatile (
	78	asm volatile (
81	79	\\ cbz w16, 8f
82	80	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
83		\\ cbz wzr, 1f
	81	\\ ret
84	82	\\8:
85	83	\\ mov w16, w0
86	84	\\0:
...	...	@@ -89,19 +87,19 @@ fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
89	87	\\ stxrb w15, w17, [x1]
90	88	\\ cbnz w15, 0b
91	89	\\1:
92		: [ret] "={w0}" (-> u8),
93		: [val] "{w0}" (val),
94		[ptr] "{x1}" (ptr),
95		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	90	\\ ret
	91	:
	92	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
96	93	: "w15", "w16", "w17", "memory"
97	94	);
	95	unreachable;
98	96	}
99		fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
	97	fn __aarch64_ldeor1_relax() align(16) callconv(.Naked) void {
100	98	@setRuntimeSafety(false);
101		return asm volatile (
	99	asm volatile (
102	100	\\ cbz w16, 8f
103	101	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
104		\\ cbz wzr, 1f
	102	\\ ret
105	103	\\8:
106	104	\\ mov w16, w0
107	105	\\0:
...	...	@@ -110,19 +108,19 @@ fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
110	108	\\ stxrb w15, w17, [x1]
111	109	\\ cbnz w15, 0b
112	110	\\1:
113		: [ret] "={w0}" (-> u8),
114		: [val] "{w0}" (val),
115		[ptr] "{x1}" (ptr),
116		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	111	\\ ret
	112	:
	113	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
117	114	: "w15", "w16", "w17", "memory"
118	115	);
	116	unreachable;
119	117	}
120		fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
	118	fn __aarch64_ldset1_relax() align(16) callconv(.Naked) void {
121	119	@setRuntimeSafety(false);
122		return asm volatile (
	120	asm volatile (
123	121	\\ cbz w16, 8f
124	122	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
125		\\ cbz wzr, 1f
	123	\\ ret
126	124	\\8:
127	125	\\ mov w16, w0
128	126	\\0:
...	...	@@ -131,20 +129,19 @@ fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
131	129	\\ stxrb w15, w17, [x1]
132	130	\\ cbnz w15, 0b
133	131	\\1:
134		: [ret] "={w0}" (-> u8),
135		: [val] "{w0}" (val),
136		[ptr] "{x1}" (ptr),
137		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	132	\\ ret
	133	:
	134	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
138	135	: "w15", "w16", "w17", "memory"
139	136	);
	137	unreachable;
140	138	}
141		fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	139	fn __aarch64_cas1_acq() align(16) callconv(.Naked) void {
142	140	@setRuntimeSafety(false);
143		return asm volatile (
	141	asm volatile (
144	142	\\ cbz w16, 8f
145	143	\\ .inst 0x08a07c41 + 0x00000000 + 0x400000
146		\\
147		\\ cbz wzr, 1f
	144	\\ ret
148	145	\\8:
149	146	\\ uxtb w16, w0
150	147	\\0:
...	...	@@ -154,20 +151,19 @@ fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
154	151	\\ stxrb w17, w1, [x2]
155	152	\\ cbnz w17, 0b
156	153	\\1:
157		: [ret] "={w0}" (-> u8),
158		: [expected] "{w0}" (expected),
159		[desired] "{w1}" (desired),
160		[ptr] "{x2}" (ptr),
161		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	154	\\ ret
	155	:
	156	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
162	157	: "w15", "w16", "w17", "memory"
163	158	);
	159	unreachable;
164	160	}
165		fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
	161	fn __aarch64_swp1_acq() align(16) callconv(.Naked) void {
166	162	@setRuntimeSafety(false);
167		return asm volatile (
	163	asm volatile (
168	164	\\ cbz w16, 8f
169	165	\\ .inst 0x38208020 + 0x00000000 + 0x800000
170		\\ cbz wzr, 1f
	166	\\ ret
171	167	\\8:
172	168	\\ mov w16, w0
173	169	\\0:
...	...	@@ -175,19 +171,19 @@ fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
175	171	\\ stxrb w17, w16, [x1]
176	172	\\ cbnz w17, 0b
177	173	\\1:
178		: [ret] "={w0}" (-> u8),
179		: [val] "{w0}" (val),
180		[ptr] "{x1}" (ptr),
181		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	174	\\ ret
	175	:
	176	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
182	177	: "w15", "w16", "w17", "memory"
183	178	);
	179	unreachable;
184	180	}
185		fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
	181	fn __aarch64_ldadd1_acq() align(16) callconv(.Naked) void {
186	182	@setRuntimeSafety(false);
187		return asm volatile (
	183	asm volatile (
188	184	\\ cbz w16, 8f
189	185	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
190		\\ cbz wzr, 1f
	186	\\ ret
191	187	\\8:
192	188	\\ mov w16, w0
193	189	\\0:
...	...	@@ -196,19 +192,19 @@ fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
196	192	\\ stxrb w15, w17, [x1]
197	193	\\ cbnz w15, 0b
198	194	\\1:
199		: [ret] "={w0}" (-> u8),
200		: [val] "{w0}" (val),
201		[ptr] "{x1}" (ptr),
202		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	195	\\ ret
	196	:
	197	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
203	198	: "w15", "w16", "w17", "memory"
204	199	);
	200	unreachable;
205	201	}
206		fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
	202	fn __aarch64_ldclr1_acq() align(16) callconv(.Naked) void {
207	203	@setRuntimeSafety(false);
208		return asm volatile (
	204	asm volatile (
209	205	\\ cbz w16, 8f
210	206	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
211		\\ cbz wzr, 1f
	207	\\ ret
212	208	\\8:
213	209	\\ mov w16, w0
214	210	\\0:
...	...	@@ -217,19 +213,19 @@ fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
217	213	\\ stxrb w15, w17, [x1]
218	214	\\ cbnz w15, 0b
219	215	\\1:
220		: [ret] "={w0}" (-> u8),
221		: [val] "{w0}" (val),
222		[ptr] "{x1}" (ptr),
223		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	216	\\ ret
	217	:
	218	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
224	219	: "w15", "w16", "w17", "memory"
225	220	);
	221	unreachable;
226	222	}
227		fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
	223	fn __aarch64_ldeor1_acq() align(16) callconv(.Naked) void {
228	224	@setRuntimeSafety(false);
229		return asm volatile (
	225	asm volatile (
230	226	\\ cbz w16, 8f
231	227	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
232		\\ cbz wzr, 1f
	228	\\ ret
233	229	\\8:
234	230	\\ mov w16, w0
235	231	\\0:
...	...	@@ -238,19 +234,19 @@ fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
238	234	\\ stxrb w15, w17, [x1]
239	235	\\ cbnz w15, 0b
240	236	\\1:
241		: [ret] "={w0}" (-> u8),
242		: [val] "{w0}" (val),
243		[ptr] "{x1}" (ptr),
244		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	237	\\ ret
	238	:
	239	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
245	240	: "w15", "w16", "w17", "memory"
246	241	);
	242	unreachable;
247	243	}
248		fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
	244	fn __aarch64_ldset1_acq() align(16) callconv(.Naked) void {
249	245	@setRuntimeSafety(false);
250		return asm volatile (
	246	asm volatile (
251	247	\\ cbz w16, 8f
252	248	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
253		\\ cbz wzr, 1f
	249	\\ ret
254	250	\\8:
255	251	\\ mov w16, w0
256	252	\\0:
...	...	@@ -259,20 +255,19 @@ fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
259	255	\\ stxrb w15, w17, [x1]
260	256	\\ cbnz w15, 0b
261	257	\\1:
262		: [ret] "={w0}" (-> u8),
263		: [val] "{w0}" (val),
264		[ptr] "{x1}" (ptr),
265		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	258	\\ ret
	259	:
	260	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
266	261	: "w15", "w16", "w17", "memory"
267	262	);
	263	unreachable;
268	264	}
269		fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	265	fn __aarch64_cas1_rel() align(16) callconv(.Naked) void {
270	266	@setRuntimeSafety(false);
271		return asm volatile (
	267	asm volatile (
272	268	\\ cbz w16, 8f
273	269	\\ .inst 0x08a07c41 + 0x00000000 + 0x008000
274		\\
275		\\ cbz wzr, 1f
	270	\\ ret
276	271	\\8:
277	272	\\ uxtb w16, w0
278	273	\\0:
...	...	@@ -282,20 +277,19 @@ fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
282	277	\\ stlxrb w17, w1, [x2]
283	278	\\ cbnz w17, 0b
284	279	\\1:
285		: [ret] "={w0}" (-> u8),
286		: [expected] "{w0}" (expected),
287		[desired] "{w1}" (desired),
288		[ptr] "{x2}" (ptr),
289		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	280	\\ ret
	281	:
	282	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
290	283	: "w15", "w16", "w17", "memory"
291	284	);
	285	unreachable;
292	286	}
293		fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	287	fn __aarch64_swp1_rel() align(16) callconv(.Naked) void {
294	288	@setRuntimeSafety(false);
295		return asm volatile (
	289	asm volatile (
296	290	\\ cbz w16, 8f
297	291	\\ .inst 0x38208020 + 0x00000000 + 0x400000
298		\\ cbz wzr, 1f
	292	\\ ret
299	293	\\8:
300	294	\\ mov w16, w0
301	295	\\0:
...	...	@@ -303,19 +297,19 @@ fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
303	297	\\ stlxrb w17, w16, [x1]
304	298	\\ cbnz w17, 0b
305	299	\\1:
306		: [ret] "={w0}" (-> u8),
307		: [val] "{w0}" (val),
308		[ptr] "{x1}" (ptr),
309		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	300	\\ ret
	301	:
	302	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
310	303	: "w15", "w16", "w17", "memory"
311	304	);
	305	unreachable;
312	306	}
313		fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	307	fn __aarch64_ldadd1_rel() align(16) callconv(.Naked) void {
314	308	@setRuntimeSafety(false);
315		return asm volatile (
	309	asm volatile (
316	310	\\ cbz w16, 8f
317	311	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
318		\\ cbz wzr, 1f
	312	\\ ret
319	313	\\8:
320	314	\\ mov w16, w0
321	315	\\0:
...	...	@@ -324,19 +318,19 @@ fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
324	318	\\ stlxrb w15, w17, [x1]
325	319	\\ cbnz w15, 0b
326	320	\\1:
327		: [ret] "={w0}" (-> u8),
328		: [val] "{w0}" (val),
329		[ptr] "{x1}" (ptr),
330		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	321	\\ ret
	322	:
	323	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
331	324	: "w15", "w16", "w17", "memory"
332	325	);
	326	unreachable;
333	327	}
334		fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	328	fn __aarch64_ldclr1_rel() align(16) callconv(.Naked) void {
335	329	@setRuntimeSafety(false);
336		return asm volatile (
	330	asm volatile (
337	331	\\ cbz w16, 8f
338	332	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
339		\\ cbz wzr, 1f
	333	\\ ret
340	334	\\8:
341	335	\\ mov w16, w0
342	336	\\0:
...	...	@@ -345,19 +339,19 @@ fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
345	339	\\ stlxrb w15, w17, [x1]
346	340	\\ cbnz w15, 0b
347	341	\\1:
348		: [ret] "={w0}" (-> u8),
349		: [val] "{w0}" (val),
350		[ptr] "{x1}" (ptr),
351		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	342	\\ ret
	343	:
	344	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
352	345	: "w15", "w16", "w17", "memory"
353	346	);
	347	unreachable;
354	348	}
355		fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	349	fn __aarch64_ldeor1_rel() align(16) callconv(.Naked) void {
356	350	@setRuntimeSafety(false);
357		return asm volatile (
	351	asm volatile (
358	352	\\ cbz w16, 8f
359	353	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
360		\\ cbz wzr, 1f
	354	\\ ret
361	355	\\8:
362	356	\\ mov w16, w0
363	357	\\0:
...	...	@@ -366,19 +360,19 @@ fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
366	360	\\ stlxrb w15, w17, [x1]
367	361	\\ cbnz w15, 0b
368	362	\\1:
369		: [ret] "={w0}" (-> u8),
370		: [val] "{w0}" (val),
371		[ptr] "{x1}" (ptr),
372		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	363	\\ ret
	364	:
	365	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
373	366	: "w15", "w16", "w17", "memory"
374	367	);
	368	unreachable;
375	369	}
376		fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	370	fn __aarch64_ldset1_rel() align(16) callconv(.Naked) void {
377	371	@setRuntimeSafety(false);
378		return asm volatile (
	372	asm volatile (
379	373	\\ cbz w16, 8f
380	374	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
381		\\ cbz wzr, 1f
	375	\\ ret
382	376	\\8:
383	377	\\ mov w16, w0
384	378	\\0:
...	...	@@ -387,20 +381,19 @@ fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
387	381	\\ stlxrb w15, w17, [x1]
388	382	\\ cbnz w15, 0b
389	383	\\1:
390		: [ret] "={w0}" (-> u8),
391		: [val] "{w0}" (val),
392		[ptr] "{x1}" (ptr),
393		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	384	\\ ret
	385	:
	386	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
394	387	: "w15", "w16", "w17", "memory"
395	388	);
	389	unreachable;
396	390	}
397		fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	391	fn __aarch64_cas1_acq_rel() align(16) callconv(.Naked) void {
398	392	@setRuntimeSafety(false);
399		return asm volatile (
	393	asm volatile (
400	394	\\ cbz w16, 8f
401	395	\\ .inst 0x08a07c41 + 0x00000000 + 0x408000
402		\\
403		\\ cbz wzr, 1f
	396	\\ ret
404	397	\\8:
405	398	\\ uxtb w16, w0
406	399	\\0:
...	...	@@ -410,20 +403,19 @@ fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
410	403	\\ stlxrb w17, w1, [x2]
411	404	\\ cbnz w17, 0b
412	405	\\1:
413		: [ret] "={w0}" (-> u8),
414		: [expected] "{w0}" (expected),
415		[desired] "{w1}" (desired),
416		[ptr] "{x2}" (ptr),
417		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	406	\\ ret
	407	:
	408	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
418	409	: "w15", "w16", "w17", "memory"
419	410	);
	411	unreachable;
420	412	}
421		fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	413	fn __aarch64_swp1_acq_rel() align(16) callconv(.Naked) void {
422	414	@setRuntimeSafety(false);
423		return asm volatile (
	415	asm volatile (
424	416	\\ cbz w16, 8f
425	417	\\ .inst 0x38208020 + 0x00000000 + 0xc00000
426		\\ cbz wzr, 1f
	418	\\ ret
427	419	\\8:
428	420	\\ mov w16, w0
429	421	\\0:
...	...	@@ -431,19 +423,19 @@ fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
431	423	\\ stlxrb w17, w16, [x1]
432	424	\\ cbnz w17, 0b
433	425	\\1:
434		: [ret] "={w0}" (-> u8),
435		: [val] "{w0}" (val),
436		[ptr] "{x1}" (ptr),
437		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	426	\\ ret
	427	:
	428	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
438	429	: "w15", "w16", "w17", "memory"
439	430	);
	431	unreachable;
440	432	}
441		fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	433	fn __aarch64_ldadd1_acq_rel() align(16) callconv(.Naked) void {
442	434	@setRuntimeSafety(false);
443		return asm volatile (
	435	asm volatile (
444	436	\\ cbz w16, 8f
445	437	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
446		\\ cbz wzr, 1f
	438	\\ ret
447	439	\\8:
448	440	\\ mov w16, w0
449	441	\\0:
...	...	@@ -452,19 +444,19 @@ fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
452	444	\\ stlxrb w15, w17, [x1]
453	445	\\ cbnz w15, 0b
454	446	\\1:
455		: [ret] "={w0}" (-> u8),
456		: [val] "{w0}" (val),
457		[ptr] "{x1}" (ptr),
458		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	447	\\ ret
	448	:
	449	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
459	450	: "w15", "w16", "w17", "memory"
460	451	);
	452	unreachable;
461	453	}
462		fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	454	fn __aarch64_ldclr1_acq_rel() align(16) callconv(.Naked) void {
463	455	@setRuntimeSafety(false);
464		return asm volatile (
	456	asm volatile (
465	457	\\ cbz w16, 8f
466	458	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
467		\\ cbz wzr, 1f
	459	\\ ret
468	460	\\8:
469	461	\\ mov w16, w0
470	462	\\0:
...	...	@@ -473,19 +465,19 @@ fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
473	465	\\ stlxrb w15, w17, [x1]
474	466	\\ cbnz w15, 0b
475	467	\\1:
476		: [ret] "={w0}" (-> u8),
477		: [val] "{w0}" (val),
478		[ptr] "{x1}" (ptr),
479		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	468	\\ ret
	469	:
	470	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
480	471	: "w15", "w16", "w17", "memory"
481	472	);
	473	unreachable;
482	474	}
483		fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	475	fn __aarch64_ldeor1_acq_rel() align(16) callconv(.Naked) void {
484	476	@setRuntimeSafety(false);
485		return asm volatile (
	477	asm volatile (
486	478	\\ cbz w16, 8f
487	479	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
488		\\ cbz wzr, 1f
	480	\\ ret
489	481	\\8:
490	482	\\ mov w16, w0
491	483	\\0:
...	...	@@ -494,19 +486,19 @@ fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
494	486	\\ stlxrb w15, w17, [x1]
495	487	\\ cbnz w15, 0b
496	488	\\1:
497		: [ret] "={w0}" (-> u8),
498		: [val] "{w0}" (val),
499		[ptr] "{x1}" (ptr),
500		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	489	\\ ret
	490	:
	491	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
501	492	: "w15", "w16", "w17", "memory"
502	493	);
	494	unreachable;
503	495	}
504		fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
	496	fn __aarch64_ldset1_acq_rel() align(16) callconv(.Naked) void {
505	497	@setRuntimeSafety(false);
506		return asm volatile (
	498	asm volatile (
507	499	\\ cbz w16, 8f
508	500	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
509		\\ cbz wzr, 1f
	501	\\ ret
510	502	\\8:
511	503	\\ mov w16, w0
512	504	\\0:
...	...	@@ -515,20 +507,19 @@ fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
515	507	\\ stlxrb w15, w17, [x1]
516	508	\\ cbnz w15, 0b
517	509	\\1:
518		: [ret] "={w0}" (-> u8),
519		: [val] "{w0}" (val),
520		[ptr] "{x1}" (ptr),
521		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	510	\\ ret
	511	:
	512	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
522	513	: "w15", "w16", "w17", "memory"
523	514	);
	515	unreachable;
524	516	}
525		fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
	517	fn __aarch64_cas2_relax() align(16) callconv(.Naked) void {
526	518	@setRuntimeSafety(false);
527		return asm volatile (
	519	asm volatile (
528	520	\\ cbz w16, 8f
529	521	\\ .inst 0x08a07c41 + 0x40000000 + 0x000000
530		\\
531		\\ cbz wzr, 1f
	522	\\ ret
532	523	\\8:
533	524	\\ uxth w16, w0
534	525	\\0:
...	...	@@ -538,20 +529,19 @@ fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16
538	529	\\ stxrh w17, w1, [x2]
539	530	\\ cbnz w17, 0b
540	531	\\1:
541		: [ret] "={w0}" (-> u16),
542		: [expected] "{w0}" (expected),
543		[desired] "{w1}" (desired),
544		[ptr] "{x2}" (ptr),
545		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	532	\\ ret
	533	:
	534	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
546	535	: "w15", "w16", "w17", "memory"
547	536	);
	537	unreachable;
548	538	}
549		fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
	539	fn __aarch64_swp2_relax() align(16) callconv(.Naked) void {
550	540	@setRuntimeSafety(false);
551		return asm volatile (
	541	asm volatile (
552	542	\\ cbz w16, 8f
553	543	\\ .inst 0x38208020 + 0x40000000 + 0x000000
554		\\ cbz wzr, 1f
	544	\\ ret
555	545	\\8:
556	546	\\ mov w16, w0
557	547	\\0:
...	...	@@ -559,19 +549,19 @@ fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
559	549	\\ stxrh w17, w16, [x1]
560	550	\\ cbnz w17, 0b
561	551	\\1:
562		: [ret] "={w0}" (-> u16),
563		: [val] "{w0}" (val),
564		[ptr] "{x1}" (ptr),
565		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	552	\\ ret
	553	:
	554	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
566	555	: "w15", "w16", "w17", "memory"
567	556	);
	557	unreachable;
568	558	}
569		fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
	559	fn __aarch64_ldadd2_relax() align(16) callconv(.Naked) void {
570	560	@setRuntimeSafety(false);
571		return asm volatile (
	561	asm volatile (
572	562	\\ cbz w16, 8f
573	563	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
574		\\ cbz wzr, 1f
	564	\\ ret
575	565	\\8:
576	566	\\ mov w16, w0
577	567	\\0:
...	...	@@ -580,19 +570,19 @@ fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
580	570	\\ stxrh w15, w17, [x1]
581	571	\\ cbnz w15, 0b
582	572	\\1:
583		: [ret] "={w0}" (-> u16),
584		: [val] "{w0}" (val),
585		[ptr] "{x1}" (ptr),
586		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	573	\\ ret
	574	:
	575	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
587	576	: "w15", "w16", "w17", "memory"
588	577	);
	578	unreachable;
589	579	}
590		fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
	580	fn __aarch64_ldclr2_relax() align(16) callconv(.Naked) void {
591	581	@setRuntimeSafety(false);
592		return asm volatile (
	582	asm volatile (
593	583	\\ cbz w16, 8f
594	584	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
595		\\ cbz wzr, 1f
	585	\\ ret
596	586	\\8:
597	587	\\ mov w16, w0
598	588	\\0:
...	...	@@ -601,19 +591,19 @@ fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
601	591	\\ stxrh w15, w17, [x1]
602	592	\\ cbnz w15, 0b
603	593	\\1:
604		: [ret] "={w0}" (-> u16),
605		: [val] "{w0}" (val),
606		[ptr] "{x1}" (ptr),
607		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	594	\\ ret
	595	:
	596	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
608	597	: "w15", "w16", "w17", "memory"
609	598	);
	599	unreachable;
610	600	}
611		fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
	601	fn __aarch64_ldeor2_relax() align(16) callconv(.Naked) void {
612	602	@setRuntimeSafety(false);
613		return asm volatile (
	603	asm volatile (
614	604	\\ cbz w16, 8f
615	605	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
616		\\ cbz wzr, 1f
	606	\\ ret
617	607	\\8:
618	608	\\ mov w16, w0
619	609	\\0:
...	...	@@ -622,19 +612,19 @@ fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
622	612	\\ stxrh w15, w17, [x1]
623	613	\\ cbnz w15, 0b
624	614	\\1:
625		: [ret] "={w0}" (-> u16),
626		: [val] "{w0}" (val),
627		[ptr] "{x1}" (ptr),
628		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	615	\\ ret
	616	:
	617	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
629	618	: "w15", "w16", "w17", "memory"
630	619	);
	620	unreachable;
631	621	}
632		fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
	622	fn __aarch64_ldset2_relax() align(16) callconv(.Naked) void {
633	623	@setRuntimeSafety(false);
634		return asm volatile (
	624	asm volatile (
635	625	\\ cbz w16, 8f
636	626	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
637		\\ cbz wzr, 1f
	627	\\ ret
638	628	\\8:
639	629	\\ mov w16, w0
640	630	\\0:
...	...	@@ -643,20 +633,19 @@ fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
643	633	\\ stxrh w15, w17, [x1]
644	634	\\ cbnz w15, 0b
645	635	\\1:
646		: [ret] "={w0}" (-> u16),
647		: [val] "{w0}" (val),
648		[ptr] "{x1}" (ptr),
649		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	636	\\ ret
	637	:
	638	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
650	639	: "w15", "w16", "w17", "memory"
651	640	);
	641	unreachable;
652	642	}
653		fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
	643	fn __aarch64_cas2_acq() align(16) callconv(.Naked) void {
654	644	@setRuntimeSafety(false);
655		return asm volatile (
	645	asm volatile (
656	646	\\ cbz w16, 8f
657	647	\\ .inst 0x08a07c41 + 0x40000000 + 0x400000
658		\\
659		\\ cbz wzr, 1f
	648	\\ ret
660	649	\\8:
661	650	\\ uxth w16, w0
662	651	\\0:
...	...	@@ -666,20 +655,19 @@ fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
666	655	\\ stxrh w17, w1, [x2]
667	656	\\ cbnz w17, 0b
668	657	\\1:
669		: [ret] "={w0}" (-> u16),
670		: [expected] "{w0}" (expected),
671		[desired] "{w1}" (desired),
672		[ptr] "{x2}" (ptr),
673		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	658	\\ ret
	659	:
	660	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
674	661	: "w15", "w16", "w17", "memory"
675	662	);
	663	unreachable;
676	664	}
677		fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
	665	fn __aarch64_swp2_acq() align(16) callconv(.Naked) void {
678	666	@setRuntimeSafety(false);
679		return asm volatile (
	667	asm volatile (
680	668	\\ cbz w16, 8f
681	669	\\ .inst 0x38208020 + 0x40000000 + 0x800000
682		\\ cbz wzr, 1f
	670	\\ ret
683	671	\\8:
684	672	\\ mov w16, w0
685	673	\\0:
...	...	@@ -687,19 +675,19 @@ fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
687	675	\\ stxrh w17, w16, [x1]
688	676	\\ cbnz w17, 0b
689	677	\\1:
690		: [ret] "={w0}" (-> u16),
691		: [val] "{w0}" (val),
692		[ptr] "{x1}" (ptr),
693		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	678	\\ ret
	679	:
	680	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
694	681	: "w15", "w16", "w17", "memory"
695	682	);
	683	unreachable;
696	684	}
697		fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
	685	fn __aarch64_ldadd2_acq() align(16) callconv(.Naked) void {
698	686	@setRuntimeSafety(false);
699		return asm volatile (
	687	asm volatile (
700	688	\\ cbz w16, 8f
701	689	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
702		\\ cbz wzr, 1f
	690	\\ ret
703	691	\\8:
704	692	\\ mov w16, w0
705	693	\\0:
...	...	@@ -708,19 +696,19 @@ fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
708	696	\\ stxrh w15, w17, [x1]
709	697	\\ cbnz w15, 0b
710	698	\\1:
711		: [ret] "={w0}" (-> u16),
712		: [val] "{w0}" (val),
713		[ptr] "{x1}" (ptr),
714		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	699	\\ ret
	700	:
	701	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
715	702	: "w15", "w16", "w17", "memory"
716	703	);
	704	unreachable;
717	705	}
718		fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
	706	fn __aarch64_ldclr2_acq() align(16) callconv(.Naked) void {
719	707	@setRuntimeSafety(false);
720		return asm volatile (
	708	asm volatile (
721	709	\\ cbz w16, 8f
722	710	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
723		\\ cbz wzr, 1f
	711	\\ ret
724	712	\\8:
725	713	\\ mov w16, w0
726	714	\\0:
...	...	@@ -729,19 +717,19 @@ fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
729	717	\\ stxrh w15, w17, [x1]
730	718	\\ cbnz w15, 0b
731	719	\\1:
732		: [ret] "={w0}" (-> u16),
733		: [val] "{w0}" (val),
734		[ptr] "{x1}" (ptr),
735		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	720	\\ ret
	721	:
	722	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
736	723	: "w15", "w16", "w17", "memory"
737	724	);
	725	unreachable;
738	726	}
739		fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
	727	fn __aarch64_ldeor2_acq() align(16) callconv(.Naked) void {
740	728	@setRuntimeSafety(false);
741		return asm volatile (
	729	asm volatile (
742	730	\\ cbz w16, 8f
743	731	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
744		\\ cbz wzr, 1f
	732	\\ ret
745	733	\\8:
746	734	\\ mov w16, w0
747	735	\\0:
...	...	@@ -750,19 +738,19 @@ fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
750	738	\\ stxrh w15, w17, [x1]
751	739	\\ cbnz w15, 0b
752	740	\\1:
753		: [ret] "={w0}" (-> u16),
754		: [val] "{w0}" (val),
755		[ptr] "{x1}" (ptr),
756		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	741	\\ ret
	742	:
	743	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
757	744	: "w15", "w16", "w17", "memory"
758	745	);
	746	unreachable;
759	747	}
760		fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
	748	fn __aarch64_ldset2_acq() align(16) callconv(.Naked) void {
761	749	@setRuntimeSafety(false);
762		return asm volatile (
	750	asm volatile (
763	751	\\ cbz w16, 8f
764	752	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
765		\\ cbz wzr, 1f
	753	\\ ret
766	754	\\8:
767	755	\\ mov w16, w0
768	756	\\0:
...	...	@@ -771,20 +759,19 @@ fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
771	759	\\ stxrh w15, w17, [x1]
772	760	\\ cbnz w15, 0b
773	761	\\1:
774		: [ret] "={w0}" (-> u16),
775		: [val] "{w0}" (val),
776		[ptr] "{x1}" (ptr),
777		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	762	\\ ret
	763	:
	764	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
778	765	: "w15", "w16", "w17", "memory"
779	766	);
	767	unreachable;
780	768	}
781		fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
	769	fn __aarch64_cas2_rel() align(16) callconv(.Naked) void {
782	770	@setRuntimeSafety(false);
783		return asm volatile (
	771	asm volatile (
784	772	\\ cbz w16, 8f
785	773	\\ .inst 0x08a07c41 + 0x40000000 + 0x008000
786		\\
787		\\ cbz wzr, 1f
	774	\\ ret
788	775	\\8:
789	776	\\ uxth w16, w0
790	777	\\0:
...	...	@@ -794,20 +781,19 @@ fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
794	781	\\ stlxrh w17, w1, [x2]
795	782	\\ cbnz w17, 0b
796	783	\\1:
797		: [ret] "={w0}" (-> u16),
798		: [expected] "{w0}" (expected),
799		[desired] "{w1}" (desired),
800		[ptr] "{x2}" (ptr),
801		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	784	\\ ret
	785	:
	786	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
802	787	: "w15", "w16", "w17", "memory"
803	788	);
	789	unreachable;
804	790	}
805		fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	791	fn __aarch64_swp2_rel() align(16) callconv(.Naked) void {
806	792	@setRuntimeSafety(false);
807		return asm volatile (
	793	asm volatile (
808	794	\\ cbz w16, 8f
809	795	\\ .inst 0x38208020 + 0x40000000 + 0x400000
810		\\ cbz wzr, 1f
	796	\\ ret
811	797	\\8:
812	798	\\ mov w16, w0
813	799	\\0:
...	...	@@ -815,19 +801,19 @@ fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
815	801	\\ stlxrh w17, w16, [x1]
816	802	\\ cbnz w17, 0b
817	803	\\1:
818		: [ret] "={w0}" (-> u16),
819		: [val] "{w0}" (val),
820		[ptr] "{x1}" (ptr),
821		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	804	\\ ret
	805	:
	806	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
822	807	: "w15", "w16", "w17", "memory"
823	808	);
	809	unreachable;
824	810	}
825		fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	811	fn __aarch64_ldadd2_rel() align(16) callconv(.Naked) void {
826	812	@setRuntimeSafety(false);
827		return asm volatile (
	813	asm volatile (
828	814	\\ cbz w16, 8f
829	815	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
830		\\ cbz wzr, 1f
	816	\\ ret
831	817	\\8:
832	818	\\ mov w16, w0
833	819	\\0:
...	...	@@ -836,19 +822,19 @@ fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
836	822	\\ stlxrh w15, w17, [x1]
837	823	\\ cbnz w15, 0b
838	824	\\1:
839		: [ret] "={w0}" (-> u16),
840		: [val] "{w0}" (val),
841		[ptr] "{x1}" (ptr),
842		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	825	\\ ret
	826	:
	827	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
843	828	: "w15", "w16", "w17", "memory"
844	829	);
	830	unreachable;
845	831	}
846		fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	832	fn __aarch64_ldclr2_rel() align(16) callconv(.Naked) void {
847	833	@setRuntimeSafety(false);
848		return asm volatile (
	834	asm volatile (
849	835	\\ cbz w16, 8f
850	836	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
851		\\ cbz wzr, 1f
	837	\\ ret
852	838	\\8:
853	839	\\ mov w16, w0
854	840	\\0:
...	...	@@ -857,19 +843,19 @@ fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
857	843	\\ stlxrh w15, w17, [x1]
858	844	\\ cbnz w15, 0b
859	845	\\1:
860		: [ret] "={w0}" (-> u16),
861		: [val] "{w0}" (val),
862		[ptr] "{x1}" (ptr),
863		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	846	\\ ret
	847	:
	848	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
864	849	: "w15", "w16", "w17", "memory"
865	850	);
	851	unreachable;
866	852	}
867		fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	853	fn __aarch64_ldeor2_rel() align(16) callconv(.Naked) void {
868	854	@setRuntimeSafety(false);
869		return asm volatile (
	855	asm volatile (
870	856	\\ cbz w16, 8f
871	857	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
872		\\ cbz wzr, 1f
	858	\\ ret
873	859	\\8:
874	860	\\ mov w16, w0
875	861	\\0:
...	...	@@ -878,19 +864,19 @@ fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
878	864	\\ stlxrh w15, w17, [x1]
879	865	\\ cbnz w15, 0b
880	866	\\1:
881		: [ret] "={w0}" (-> u16),
882		: [val] "{w0}" (val),
883		[ptr] "{x1}" (ptr),
884		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	867	\\ ret
	868	:
	869	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
885	870	: "w15", "w16", "w17", "memory"
886	871	);
	872	unreachable;
887	873	}
888		fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	874	fn __aarch64_ldset2_rel() align(16) callconv(.Naked) void {
889	875	@setRuntimeSafety(false);
890		return asm volatile (
	876	asm volatile (
891	877	\\ cbz w16, 8f
892	878	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
893		\\ cbz wzr, 1f
	879	\\ ret
894	880	\\8:
895	881	\\ mov w16, w0
896	882	\\0:
...	...	@@ -899,20 +885,19 @@ fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
899	885	\\ stlxrh w15, w17, [x1]
900	886	\\ cbnz w15, 0b
901	887	\\1:
902		: [ret] "={w0}" (-> u16),
903		: [val] "{w0}" (val),
904		[ptr] "{x1}" (ptr),
905		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	888	\\ ret
	889	:
	890	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
906	891	: "w15", "w16", "w17", "memory"
907	892	);
	893	unreachable;
908	894	}
909		fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
	895	fn __aarch64_cas2_acq_rel() align(16) callconv(.Naked) void {
910	896	@setRuntimeSafety(false);
911		return asm volatile (
	897	asm volatile (
912	898	\\ cbz w16, 8f
913	899	\\ .inst 0x08a07c41 + 0x40000000 + 0x408000
914		\\
915		\\ cbz wzr, 1f
	900	\\ ret
916	901	\\8:
917	902	\\ uxth w16, w0
918	903	\\0:
...	...	@@ -922,20 +907,19 @@ fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u
922	907	\\ stlxrh w17, w1, [x2]
923	908	\\ cbnz w17, 0b
924	909	\\1:
925		: [ret] "={w0}" (-> u16),
926		: [expected] "{w0}" (expected),
927		[desired] "{w1}" (desired),
928		[ptr] "{x2}" (ptr),
929		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	910	\\ ret
	911	:
	912	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
930	913	: "w15", "w16", "w17", "memory"
931	914	);
	915	unreachable;
932	916	}
933		fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	917	fn __aarch64_swp2_acq_rel() align(16) callconv(.Naked) void {
934	918	@setRuntimeSafety(false);
935		return asm volatile (
	919	asm volatile (
936	920	\\ cbz w16, 8f
937	921	\\ .inst 0x38208020 + 0x40000000 + 0xc00000
938		\\ cbz wzr, 1f
	922	\\ ret
939	923	\\8:
940	924	\\ mov w16, w0
941	925	\\0:
...	...	@@ -943,19 +927,19 @@ fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
943	927	\\ stlxrh w17, w16, [x1]
944	928	\\ cbnz w17, 0b
945	929	\\1:
946		: [ret] "={w0}" (-> u16),
947		: [val] "{w0}" (val),
948		[ptr] "{x1}" (ptr),
949		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	930	\\ ret
	931	:
	932	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
950	933	: "w15", "w16", "w17", "memory"
951	934	);
	935	unreachable;
952	936	}
953		fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	937	fn __aarch64_ldadd2_acq_rel() align(16) callconv(.Naked) void {
954	938	@setRuntimeSafety(false);
955		return asm volatile (
	939	asm volatile (
956	940	\\ cbz w16, 8f
957	941	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
958		\\ cbz wzr, 1f
	942	\\ ret
959	943	\\8:
960	944	\\ mov w16, w0
961	945	\\0:
...	...	@@ -964,19 +948,19 @@ fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
964	948	\\ stlxrh w15, w17, [x1]
965	949	\\ cbnz w15, 0b
966	950	\\1:
967		: [ret] "={w0}" (-> u16),
968		: [val] "{w0}" (val),
969		[ptr] "{x1}" (ptr),
970		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	951	\\ ret
	952	:
	953	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
971	954	: "w15", "w16", "w17", "memory"
972	955	);
	956	unreachable;
973	957	}
974		fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	958	fn __aarch64_ldclr2_acq_rel() align(16) callconv(.Naked) void {
975	959	@setRuntimeSafety(false);
976		return asm volatile (
	960	asm volatile (
977	961	\\ cbz w16, 8f
978	962	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
979		\\ cbz wzr, 1f
	963	\\ ret
980	964	\\8:
981	965	\\ mov w16, w0
982	966	\\0:
...	...	@@ -985,19 +969,19 @@ fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
985	969	\\ stlxrh w15, w17, [x1]
986	970	\\ cbnz w15, 0b
987	971	\\1:
988		: [ret] "={w0}" (-> u16),
989		: [val] "{w0}" (val),
990		[ptr] "{x1}" (ptr),
991		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	972	\\ ret
	973	:
	974	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
992	975	: "w15", "w16", "w17", "memory"
993	976	);
	977	unreachable;
994	978	}
995		fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	979	fn __aarch64_ldeor2_acq_rel() align(16) callconv(.Naked) void {
996	980	@setRuntimeSafety(false);
997		return asm volatile (
	981	asm volatile (
998	982	\\ cbz w16, 8f
999	983	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
1000		\\ cbz wzr, 1f
	984	\\ ret
1001	985	\\8:
1002	986	\\ mov w16, w0
1003	987	\\0:
...	...	@@ -1006,19 +990,19 @@ fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1006	990	\\ stlxrh w15, w17, [x1]
1007	991	\\ cbnz w15, 0b
1008	992	\\1:
1009		: [ret] "={w0}" (-> u16),
1010		: [val] "{w0}" (val),
1011		[ptr] "{x1}" (ptr),
1012		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	993	\\ ret
	994	:
	995	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1013	996	: "w15", "w16", "w17", "memory"
1014	997	);
	998	unreachable;
1015	999	}
1016		fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
	1000	fn __aarch64_ldset2_acq_rel() align(16) callconv(.Naked) void {
1017	1001	@setRuntimeSafety(false);
1018		return asm volatile (
	1002	asm volatile (
1019	1003	\\ cbz w16, 8f
1020	1004	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
1021		\\ cbz wzr, 1f
	1005	\\ ret
1022	1006	\\8:
1023	1007	\\ mov w16, w0
1024	1008	\\0:
...	...	@@ -1027,20 +1011,19 @@ fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1027	1011	\\ stlxrh w15, w17, [x1]
1028	1012	\\ cbnz w15, 0b
1029	1013	\\1:
1030		: [ret] "={w0}" (-> u16),
1031		: [val] "{w0}" (val),
1032		[ptr] "{x1}" (ptr),
1033		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1014	\\ ret
	1015	:
	1016	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1034	1017	: "w15", "w16", "w17", "memory"
1035	1018	);
	1019	unreachable;
1036	1020	}
1037		fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
	1021	fn __aarch64_cas4_relax() align(16) callconv(.Naked) void {
1038	1022	@setRuntimeSafety(false);
1039		return asm volatile (
	1023	asm volatile (
1040	1024	\\ cbz w16, 8f
1041	1025	\\ .inst 0x08a07c41 + 0x80000000 + 0x000000
1042		\\
1043		\\ cbz wzr, 1f
	1026	\\ ret
1044	1027	\\8:
1045	1028	\\ mov w16, w0
1046	1029	\\0:
...	...	@@ -1050,20 +1033,19 @@ fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32
1050	1033	\\ stxr w17, w1, [x2]
1051	1034	\\ cbnz w17, 0b
1052	1035	\\1:
1053		: [ret] "={w0}" (-> u32),
1054		: [expected] "{w0}" (expected),
1055		[desired] "{w1}" (desired),
1056		[ptr] "{x2}" (ptr),
1057		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1036	\\ ret
	1037	:
	1038	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1058	1039	: "w15", "w16", "w17", "memory"
1059	1040	);
	1041	unreachable;
1060	1042	}
1061		fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
	1043	fn __aarch64_swp4_relax() align(16) callconv(.Naked) void {
1062	1044	@setRuntimeSafety(false);
1063		return asm volatile (
	1045	asm volatile (
1064	1046	\\ cbz w16, 8f
1065	1047	\\ .inst 0x38208020 + 0x80000000 + 0x000000
1066		\\ cbz wzr, 1f
	1048	\\ ret
1067	1049	\\8:
1068	1050	\\ mov w16, w0
1069	1051	\\0:
...	...	@@ -1071,19 +1053,19 @@ fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1071	1053	\\ stxr w17, w16, [x1]
1072	1054	\\ cbnz w17, 0b
1073	1055	\\1:
1074		: [ret] "={w0}" (-> u32),
1075		: [val] "{w0}" (val),
1076		[ptr] "{x1}" (ptr),
1077		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1056	\\ ret
	1057	:
	1058	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1078	1059	: "w15", "w16", "w17", "memory"
1079	1060	);
	1061	unreachable;
1080	1062	}
1081		fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
	1063	fn __aarch64_ldadd4_relax() align(16) callconv(.Naked) void {
1082	1064	@setRuntimeSafety(false);
1083		return asm volatile (
	1065	asm volatile (
1084	1066	\\ cbz w16, 8f
1085	1067	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
1086		\\ cbz wzr, 1f
	1068	\\ ret
1087	1069	\\8:
1088	1070	\\ mov w16, w0
1089	1071	\\0:
...	...	@@ -1092,19 +1074,19 @@ fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1092	1074	\\ stxr w15, w17, [x1]
1093	1075	\\ cbnz w15, 0b
1094	1076	\\1:
1095		: [ret] "={w0}" (-> u32),
1096		: [val] "{w0}" (val),
1097		[ptr] "{x1}" (ptr),
1098		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1077	\\ ret
	1078	:
	1079	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1099	1080	: "w15", "w16", "w17", "memory"
1100	1081	);
	1082	unreachable;
1101	1083	}
1102		fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
	1084	fn __aarch64_ldclr4_relax() align(16) callconv(.Naked) void {
1103	1085	@setRuntimeSafety(false);
1104		return asm volatile (
	1086	asm volatile (
1105	1087	\\ cbz w16, 8f
1106	1088	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
1107		\\ cbz wzr, 1f
	1089	\\ ret
1108	1090	\\8:
1109	1091	\\ mov w16, w0
1110	1092	\\0:
...	...	@@ -1113,19 +1095,19 @@ fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1113	1095	\\ stxr w15, w17, [x1]
1114	1096	\\ cbnz w15, 0b
1115	1097	\\1:
1116		: [ret] "={w0}" (-> u32),
1117		: [val] "{w0}" (val),
1118		[ptr] "{x1}" (ptr),
1119		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1098	\\ ret
	1099	:
	1100	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1120	1101	: "w15", "w16", "w17", "memory"
1121	1102	);
	1103	unreachable;
1122	1104	}
1123		fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
	1105	fn __aarch64_ldeor4_relax() align(16) callconv(.Naked) void {
1124	1106	@setRuntimeSafety(false);
1125		return asm volatile (
	1107	asm volatile (
1126	1108	\\ cbz w16, 8f
1127	1109	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
1128		\\ cbz wzr, 1f
	1110	\\ ret
1129	1111	\\8:
1130	1112	\\ mov w16, w0
1131	1113	\\0:
...	...	@@ -1134,19 +1116,19 @@ fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1134	1116	\\ stxr w15, w17, [x1]
1135	1117	\\ cbnz w15, 0b
1136	1118	\\1:
1137		: [ret] "={w0}" (-> u32),
1138		: [val] "{w0}" (val),
1139		[ptr] "{x1}" (ptr),
1140		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1119	\\ ret
	1120	:
	1121	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1141	1122	: "w15", "w16", "w17", "memory"
1142	1123	);
	1124	unreachable;
1143	1125	}
1144		fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
	1126	fn __aarch64_ldset4_relax() align(16) callconv(.Naked) void {
1145	1127	@setRuntimeSafety(false);
1146		return asm volatile (
	1128	asm volatile (
1147	1129	\\ cbz w16, 8f
1148	1130	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
1149		\\ cbz wzr, 1f
	1131	\\ ret
1150	1132	\\8:
1151	1133	\\ mov w16, w0
1152	1134	\\0:
...	...	@@ -1155,20 +1137,19 @@ fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1155	1137	\\ stxr w15, w17, [x1]
1156	1138	\\ cbnz w15, 0b
1157	1139	\\1:
1158		: [ret] "={w0}" (-> u32),
1159		: [val] "{w0}" (val),
1160		[ptr] "{x1}" (ptr),
1161		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1140	\\ ret
	1141	:
	1142	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1162	1143	: "w15", "w16", "w17", "memory"
1163	1144	);
	1145	unreachable;
1164	1146	}
1165		fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
	1147	fn __aarch64_cas4_acq() align(16) callconv(.Naked) void {
1166	1148	@setRuntimeSafety(false);
1167		return asm volatile (
	1149	asm volatile (
1168	1150	\\ cbz w16, 8f
1169	1151	\\ .inst 0x08a07c41 + 0x80000000 + 0x400000
1170		\\
1171		\\ cbz wzr, 1f
	1152	\\ ret
1172	1153	\\8:
1173	1154	\\ mov w16, w0
1174	1155	\\0:
...	...	@@ -1178,20 +1159,19 @@ fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1178	1159	\\ stxr w17, w1, [x2]
1179	1160	\\ cbnz w17, 0b
1180	1161	\\1:
1181		: [ret] "={w0}" (-> u32),
1182		: [expected] "{w0}" (expected),
1183		[desired] "{w1}" (desired),
1184		[ptr] "{x2}" (ptr),
1185		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1162	\\ ret
	1163	:
	1164	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1186	1165	: "w15", "w16", "w17", "memory"
1187	1166	);
	1167	unreachable;
1188	1168	}
1189		fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
	1169	fn __aarch64_swp4_acq() align(16) callconv(.Naked) void {
1190	1170	@setRuntimeSafety(false);
1191		return asm volatile (
	1171	asm volatile (
1192	1172	\\ cbz w16, 8f
1193	1173	\\ .inst 0x38208020 + 0x80000000 + 0x800000
1194		\\ cbz wzr, 1f
	1174	\\ ret
1195	1175	\\8:
1196	1176	\\ mov w16, w0
1197	1177	\\0:
...	...	@@ -1199,19 +1179,19 @@ fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1199	1179	\\ stxr w17, w16, [x1]
1200	1180	\\ cbnz w17, 0b
1201	1181	\\1:
1202		: [ret] "={w0}" (-> u32),
1203		: [val] "{w0}" (val),
1204		[ptr] "{x1}" (ptr),
1205		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1182	\\ ret
	1183	:
	1184	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1206	1185	: "w15", "w16", "w17", "memory"
1207	1186	);
	1187	unreachable;
1208	1188	}
1209		fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
	1189	fn __aarch64_ldadd4_acq() align(16) callconv(.Naked) void {
1210	1190	@setRuntimeSafety(false);
1211		return asm volatile (
	1191	asm volatile (
1212	1192	\\ cbz w16, 8f
1213	1193	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
1214		\\ cbz wzr, 1f
	1194	\\ ret
1215	1195	\\8:
1216	1196	\\ mov w16, w0
1217	1197	\\0:
...	...	@@ -1220,19 +1200,19 @@ fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1220	1200	\\ stxr w15, w17, [x1]
1221	1201	\\ cbnz w15, 0b
1222	1202	\\1:
1223		: [ret] "={w0}" (-> u32),
1224		: [val] "{w0}" (val),
1225		[ptr] "{x1}" (ptr),
1226		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1203	\\ ret
	1204	:
	1205	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1227	1206	: "w15", "w16", "w17", "memory"
1228	1207	);
	1208	unreachable;
1229	1209	}
1230		fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
	1210	fn __aarch64_ldclr4_acq() align(16) callconv(.Naked) void {
1231	1211	@setRuntimeSafety(false);
1232		return asm volatile (
	1212	asm volatile (
1233	1213	\\ cbz w16, 8f
1234	1214	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
1235		\\ cbz wzr, 1f
	1215	\\ ret
1236	1216	\\8:
1237	1217	\\ mov w16, w0
1238	1218	\\0:
...	...	@@ -1241,19 +1221,19 @@ fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1241	1221	\\ stxr w15, w17, [x1]
1242	1222	\\ cbnz w15, 0b
1243	1223	\\1:
1244		: [ret] "={w0}" (-> u32),
1245		: [val] "{w0}" (val),
1246		[ptr] "{x1}" (ptr),
1247		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1224	\\ ret
	1225	:
	1226	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1248	1227	: "w15", "w16", "w17", "memory"
1249	1228	);
	1229	unreachable;
1250	1230	}
1251		fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
	1231	fn __aarch64_ldeor4_acq() align(16) callconv(.Naked) void {
1252	1232	@setRuntimeSafety(false);
1253		return asm volatile (
	1233	asm volatile (
1254	1234	\\ cbz w16, 8f
1255	1235	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
1256		\\ cbz wzr, 1f
	1236	\\ ret
1257	1237	\\8:
1258	1238	\\ mov w16, w0
1259	1239	\\0:
...	...	@@ -1262,19 +1242,19 @@ fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1262	1242	\\ stxr w15, w17, [x1]
1263	1243	\\ cbnz w15, 0b
1264	1244	\\1:
1265		: [ret] "={w0}" (-> u32),
1266		: [val] "{w0}" (val),
1267		[ptr] "{x1}" (ptr),
1268		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1245	\\ ret
	1246	:
	1247	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1269	1248	: "w15", "w16", "w17", "memory"
1270	1249	);
	1250	unreachable;
1271	1251	}
1272		fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
	1252	fn __aarch64_ldset4_acq() align(16) callconv(.Naked) void {
1273	1253	@setRuntimeSafety(false);
1274		return asm volatile (
	1254	asm volatile (
1275	1255	\\ cbz w16, 8f
1276	1256	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
1277		\\ cbz wzr, 1f
	1257	\\ ret
1278	1258	\\8:
1279	1259	\\ mov w16, w0
1280	1260	\\0:
...	...	@@ -1283,20 +1263,19 @@ fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1283	1263	\\ stxr w15, w17, [x1]
1284	1264	\\ cbnz w15, 0b
1285	1265	\\1:
1286		: [ret] "={w0}" (-> u32),
1287		: [val] "{w0}" (val),
1288		[ptr] "{x1}" (ptr),
1289		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1266	\\ ret
	1267	:
	1268	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1290	1269	: "w15", "w16", "w17", "memory"
1291	1270	);
	1271	unreachable;
1292	1272	}
1293		fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
	1273	fn __aarch64_cas4_rel() align(16) callconv(.Naked) void {
1294	1274	@setRuntimeSafety(false);
1295		return asm volatile (
	1275	asm volatile (
1296	1276	\\ cbz w16, 8f
1297	1277	\\ .inst 0x08a07c41 + 0x80000000 + 0x008000
1298		\\
1299		\\ cbz wzr, 1f
	1278	\\ ret
1300	1279	\\8:
1301	1280	\\ mov w16, w0
1302	1281	\\0:
...	...	@@ -1306,20 +1285,19 @@ fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1306	1285	\\ stlxr w17, w1, [x2]
1307	1286	\\ cbnz w17, 0b
1308	1287	\\1:
1309		: [ret] "={w0}" (-> u32),
1310		: [expected] "{w0}" (expected),
1311		[desired] "{w1}" (desired),
1312		[ptr] "{x2}" (ptr),
1313		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1288	\\ ret
	1289	:
	1290	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1314	1291	: "w15", "w16", "w17", "memory"
1315	1292	);
	1293	unreachable;
1316	1294	}
1317		fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1295	fn __aarch64_swp4_rel() align(16) callconv(.Naked) void {
1318	1296	@setRuntimeSafety(false);
1319		return asm volatile (
	1297	asm volatile (
1320	1298	\\ cbz w16, 8f
1321	1299	\\ .inst 0x38208020 + 0x80000000 + 0x400000
1322		\\ cbz wzr, 1f
	1300	\\ ret
1323	1301	\\8:
1324	1302	\\ mov w16, w0
1325	1303	\\0:
...	...	@@ -1327,19 +1305,19 @@ fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1327	1305	\\ stlxr w17, w16, [x1]
1328	1306	\\ cbnz w17, 0b
1329	1307	\\1:
1330		: [ret] "={w0}" (-> u32),
1331		: [val] "{w0}" (val),
1332		[ptr] "{x1}" (ptr),
1333		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1308	\\ ret
	1309	:
	1310	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1334	1311	: "w15", "w16", "w17", "memory"
1335	1312	);
	1313	unreachable;
1336	1314	}
1337		fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1315	fn __aarch64_ldadd4_rel() align(16) callconv(.Naked) void {
1338	1316	@setRuntimeSafety(false);
1339		return asm volatile (
	1317	asm volatile (
1340	1318	\\ cbz w16, 8f
1341	1319	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
1342		\\ cbz wzr, 1f
	1320	\\ ret
1343	1321	\\8:
1344	1322	\\ mov w16, w0
1345	1323	\\0:
...	...	@@ -1348,19 +1326,19 @@ fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1348	1326	\\ stlxr w15, w17, [x1]
1349	1327	\\ cbnz w15, 0b
1350	1328	\\1:
1351		: [ret] "={w0}" (-> u32),
1352		: [val] "{w0}" (val),
1353		[ptr] "{x1}" (ptr),
1354		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1329	\\ ret
	1330	:
	1331	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1355	1332	: "w15", "w16", "w17", "memory"
1356	1333	);
	1334	unreachable;
1357	1335	}
1358		fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1336	fn __aarch64_ldclr4_rel() align(16) callconv(.Naked) void {
1359	1337	@setRuntimeSafety(false);
1360		return asm volatile (
	1338	asm volatile (
1361	1339	\\ cbz w16, 8f
1362	1340	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
1363		\\ cbz wzr, 1f
	1341	\\ ret
1364	1342	\\8:
1365	1343	\\ mov w16, w0
1366	1344	\\0:
...	...	@@ -1369,19 +1347,19 @@ fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1369	1347	\\ stlxr w15, w17, [x1]
1370	1348	\\ cbnz w15, 0b
1371	1349	\\1:
1372		: [ret] "={w0}" (-> u32),
1373		: [val] "{w0}" (val),
1374		[ptr] "{x1}" (ptr),
1375		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1350	\\ ret
	1351	:
	1352	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1376	1353	: "w15", "w16", "w17", "memory"
1377	1354	);
	1355	unreachable;
1378	1356	}
1379		fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1357	fn __aarch64_ldeor4_rel() align(16) callconv(.Naked) void {
1380	1358	@setRuntimeSafety(false);
1381		return asm volatile (
	1359	asm volatile (
1382	1360	\\ cbz w16, 8f
1383	1361	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
1384		\\ cbz wzr, 1f
	1362	\\ ret
1385	1363	\\8:
1386	1364	\\ mov w16, w0
1387	1365	\\0:
...	...	@@ -1390,19 +1368,19 @@ fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1390	1368	\\ stlxr w15, w17, [x1]
1391	1369	\\ cbnz w15, 0b
1392	1370	\\1:
1393		: [ret] "={w0}" (-> u32),
1394		: [val] "{w0}" (val),
1395		[ptr] "{x1}" (ptr),
1396		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1371	\\ ret
	1372	:
	1373	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1397	1374	: "w15", "w16", "w17", "memory"
1398	1375	);
	1376	unreachable;
1399	1377	}
1400		fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1378	fn __aarch64_ldset4_rel() align(16) callconv(.Naked) void {
1401	1379	@setRuntimeSafety(false);
1402		return asm volatile (
	1380	asm volatile (
1403	1381	\\ cbz w16, 8f
1404	1382	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
1405		\\ cbz wzr, 1f
	1383	\\ ret
1406	1384	\\8:
1407	1385	\\ mov w16, w0
1408	1386	\\0:
...	...	@@ -1411,20 +1389,19 @@ fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1411	1389	\\ stlxr w15, w17, [x1]
1412	1390	\\ cbnz w15, 0b
1413	1391	\\1:
1414		: [ret] "={w0}" (-> u32),
1415		: [val] "{w0}" (val),
1416		[ptr] "{x1}" (ptr),
1417		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1392	\\ ret
	1393	:
	1394	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1418	1395	: "w15", "w16", "w17", "memory"
1419	1396	);
	1397	unreachable;
1420	1398	}
1421		fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
	1399	fn __aarch64_cas4_acq_rel() align(16) callconv(.Naked) void {
1422	1400	@setRuntimeSafety(false);
1423		return asm volatile (
	1401	asm volatile (
1424	1402	\\ cbz w16, 8f
1425	1403	\\ .inst 0x08a07c41 + 0x80000000 + 0x408000
1426		\\
1427		\\ cbz wzr, 1f
	1404	\\ ret
1428	1405	\\8:
1429	1406	\\ mov w16, w0
1430	1407	\\0:
...	...	@@ -1434,20 +1411,19 @@ fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u
1434	1411	\\ stlxr w17, w1, [x2]
1435	1412	\\ cbnz w17, 0b
1436	1413	\\1:
1437		: [ret] "={w0}" (-> u32),
1438		: [expected] "{w0}" (expected),
1439		[desired] "{w1}" (desired),
1440		[ptr] "{x2}" (ptr),
1441		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1414	\\ ret
	1415	:
	1416	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1442	1417	: "w15", "w16", "w17", "memory"
1443	1418	);
	1419	unreachable;
1444	1420	}
1445		fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1421	fn __aarch64_swp4_acq_rel() align(16) callconv(.Naked) void {
1446	1422	@setRuntimeSafety(false);
1447		return asm volatile (
	1423	asm volatile (
1448	1424	\\ cbz w16, 8f
1449	1425	\\ .inst 0x38208020 + 0x80000000 + 0xc00000
1450		\\ cbz wzr, 1f
	1426	\\ ret
1451	1427	\\8:
1452	1428	\\ mov w16, w0
1453	1429	\\0:
...	...	@@ -1455,19 +1431,19 @@ fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1455	1431	\\ stlxr w17, w16, [x1]
1456	1432	\\ cbnz w17, 0b
1457	1433	\\1:
1458		: [ret] "={w0}" (-> u32),
1459		: [val] "{w0}" (val),
1460		[ptr] "{x1}" (ptr),
1461		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1434	\\ ret
	1435	:
	1436	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1462	1437	: "w15", "w16", "w17", "memory"
1463	1438	);
	1439	unreachable;
1464	1440	}
1465		fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1441	fn __aarch64_ldadd4_acq_rel() align(16) callconv(.Naked) void {
1466	1442	@setRuntimeSafety(false);
1467		return asm volatile (
	1443	asm volatile (
1468	1444	\\ cbz w16, 8f
1469	1445	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
1470		\\ cbz wzr, 1f
	1446	\\ ret
1471	1447	\\8:
1472	1448	\\ mov w16, w0
1473	1449	\\0:
...	...	@@ -1476,19 +1452,19 @@ fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1476	1452	\\ stlxr w15, w17, [x1]
1477	1453	\\ cbnz w15, 0b
1478	1454	\\1:
1479		: [ret] "={w0}" (-> u32),
1480		: [val] "{w0}" (val),
1481		[ptr] "{x1}" (ptr),
1482		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1455	\\ ret
	1456	:
	1457	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1483	1458	: "w15", "w16", "w17", "memory"
1484	1459	);
	1460	unreachable;
1485	1461	}
1486		fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1462	fn __aarch64_ldclr4_acq_rel() align(16) callconv(.Naked) void {
1487	1463	@setRuntimeSafety(false);
1488		return asm volatile (
	1464	asm volatile (
1489	1465	\\ cbz w16, 8f
1490	1466	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
1491		\\ cbz wzr, 1f
	1467	\\ ret
1492	1468	\\8:
1493	1469	\\ mov w16, w0
1494	1470	\\0:
...	...	@@ -1497,19 +1473,19 @@ fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1497	1473	\\ stlxr w15, w17, [x1]
1498	1474	\\ cbnz w15, 0b
1499	1475	\\1:
1500		: [ret] "={w0}" (-> u32),
1501		: [val] "{w0}" (val),
1502		[ptr] "{x1}" (ptr),
1503		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1476	\\ ret
	1477	:
	1478	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1504	1479	: "w15", "w16", "w17", "memory"
1505	1480	);
	1481	unreachable;
1506	1482	}
1507		fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1483	fn __aarch64_ldeor4_acq_rel() align(16) callconv(.Naked) void {
1508	1484	@setRuntimeSafety(false);
1509		return asm volatile (
	1485	asm volatile (
1510	1486	\\ cbz w16, 8f
1511	1487	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
1512		\\ cbz wzr, 1f
	1488	\\ ret
1513	1489	\\8:
1514	1490	\\ mov w16, w0
1515	1491	\\0:
...	...	@@ -1518,19 +1494,19 @@ fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1518	1494	\\ stlxr w15, w17, [x1]
1519	1495	\\ cbnz w15, 0b
1520	1496	\\1:
1521		: [ret] "={w0}" (-> u32),
1522		: [val] "{w0}" (val),
1523		[ptr] "{x1}" (ptr),
1524		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1497	\\ ret
	1498	:
	1499	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1525	1500	: "w15", "w16", "w17", "memory"
1526	1501	);
	1502	unreachable;
1527	1503	}
1528		fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
	1504	fn __aarch64_ldset4_acq_rel() align(16) callconv(.Naked) void {
1529	1505	@setRuntimeSafety(false);
1530		return asm volatile (
	1506	asm volatile (
1531	1507	\\ cbz w16, 8f
1532	1508	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
1533		\\ cbz wzr, 1f
	1509	\\ ret
1534	1510	\\8:
1535	1511	\\ mov w16, w0
1536	1512	\\0:
...	...	@@ -1539,20 +1515,19 @@ fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1539	1515	\\ stlxr w15, w17, [x1]
1540	1516	\\ cbnz w15, 0b
1541	1517	\\1:
1542		: [ret] "={w0}" (-> u32),
1543		: [val] "{w0}" (val),
1544		[ptr] "{x1}" (ptr),
1545		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1518	\\ ret
	1519	:
	1520	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1546	1521	: "w15", "w16", "w17", "memory"
1547	1522	);
	1523	unreachable;
1548	1524	}
1549		fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
	1525	fn __aarch64_cas8_relax() align(16) callconv(.Naked) void {
1550	1526	@setRuntimeSafety(false);
1551		return asm volatile (
	1527	asm volatile (
1552	1528	\\ cbz w16, 8f
1553	1529	\\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
1554		\\
1555		\\ cbz wzr, 1f
	1530	\\ ret
1556	1531	\\8:
1557	1532	\\ mov x16, x0
1558	1533	\\0:
...	...	@@ -1562,20 +1537,19 @@ fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64
1562	1537	\\ stxr w17, x1, [x2]
1563	1538	\\ cbnz w17, 0b
1564	1539	\\1:
1565		: [ret] "={x0}" (-> u64),
1566		: [expected] "{x0}" (expected),
1567		[desired] "{x1}" (desired),
1568		[ptr] "{x2}" (ptr),
1569		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1540	\\ ret
	1541	:
	1542	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1570	1543	: "w15", "w16", "w17", "memory"
1571	1544	);
	1545	unreachable;
1572	1546	}
1573		fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
	1547	fn __aarch64_swp8_relax() align(16) callconv(.Naked) void {
1574	1548	@setRuntimeSafety(false);
1575		return asm volatile (
	1549	asm volatile (
1576	1550	\\ cbz w16, 8f
1577	1551	\\ .inst 0x38208020 + 0xc0000000 + 0x000000
1578		\\ cbz wzr, 1f
	1552	\\ ret
1579	1553	\\8:
1580	1554	\\ mov x16, x0
1581	1555	\\0:
...	...	@@ -1583,19 +1557,19 @@ fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1583	1557	\\ stxr w17, x16, [x1]
1584	1558	\\ cbnz w17, 0b
1585	1559	\\1:
1586		: [ret] "={x0}" (-> u64),
1587		: [val] "{x0}" (val),
1588		[ptr] "{x1}" (ptr),
1589		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1560	\\ ret
	1561	:
	1562	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1590	1563	: "w15", "w16", "w17", "memory"
1591	1564	);
	1565	unreachable;
1592	1566	}
1593		fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
	1567	fn __aarch64_ldadd8_relax() align(16) callconv(.Naked) void {
1594	1568	@setRuntimeSafety(false);
1595		return asm volatile (
	1569	asm volatile (
1596	1570	\\ cbz w16, 8f
1597	1571	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
1598		\\ cbz wzr, 1f
	1572	\\ ret
1599	1573	\\8:
1600	1574	\\ mov x16, x0
1601	1575	\\0:
...	...	@@ -1604,19 +1578,19 @@ fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1604	1578	\\ stxr w15, x17, [x1]
1605	1579	\\ cbnz w15, 0b
1606	1580	\\1:
1607		: [ret] "={x0}" (-> u64),
1608		: [val] "{x0}" (val),
1609		[ptr] "{x1}" (ptr),
1610		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1581	\\ ret
	1582	:
	1583	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1611	1584	: "w15", "w16", "w17", "memory"
1612	1585	);
	1586	unreachable;
1613	1587	}
1614		fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
	1588	fn __aarch64_ldclr8_relax() align(16) callconv(.Naked) void {
1615	1589	@setRuntimeSafety(false);
1616		return asm volatile (
	1590	asm volatile (
1617	1591	\\ cbz w16, 8f
1618	1592	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
1619		\\ cbz wzr, 1f
	1593	\\ ret
1620	1594	\\8:
1621	1595	\\ mov x16, x0
1622	1596	\\0:
...	...	@@ -1625,19 +1599,19 @@ fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1625	1599	\\ stxr w15, x17, [x1]
1626	1600	\\ cbnz w15, 0b
1627	1601	\\1:
1628		: [ret] "={x0}" (-> u64),
1629		: [val] "{x0}" (val),
1630		[ptr] "{x1}" (ptr),
1631		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1602	\\ ret
	1603	:
	1604	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1632	1605	: "w15", "w16", "w17", "memory"
1633	1606	);
	1607	unreachable;
1634	1608	}
1635		fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
	1609	fn __aarch64_ldeor8_relax() align(16) callconv(.Naked) void {
1636	1610	@setRuntimeSafety(false);
1637		return asm volatile (
	1611	asm volatile (
1638	1612	\\ cbz w16, 8f
1639	1613	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
1640		\\ cbz wzr, 1f
	1614	\\ ret
1641	1615	\\8:
1642	1616	\\ mov x16, x0
1643	1617	\\0:
...	...	@@ -1646,19 +1620,19 @@ fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1646	1620	\\ stxr w15, x17, [x1]
1647	1621	\\ cbnz w15, 0b
1648	1622	\\1:
1649		: [ret] "={x0}" (-> u64),
1650		: [val] "{x0}" (val),
1651		[ptr] "{x1}" (ptr),
1652		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1623	\\ ret
	1624	:
	1625	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1653	1626	: "w15", "w16", "w17", "memory"
1654	1627	);
	1628	unreachable;
1655	1629	}
1656		fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
	1630	fn __aarch64_ldset8_relax() align(16) callconv(.Naked) void {
1657	1631	@setRuntimeSafety(false);
1658		return asm volatile (
	1632	asm volatile (
1659	1633	\\ cbz w16, 8f
1660	1634	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
1661		\\ cbz wzr, 1f
	1635	\\ ret
1662	1636	\\8:
1663	1637	\\ mov x16, x0
1664	1638	\\0:
...	...	@@ -1667,20 +1641,19 @@ fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
1667	1641	\\ stxr w15, x17, [x1]
1668	1642	\\ cbnz w15, 0b
1669	1643	\\1:
1670		: [ret] "={x0}" (-> u64),
1671		: [val] "{x0}" (val),
1672		[ptr] "{x1}" (ptr),
1673		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1644	\\ ret
	1645	:
	1646	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1674	1647	: "w15", "w16", "w17", "memory"
1675	1648	);
	1649	unreachable;
1676	1650	}
1677		fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
	1651	fn __aarch64_cas8_acq() align(16) callconv(.Naked) void {
1678	1652	@setRuntimeSafety(false);
1679		return asm volatile (
	1653	asm volatile (
1680	1654	\\ cbz w16, 8f
1681	1655	\\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
1682		\\
1683		\\ cbz wzr, 1f
	1656	\\ ret
1684	1657	\\8:
1685	1658	\\ mov x16, x0
1686	1659	\\0:
...	...	@@ -1690,20 +1663,19 @@ fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
1690	1663	\\ stxr w17, x1, [x2]
1691	1664	\\ cbnz w17, 0b
1692	1665	\\1:
1693		: [ret] "={x0}" (-> u64),
1694		: [expected] "{x0}" (expected),
1695		[desired] "{x1}" (desired),
1696		[ptr] "{x2}" (ptr),
1697		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1666	\\ ret
	1667	:
	1668	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1698	1669	: "w15", "w16", "w17", "memory"
1699	1670	);
	1671	unreachable;
1700	1672	}
1701		fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
	1673	fn __aarch64_swp8_acq() align(16) callconv(.Naked) void {
1702	1674	@setRuntimeSafety(false);
1703		return asm volatile (
	1675	asm volatile (
1704	1676	\\ cbz w16, 8f
1705	1677	\\ .inst 0x38208020 + 0xc0000000 + 0x800000
1706		\\ cbz wzr, 1f
	1678	\\ ret
1707	1679	\\8:
1708	1680	\\ mov x16, x0
1709	1681	\\0:
...	...	@@ -1711,19 +1683,19 @@ fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1711	1683	\\ stxr w17, x16, [x1]
1712	1684	\\ cbnz w17, 0b
1713	1685	\\1:
1714		: [ret] "={x0}" (-> u64),
1715		: [val] "{x0}" (val),
1716		[ptr] "{x1}" (ptr),
1717		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1686	\\ ret
	1687	:
	1688	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1718	1689	: "w15", "w16", "w17", "memory"
1719	1690	);
	1691	unreachable;
1720	1692	}
1721		fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
	1693	fn __aarch64_ldadd8_acq() align(16) callconv(.Naked) void {
1722	1694	@setRuntimeSafety(false);
1723		return asm volatile (
	1695	asm volatile (
1724	1696	\\ cbz w16, 8f
1725	1697	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
1726		\\ cbz wzr, 1f
	1698	\\ ret
1727	1699	\\8:
1728	1700	\\ mov x16, x0
1729	1701	\\0:
...	...	@@ -1732,19 +1704,19 @@ fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1732	1704	\\ stxr w15, x17, [x1]
1733	1705	\\ cbnz w15, 0b
1734	1706	\\1:
1735		: [ret] "={x0}" (-> u64),
1736		: [val] "{x0}" (val),
1737		[ptr] "{x1}" (ptr),
1738		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1707	\\ ret
	1708	:
	1709	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1739	1710	: "w15", "w16", "w17", "memory"
1740	1711	);
	1712	unreachable;
1741	1713	}
1742		fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
	1714	fn __aarch64_ldclr8_acq() align(16) callconv(.Naked) void {
1743	1715	@setRuntimeSafety(false);
1744		return asm volatile (
	1716	asm volatile (
1745	1717	\\ cbz w16, 8f
1746	1718	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
1747		\\ cbz wzr, 1f
	1719	\\ ret
1748	1720	\\8:
1749	1721	\\ mov x16, x0
1750	1722	\\0:
...	...	@@ -1753,19 +1725,19 @@ fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1753	1725	\\ stxr w15, x17, [x1]
1754	1726	\\ cbnz w15, 0b
1755	1727	\\1:
1756		: [ret] "={x0}" (-> u64),
1757		: [val] "{x0}" (val),
1758		[ptr] "{x1}" (ptr),
1759		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1728	\\ ret
	1729	:
	1730	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1760	1731	: "w15", "w16", "w17", "memory"
1761	1732	);
	1733	unreachable;
1762	1734	}
1763		fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
	1735	fn __aarch64_ldeor8_acq() align(16) callconv(.Naked) void {
1764	1736	@setRuntimeSafety(false);
1765		return asm volatile (
	1737	asm volatile (
1766	1738	\\ cbz w16, 8f
1767	1739	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
1768		\\ cbz wzr, 1f
	1740	\\ ret
1769	1741	\\8:
1770	1742	\\ mov x16, x0
1771	1743	\\0:
...	...	@@ -1774,19 +1746,19 @@ fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1774	1746	\\ stxr w15, x17, [x1]
1775	1747	\\ cbnz w15, 0b
1776	1748	\\1:
1777		: [ret] "={x0}" (-> u64),
1778		: [val] "{x0}" (val),
1779		[ptr] "{x1}" (ptr),
1780		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1749	\\ ret
	1750	:
	1751	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1781	1752	: "w15", "w16", "w17", "memory"
1782	1753	);
	1754	unreachable;
1783	1755	}
1784		fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
	1756	fn __aarch64_ldset8_acq() align(16) callconv(.Naked) void {
1785	1757	@setRuntimeSafety(false);
1786		return asm volatile (
	1758	asm volatile (
1787	1759	\\ cbz w16, 8f
1788	1760	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
1789		\\ cbz wzr, 1f
	1761	\\ ret
1790	1762	\\8:
1791	1763	\\ mov x16, x0
1792	1764	\\0:
...	...	@@ -1795,20 +1767,19 @@ fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
1795	1767	\\ stxr w15, x17, [x1]
1796	1768	\\ cbnz w15, 0b
1797	1769	\\1:
1798		: [ret] "={x0}" (-> u64),
1799		: [val] "{x0}" (val),
1800		[ptr] "{x1}" (ptr),
1801		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1770	\\ ret
	1771	:
	1772	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1802	1773	: "w15", "w16", "w17", "memory"
1803	1774	);
	1775	unreachable;
1804	1776	}
1805		fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
	1777	fn __aarch64_cas8_rel() align(16) callconv(.Naked) void {
1806	1778	@setRuntimeSafety(false);
1807		return asm volatile (
	1779	asm volatile (
1808	1780	\\ cbz w16, 8f
1809	1781	\\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
1810		\\
1811		\\ cbz wzr, 1f
	1782	\\ ret
1812	1783	\\8:
1813	1784	\\ mov x16, x0
1814	1785	\\0:
...	...	@@ -1818,20 +1789,19 @@ fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
1818	1789	\\ stlxr w17, x1, [x2]
1819	1790	\\ cbnz w17, 0b
1820	1791	\\1:
1821		: [ret] "={x0}" (-> u64),
1822		: [expected] "{x0}" (expected),
1823		[desired] "{x1}" (desired),
1824		[ptr] "{x2}" (ptr),
1825		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1792	\\ ret
	1793	:
	1794	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1826	1795	: "w15", "w16", "w17", "memory"
1827	1796	);
	1797	unreachable;
1828	1798	}
1829		fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1799	fn __aarch64_swp8_rel() align(16) callconv(.Naked) void {
1830	1800	@setRuntimeSafety(false);
1831		return asm volatile (
	1801	asm volatile (
1832	1802	\\ cbz w16, 8f
1833	1803	\\ .inst 0x38208020 + 0xc0000000 + 0x400000
1834		\\ cbz wzr, 1f
	1804	\\ ret
1835	1805	\\8:
1836	1806	\\ mov x16, x0
1837	1807	\\0:
...	...	@@ -1839,19 +1809,19 @@ fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1839	1809	\\ stlxr w17, x16, [x1]
1840	1810	\\ cbnz w17, 0b
1841	1811	\\1:
1842		: [ret] "={x0}" (-> u64),
1843		: [val] "{x0}" (val),
1844		[ptr] "{x1}" (ptr),
1845		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1812	\\ ret
	1813	:
	1814	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1846	1815	: "w15", "w16", "w17", "memory"
1847	1816	);
	1817	unreachable;
1848	1818	}
1849		fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1819	fn __aarch64_ldadd8_rel() align(16) callconv(.Naked) void {
1850	1820	@setRuntimeSafety(false);
1851		return asm volatile (
	1821	asm volatile (
1852	1822	\\ cbz w16, 8f
1853	1823	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
1854		\\ cbz wzr, 1f
	1824	\\ ret
1855	1825	\\8:
1856	1826	\\ mov x16, x0
1857	1827	\\0:
...	...	@@ -1860,19 +1830,19 @@ fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1860	1830	\\ stlxr w15, x17, [x1]
1861	1831	\\ cbnz w15, 0b
1862	1832	\\1:
1863		: [ret] "={x0}" (-> u64),
1864		: [val] "{x0}" (val),
1865		[ptr] "{x1}" (ptr),
1866		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1833	\\ ret
	1834	:
	1835	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1867	1836	: "w15", "w16", "w17", "memory"
1868	1837	);
	1838	unreachable;
1869	1839	}
1870		fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1840	fn __aarch64_ldclr8_rel() align(16) callconv(.Naked) void {
1871	1841	@setRuntimeSafety(false);
1872		return asm volatile (
	1842	asm volatile (
1873	1843	\\ cbz w16, 8f
1874	1844	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
1875		\\ cbz wzr, 1f
	1845	\\ ret
1876	1846	\\8:
1877	1847	\\ mov x16, x0
1878	1848	\\0:
...	...	@@ -1881,19 +1851,19 @@ fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1881	1851	\\ stlxr w15, x17, [x1]
1882	1852	\\ cbnz w15, 0b
1883	1853	\\1:
1884		: [ret] "={x0}" (-> u64),
1885		: [val] "{x0}" (val),
1886		[ptr] "{x1}" (ptr),
1887		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1854	\\ ret
	1855	:
	1856	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1888	1857	: "w15", "w16", "w17", "memory"
1889	1858	);
	1859	unreachable;
1890	1860	}
1891		fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1861	fn __aarch64_ldeor8_rel() align(16) callconv(.Naked) void {
1892	1862	@setRuntimeSafety(false);
1893		return asm volatile (
	1863	asm volatile (
1894	1864	\\ cbz w16, 8f
1895	1865	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
1896		\\ cbz wzr, 1f
	1866	\\ ret
1897	1867	\\8:
1898	1868	\\ mov x16, x0
1899	1869	\\0:
...	...	@@ -1902,19 +1872,19 @@ fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1902	1872	\\ stlxr w15, x17, [x1]
1903	1873	\\ cbnz w15, 0b
1904	1874	\\1:
1905		: [ret] "={x0}" (-> u64),
1906		: [val] "{x0}" (val),
1907		[ptr] "{x1}" (ptr),
1908		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1875	\\ ret
	1876	:
	1877	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1909	1878	: "w15", "w16", "w17", "memory"
1910	1879	);
	1880	unreachable;
1911	1881	}
1912		fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1882	fn __aarch64_ldset8_rel() align(16) callconv(.Naked) void {
1913	1883	@setRuntimeSafety(false);
1914		return asm volatile (
	1884	asm volatile (
1915	1885	\\ cbz w16, 8f
1916	1886	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
1917		\\ cbz wzr, 1f
	1887	\\ ret
1918	1888	\\8:
1919	1889	\\ mov x16, x0
1920	1890	\\0:
...	...	@@ -1923,20 +1893,19 @@ fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1923	1893	\\ stlxr w15, x17, [x1]
1924	1894	\\ cbnz w15, 0b
1925	1895	\\1:
1926		: [ret] "={x0}" (-> u64),
1927		: [val] "{x0}" (val),
1928		[ptr] "{x1}" (ptr),
1929		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1896	\\ ret
	1897	:
	1898	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1930	1899	: "w15", "w16", "w17", "memory"
1931	1900	);
	1901	unreachable;
1932	1902	}
1933		fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
	1903	fn __aarch64_cas8_acq_rel() align(16) callconv(.Naked) void {
1934	1904	@setRuntimeSafety(false);
1935		return asm volatile (
	1905	asm volatile (
1936	1906	\\ cbz w16, 8f
1937	1907	\\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
1938		\\
1939		\\ cbz wzr, 1f
	1908	\\ ret
1940	1909	\\8:
1941	1910	\\ mov x16, x0
1942	1911	\\0:
...	...	@@ -1946,20 +1915,19 @@ fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u
1946	1915	\\ stlxr w17, x1, [x2]
1947	1916	\\ cbnz w17, 0b
1948	1917	\\1:
1949		: [ret] "={x0}" (-> u64),
1950		: [expected] "{x0}" (expected),
1951		[desired] "{x1}" (desired),
1952		[ptr] "{x2}" (ptr),
1953		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1918	\\ ret
	1919	:
	1920	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1954	1921	: "w15", "w16", "w17", "memory"
1955	1922	);
	1923	unreachable;
1956	1924	}
1957		fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1925	fn __aarch64_swp8_acq_rel() align(16) callconv(.Naked) void {
1958	1926	@setRuntimeSafety(false);
1959		return asm volatile (
	1927	asm volatile (
1960	1928	\\ cbz w16, 8f
1961	1929	\\ .inst 0x38208020 + 0xc0000000 + 0xc00000
1962		\\ cbz wzr, 1f
	1930	\\ ret
1963	1931	\\8:
1964	1932	\\ mov x16, x0
1965	1933	\\0:
...	...	@@ -1967,19 +1935,19 @@ fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1967	1935	\\ stlxr w17, x16, [x1]
1968	1936	\\ cbnz w17, 0b
1969	1937	\\1:
1970		: [ret] "={x0}" (-> u64),
1971		: [val] "{x0}" (val),
1972		[ptr] "{x1}" (ptr),
1973		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1938	\\ ret
	1939	:
	1940	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1974	1941	: "w15", "w16", "w17", "memory"
1975	1942	);
	1943	unreachable;
1976	1944	}
1977		fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1945	fn __aarch64_ldadd8_acq_rel() align(16) callconv(.Naked) void {
1978	1946	@setRuntimeSafety(false);
1979		return asm volatile (
	1947	asm volatile (
1980	1948	\\ cbz w16, 8f
1981	1949	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
1982		\\ cbz wzr, 1f
	1950	\\ ret
1983	1951	\\8:
1984	1952	\\ mov x16, x0
1985	1953	\\0:
...	...	@@ -1988,19 +1956,19 @@ fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
1988	1956	\\ stlxr w15, x17, [x1]
1989	1957	\\ cbnz w15, 0b
1990	1958	\\1:
1991		: [ret] "={x0}" (-> u64),
1992		: [val] "{x0}" (val),
1993		[ptr] "{x1}" (ptr),
1994		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1959	\\ ret
	1960	:
	1961	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1995	1962	: "w15", "w16", "w17", "memory"
1996	1963	);
	1964	unreachable;
1997	1965	}
1998		fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1966	fn __aarch64_ldclr8_acq_rel() align(16) callconv(.Naked) void {
1999	1967	@setRuntimeSafety(false);
2000		return asm volatile (
	1968	asm volatile (
2001	1969	\\ cbz w16, 8f
2002	1970	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
2003		\\ cbz wzr, 1f
	1971	\\ ret
2004	1972	\\8:
2005	1973	\\ mov x16, x0
2006	1974	\\0:
...	...	@@ -2009,19 +1977,19 @@ fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
2009	1977	\\ stlxr w15, x17, [x1]
2010	1978	\\ cbnz w15, 0b
2011	1979	\\1:
2012		: [ret] "={x0}" (-> u64),
2013		: [val] "{x0}" (val),
2014		[ptr] "{x1}" (ptr),
2015		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	1980	\\ ret
	1981	:
	1982	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2016	1983	: "w15", "w16", "w17", "memory"
2017	1984	);
	1985	unreachable;
2018	1986	}
2019		fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	1987	fn __aarch64_ldeor8_acq_rel() align(16) callconv(.Naked) void {
2020	1988	@setRuntimeSafety(false);
2021		return asm volatile (
	1989	asm volatile (
2022	1990	\\ cbz w16, 8f
2023	1991	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
2024		\\ cbz wzr, 1f
	1992	\\ ret
2025	1993	\\8:
2026	1994	\\ mov x16, x0
2027	1995	\\0:
...	...	@@ -2030,19 +1998,19 @@ fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
2030	1998	\\ stlxr w15, x17, [x1]
2031	1999	\\ cbnz w15, 0b
2032	2000	\\1:
2033		: [ret] "={x0}" (-> u64),
2034		: [val] "{x0}" (val),
2035		[ptr] "{x1}" (ptr),
2036		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	2001	\\ ret
	2002	:
	2003	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2037	2004	: "w15", "w16", "w17", "memory"
2038	2005	);
	2006	unreachable;
2039	2007	}
2040		fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
	2008	fn __aarch64_ldset8_acq_rel() align(16) callconv(.Naked) void {
2041	2009	@setRuntimeSafety(false);
2042		return asm volatile (
	2010	asm volatile (
2043	2011	\\ cbz w16, 8f
2044	2012	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
2045		\\ cbz wzr, 1f
	2013	\\ ret
2046	2014	\\8:
2047	2015	\\ mov x16, x0
2048	2016	\\0:
...	...	@@ -2051,20 +2019,19 @@ fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
2051	2019	\\ stlxr w15, x17, [x1]
2052	2020	\\ cbnz w15, 0b
2053	2021	\\1:
2054		: [ret] "={x0}" (-> u64),
2055		: [val] "{x0}" (val),
2056		[ptr] "{x1}" (ptr),
2057		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	2022	\\ ret
	2023	:
	2024	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2058	2025	: "w15", "w16", "w17", "memory"
2059	2026	);
	2027	unreachable;
2060	2028	}
2061		fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2029	fn __aarch64_cas16_relax() align(16) callconv(.Naked) void {
2062	2030	@setRuntimeSafety(false);
2063		return asm volatile (
	2031	asm volatile (
2064	2032	\\ cbz w16, 8f
2065	2033	\\ .inst 0x48207c82 + 0x000000
2066		\\
2067		\\ cbz wzr, 1f
	2034	\\ ret
2068	2035	\\8:
2069	2036	\\ mov x16, x0
2070	2037	\\ mov x17, x1
...	...	@@ -2076,21 +2043,19 @@ fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C)
2076	2043	\\ stxp w15, x2, x3, [x4]
2077	2044	\\ cbnz w15, 0b
2078	2045	\\1:
2079		: [ret] "={x0}" (-> u128),
2080		: [expected] "{x0}" (expected),
2081		[desired] "{x1}" (desired),
2082		[ptr] "{x2}" (ptr),
2083		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	2046	\\ ret
	2047	:
	2048	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2084	2049	: "w15", "w16", "w17", "memory"
2085	2050	);
	2051	unreachable;
2086	2052	}
2087		fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2053	fn __aarch64_cas16_acq() align(16) callconv(.Naked) void {
2088	2054	@setRuntimeSafety(false);
2089		return asm volatile (
	2055	asm volatile (
2090	2056	\\ cbz w16, 8f
2091	2057	\\ .inst 0x48207c82 + 0x400000
2092		\\
2093		\\ cbz wzr, 1f
	2058	\\ ret
2094	2059	\\8:
2095	2060	\\ mov x16, x0
2096	2061	\\ mov x17, x1
...	...	@@ -2102,21 +2067,19 @@ fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u
2102	2067	\\ stxp w15, x2, x3, [x4]
2103	2068	\\ cbnz w15, 0b
2104	2069	\\1:
2105		: [ret] "={x0}" (-> u128),
2106		: [expected] "{x0}" (expected),
2107		[desired] "{x1}" (desired),
2108		[ptr] "{x2}" (ptr),
2109		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	2070	\\ ret
	2071	:
	2072	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2110	2073	: "w15", "w16", "w17", "memory"
2111	2074	);
	2075	unreachable;
2112	2076	}
2113		fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2077	fn __aarch64_cas16_rel() align(16) callconv(.Naked) void {
2114	2078	@setRuntimeSafety(false);
2115		return asm volatile (
	2079	asm volatile (
2116	2080	\\ cbz w16, 8f
2117	2081	\\ .inst 0x48207c82 + 0x008000
2118		\\
2119		\\ cbz wzr, 1f
	2082	\\ ret
2120	2083	\\8:
2121	2084	\\ mov x16, x0
2122	2085	\\ mov x17, x1
...	...	@@ -2128,21 +2091,19 @@ fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u
2128	2091	\\ stlxp w15, x2, x3, [x4]
2129	2092	\\ cbnz w15, 0b
2130	2093	\\1:
2131		: [ret] "={x0}" (-> u128),
2132		: [expected] "{x0}" (expected),
2133		[desired] "{x1}" (desired),
2134		[ptr] "{x2}" (ptr),
2135		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	2094	\\ ret
	2095	:
	2096	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2136	2097	: "w15", "w16", "w17", "memory"
2137	2098	);
	2099	unreachable;
2138	2100	}
2139		fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2101	fn __aarch64_cas16_acq_rel() align(16) callconv(.Naked) void {
2140	2102	@setRuntimeSafety(false);
2141		return asm volatile (
	2103	asm volatile (
2142	2104	\\ cbz w16, 8f
2143	2105	\\ .inst 0x48207c82 + 0x408000
2144		\\
2145		\\ cbz wzr, 1f
	2106	\\ ret
2146	2107	\\8:
2147	2108	\\ mov x16, x0
2148	2109	\\ mov x17, x1
...	...	@@ -2154,13 +2115,12 @@ fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(.
2154	2115	\\ stlxp w15, x2, x3, [x4]
2155	2116	\\ cbnz w15, 0b
2156	2117	\\1:
2157		: [ret] "={x0}" (-> u128),
2158		: [expected] "{x0}" (expected),
2159		[desired] "{x1}" (desired),
2160		[ptr] "{x2}" (ptr),
2161		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	2118	\\ ret
	2119	:
	2120	: [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2162	2121	: "w15", "w16", "w17", "memory"
2163	2122	);
	2123	unreachable;
2164	2124	}
2165	2125
2166	2126	comptime {

tools/gen_outline_atomics.zig+26-53

...	...	@@ -76,21 +76,16 @@ fn writeFunction(
76	76	.ldeor => try generateLd(arena, n, order, .ldeor),
77	77	.ldset => try generateLd(arena, n, order, .ldset),
78	78	};
79		const fn_sig = if (op != .cas)
80		try std.fmt.allocPrint(arena, "fn {[name]s}(val: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{
81		.name = name,
82		.n = n.toBits(),
83		})
84		else
85		try std.fmt.allocPrint(arena, "fn {[name]s}(expected: u{[n]d}, desired: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{
86		.name = name,
87		.n = n.toBits(),
88		});
	79	const fn_sig = try std.fmt.allocPrint(
	80	arena,
	81	"fn {[name]s}() align(16) callconv(.Naked) void {{",
	82	.{ .name = name },
	83	);
89	84	try w.writeAll(fn_sig);
90	85	try w.writeAll(
91	86	\\
92	87	\\ @setRuntimeSafety(false);
93		\\ return asm volatile (
	88	\\ asm volatile (
94	89	\\
95	90	);
96	91	var iter = std.mem.split(u8, body, "\n");
...	...	@@ -99,32 +94,15 @@ fn writeFunction(
99	94	try w.writeAll(line);
100	95	try w.writeAll("\n");
101	96	}
102		const constraints = if (op != .cas)
103		try std.fmt.allocPrint(arena,
104		\\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}),
105		\\ : [val] "{{{[reg]s}0}}" (val),
106		\\ [ptr] "{{x1}}" (ptr),
107		\\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
108		\\ : "w15", "w16", "w17", "memory"
109		\\
110		, .{ .reg = n.register(), .ty = n.toBits() })
111		else
112		try std.fmt.allocPrint(arena,
113		\\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}),
114		\\ : [expected] "{{{[reg]s}0}}" (expected),
115		\\ [desired] "{{{[reg]s}1}}" (desired),
116		\\ [ptr] "{{x2}}" (ptr),
117		\\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
118		\\ : "w15", "w16", "w17", "memory"
119		\\
120		, .{ .reg = n.register(), .ty = n.toBits() });
121
122		try w.writeAll(constraints);
123	97	try w.writeAll(
	98	\\ :
	99	\\ : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
	100	\\ : "w15", "w16", "w17", "memory"
124	101	\\ );
	102	\\ unreachable;
	103	\\}
125	104	\\
126	105	);
127		try w.writeAll("}\n");
128	106	}
129	107
130	108	const N = enum(u8) {
...	...	@@ -185,11 +163,11 @@ const Ordering = enum {
185	163	acq_rel,
186	164
187	165	const Defines = struct {
188		suff: [:0]const u8,
189		a: [:0]const u8,
190		l: [:0]const u8,
191		m: [:0]const u8,
192		n: [:0]const u8,
	166	suff: []const u8,
	167	a: []const u8,
	168	l: []const u8,
	169	m: []const u8,
	170	n: []const u8,
193	171	};
194	172	fn defines(self: @This()) Defines {
195	173	const suff = switch (self) {
...	...	@@ -224,15 +202,6 @@ const Ordering = enum {
224	202	};
225	203	return .{ .suff = suff, .a = a, .l = l, .m = m, .n = n };
226	204	}
227
228		fn capName(self: @This()) [:0]const u8 {
229		return switch (self) {
230		.relax => "Relax",
231		.acq => "Acq",
232		.rel => "Rel",
233		.acq_rel => "AcqRel",
234		};
235		}
236	205	};
237	206
238	207	const LdName = enum { ldadd, ldclr, ldeor, ldset };
...	...	@@ -244,14 +213,14 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
244	213	const reg = n.register();
245	214
246	215	if (@enumToInt(n) < 16) {
247		const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m });
	216	const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}", .{ s_def.b, o_def.m });
248	217	const ldxr = try std.fmt.allocPrint(arena, "ld{s}xr{s}", .{ o_def.a, s_def.s });
249	218	const stxr = try std.fmt.allocPrint(arena, "st{s}xr{s}", .{ o_def.l, s_def.s });
250	219
251	220	return try std.fmt.allocPrint(arena,
252	221	\\ cbz w16, 8f
253	222	\\ {[cas]s}
254		\\ cbz wzr, 1f
	223	\\ ret
255	224	\\8:
256	225	\\ {[uxt]s} {[reg]s}16, {[reg]s}0
257	226	\\0:
...	...	@@ -261,6 +230,7 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
261	230	\\ {[stxr]s} w17, {[reg]s}1, [x2]
262	231	\\ cbnz w17, 0b
263	232	\\1:
	233	\\ ret
264	234	, .{
265	235	.cas = cas,
266	236	.uxt = s_def.uxt,
...	...	@@ -269,14 +239,14 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
269	239	.reg = reg,
270	240	});
271	241	} else {
272		const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}\n", .{o_def.m});
	242	const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}", .{o_def.m});
273	243	const ldxp = try std.fmt.allocPrint(arena, "ld{s}xp", .{o_def.a});
274	244	const stxp = try std.fmt.allocPrint(arena, "st{s}xp", .{o_def.l});
275	245
276	246	return try std.fmt.allocPrint(arena,
277	247	\\ cbz w16, 8f
278	248	\\ {[casp]s}
279		\\ cbz wzr, 1f
	249	\\ ret
280	250	\\8:
281	251	\\ mov x16, x0
282	252	\\ mov x17, x1
...	...	@@ -288,6 +258,7 @@ fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
288	258	\\ {[stxp]s} w15, x2, x3, [x4]
289	259	\\ cbnz w15, 0b
290	260	\\1:
	261	\\ ret
291	262	, .{
292	263	.casp = casp,
293	264	.ldxp = ldxp,
...	...	@@ -304,7 +275,7 @@ fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {
304	275	return try std.fmt.allocPrint(arena,
305	276	\\ cbz w16, 8f
306	277	\\ .inst 0x38208020 + {[b]s} + {[n]s}
307		\\ cbz wzr, 1f
	278	\\ ret
308	279	\\8:
309	280	\\ mov {[reg]s}16, {[reg]s}0
310	281	\\0:
...	...	@@ -312,6 +283,7 @@ fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {
312	283	\\ st{[l]s}xr{[s]s} w17, {[reg]s}16, [x1]
313	284	\\ cbnz w17, 0b
314	285	\\1:
	286	\\ ret
315	287	, .{
316	288	.b = s_def.b,
317	289	.n = o_def.n,
...	...	@@ -343,7 +315,7 @@ fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {
343	315	return try std.fmt.allocPrint(arena,
344	316	\\ cbz w16, 8f
345	317	\\ .inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}
346		\\ cbz wzr, 1f
	318	\\ ret
347	319	\\8:
348	320	\\ mov {[reg]s}16, {[reg]s}0
349	321	\\0:
...	...	@@ -352,6 +324,7 @@ fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {
352	324	\\ st{[l]s}xr{[s]s} w15, {[reg]s}17, [x1]
353	325	\\ cbnz w15, 0b
354	326	\\1:
	327	\\ ret
355	328	, .{
356	329	.op_n = op_n,
357	330	.b = s_def.b,