Commit 5f1173bb2b0a7012640bd5383c61b433b16a452d | rust

.github/workflows/ghcr.yml+2

...	...	@@ -55,6 +55,8 @@ jobs:
55	55	images=(
56	56	# Mirrored because used by the tidy job, which doesn't cache Docker images
57	57	"ubuntu:22.04"
	58	# Mirrored because used by x86-64-gnu-miri
	59	"ubuntu:24.04"
58	60	# Mirrored because used by all linux CI jobs, including tidy
59	61	"moby/buildkit:buildx-stable-1"
60	62	# Mirrored because used when CI is running inside a Docker container

compiler/rustc_ast/src/ast.rs+13

...	...	@@ -1259,6 +1259,19 @@ pub enum StmtKind {
1259	1259	MacCall(Box<MacCallStmt>),
1260	1260	}
1261	1261
	1262	impl StmtKind {
	1263	pub fn descr(&self) -> &'static str {
	1264	match self {
	1265	StmtKind::Let(_) => "local",
	1266	StmtKind::Item(_) => "item",
	1267	StmtKind::Expr(_) => "expression",
	1268	StmtKind::Semi(_) => "statement",
	1269	StmtKind::Empty => "semicolon",
	1270	StmtKind::MacCall(_) => "macro call",
	1271	}
	1272	}
	1273	}
	1274
1262	1275	#[derive(Clone, Encodable, Decodable, Debug, Walkable)]
1263	1276	pub struct MacCallStmt {
1264	1277	pub mac: Box<MacCall>,

compiler/rustc_attr_parsing/messages.ftl+1

...	...	@@ -87,6 +87,7 @@ attr_parsing_invalid_link_modifier =
87	87	attr_parsing_invalid_meta_item = expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found {$descr}
88	88	.remove_neg_sugg = negative numbers are not literals, try removing the `-` sign
89	89	.quote_ident_sugg = surround the identifier with quotation marks to make it into a string literal
	90	.label = {$descr}s are not allowed here
90	91
91	92	attr_parsing_invalid_predicate =
92	93	invalid predicate `{$predicate}`

compiler/rustc_attr_parsing/src/parser.rs+45-23

...	...	@@ -8,12 +8,12 @@ use std::fmt::{Debug, Display};
8	8
9	9	use rustc_ast::token::{self, Delimiter, MetaVarKind};
10	10	use rustc_ast::tokenstream::TokenStream;
11		use rustc_ast::{AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, NormalAttr, Path};
	11	use rustc_ast::{AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, NormalAttr, Path, StmtKind, UnOp};
12	12	use rustc_ast_pretty::pprust;
13	13	use rustc_errors::{Diag, PResult};
14	14	use rustc_hir::{self as hir, AttrPath};
15	15	use rustc_parse::exp;
16		use rustc_parse::parser::{Parser, PathStyle, token_descr};
	16	use rustc_parse::parser::{ForceCollect, Parser, PathStyle, token_descr};
17	17	use rustc_session::errors::{create_lit_error, report_lit_error};
18	18	use rustc_session::parse::ParseSess;
19	19	use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, sym};
...	...	@@ -488,33 +488,55 @@ impl<'a, 'sess> MetaItemListParserContext<'a, 'sess> {
488	488	descr: token_descr(&self.parser.token),
489	489	quote_ident_sugg: None,
490	490	remove_neg_sugg: None,
	491	label: None,
491	492	};
492	493
	494	if let token::OpenInvisible(_) = self.parser.token.kind {
	495	// Do not attempt to suggest anything when encountered as part of a macro expansion.
	496	return self.parser.dcx().create_err(err);
	497	}
	498
493	499	// Suggest quoting idents, e.g. in `#[cfg(key = value)]`. We don't use `Token::ident` and
494	500	// don't `uninterpolate` the token to avoid suggesting anything butchered or questionable
495	501	// when macro metavariables are involved.
496		if self.parser.prev_token == token::Eq
497		&& let token::Ident(..) = self.parser.token.kind
498		{
499		let before = self.parser.token.span.shrink_to_lo();
500		while let token::Ident(..) = self.parser.token.kind {
501		self.parser.bump();
	502	let snapshot = self.parser.create_snapshot_for_diagnostic();
	503	let stmt = self.parser.parse_stmt_without_recovery(false, ForceCollect::No, false);
	504	match stmt {
	505	Ok(Some(stmt)) => {
	506	// The user tried to write something like
	507	// `#[deprecated(note = concat!("a", "b"))]`.
	508	err.descr = stmt.kind.descr().to_string();
	509	err.label = Some(stmt.span);
	510	err.span = stmt.span;
	511	if let StmtKind::Expr(expr) = &stmt.kind
	512	&& let ExprKind::Unary(UnOp::Neg, val) = &expr.kind
	513	&& let ExprKind::Lit(_) = val.kind
	514	{
	515	err.remove_neg_sugg = Some(InvalidMetaItemRemoveNegSugg {
	516	negative_sign: expr.span.until(val.span),
	517	});
	518	} else if let StmtKind::Expr(expr) = &stmt.kind
	519	&& let ExprKind::Path(None, Path { segments, .. }) = &expr.kind
	520	&& segments.len() == 1
	521	{
	522	while let token::Ident(..) \| token::Literal(_) \| token::Dot =
	523	self.parser.token.kind
	524	{
	525	// We've got a word, so we try to consume the rest of a potential sentence.
	526	// We include `.` to correctly handle things like `A sentence here.`.
	527	self.parser.bump();
	528	}
	529	err.quote_ident_sugg = Some(InvalidMetaItemQuoteIdentSugg {
	530	before: expr.span.shrink_to_lo(),
	531	after: self.parser.prev_token.span.shrink_to_hi(),
	532	});
	533	}
	534	}
	535	Ok(None) => {}
	536	Err(e) => {
	537	e.cancel();
	538	self.parser.restore_snapshot(snapshot);
502	539	}
503		err.quote_ident_sugg = Some(InvalidMetaItemQuoteIdentSugg {
504		before,
505		after: self.parser.prev_token.span.shrink_to_hi(),
506		});
507		}
508
509		if self.parser.token == token::Minus
510		&& self
511		.parser
512		.look_ahead(1, \|t\| matches!(t.kind, rustc_ast::token::TokenKind::Literal { .. }))
513		{
514		err.remove_neg_sugg =
515		Some(InvalidMetaItemRemoveNegSugg { negative_sign: self.parser.token.span });
516		self.parser.bump();
517		self.parser.bump();
518	540	}
519	541
520	542	self.parser.dcx().create_err(err)

compiler/rustc_attr_parsing/src/session_diagnostics.rs+2

...	...	@@ -804,6 +804,8 @@ pub(crate) struct InvalidMetaItem {
804	804	pub quote_ident_sugg: Option<InvalidMetaItemQuoteIdentSugg>,
805	805	#[subdiagnostic]
806	806	pub remove_neg_sugg: Option<InvalidMetaItemRemoveNegSugg>,
	807	#[label]
	808	pub label: Option<Span>,
807	809	}
808	810
809	811	#[derive(Subdiagnostic)]

compiler/rustc_fluent_macro/src/fluent.rs+3

...	...	@@ -8,6 +8,9 @@ use fluent_syntax::ast::{
8	8	Attribute, Entry, Expression, Identifier, InlineExpression, Message, Pattern, PatternElement,
9	9	};
10	10	use fluent_syntax::parser::ParserError;
	11	#[cfg(not(bootstrap))]
	12	use proc_macro::tracked::path;
	13	#[cfg(bootstrap)]
11	14	use proc_macro::tracked_path::path;
12	15	use proc_macro::{Diagnostic, Level, Span};
13	16	use proc_macro2::TokenStream;

compiler/rustc_fluent_macro/src/lib.rs+2-1

...	...	@@ -1,7 +1,8 @@
1	1	// tidy-alphabetical-start
2	2	#![allow(rustc::default_hash_types)]
	3	#![cfg_attr(bootstrap, feature(track_path))]
	4	#![cfg_attr(not(bootstrap), feature(proc_macro_tracked_path))]
3	5	#![feature(proc_macro_diagnostic)]
4		#![feature(track_path)]
5	6	// tidy-alphabetical-end
6	7
7	8	use proc_macro::TokenStream;

compiler/rustc_macros/src/current_version.rs+4

...	...	@@ -22,7 +22,11 @@ struct RustcVersion {
22	22
23	23	impl RustcVersion {
24	24	fn parse_cfg_release(env_var: &str) -> Result<Self, Box<dyn std::error::Error>> {
	25	#[cfg(not(bootstrap))]
	26	let value = proc_macro::tracked::env_var(env_var)?;
	27	#[cfg(bootstrap)]
25	28	let value = proc_macro::tracked_env::var(env_var)?;
	29
26	30	Self::parse_str(&value)
27	31	.ok_or_else(\|\| format!("failed to parse rustc version: {:?}", value).into())
28	32	}

compiler/rustc_macros/src/symbols.rs+7-1

...	...	@@ -259,7 +259,13 @@ fn symbols_with_errors(input: TokenStream) -> (TokenStream, Vec<syn::Error>) {
259	259	break;
260	260	}
261	261
262		let value = match proc_macro::tracked_env::var(env_var.value()) {
	262	#[cfg(bootstrap)]
	263	let tracked_env = proc_macro::tracked_env::var(env_var.value());
	264
	265	#[cfg(not(bootstrap))]
	266	let tracked_env = proc_macro::tracked::env_var(env_var.value());
	267
	268	let value = match tracked_env {
263	269	Ok(value) => value,
264	270	Err(err) => {
265	271	errors.list.push(syn::Error::new_spanned(expr, err));

compiler/rustc_mir_build/src/builder/mod.rs+20-1

...	...	@@ -839,6 +839,25 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
839	839	self.infcx.typing_env(self.param_env),
840	840	);
841	841
	842	// check if the function's return type is inhabited
	843	// this was added here because of this regression
	844	// https://github.com/rust-lang/rust/issues/149571
	845	let output_is_inhabited =
	846	if matches!(self.tcx.def_kind(self.def_id), DefKind::Fn \| DefKind::AssocFn) {
	847	self.tcx
	848	.fn_sig(self.def_id)
	849	.instantiate_identity()
	850	.skip_binder()
	851	.output()
	852	.is_inhabited_from(
	853	self.tcx,
	854	self.parent_module,
	855	self.infcx.typing_env(self.param_env),
	856	)
	857	} else {
	858	true
	859	};
	860
842	861	if !ty_is_inhabited {
843	862	// Unreachable code warnings are already emitted during type checking.
844	863	// However, during type checking, full type information is being
...	...	@@ -849,7 +868,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
849	868	// uninhabited types (e.g. empty enums). The check above is used so
850	869	// that we do not emit the same warning twice if the uninhabited type
851	870	// is indeed `!`.
852		if !ty.is_never() {
	871	if !ty.is_never() && output_is_inhabited {
853	872	lints.push((target_bb, ty, term.source_info.span));
854	873	}
855	874

compiler/rustc_parse/src/parser/diagnostics.rs+1-1

...	...	@@ -284,7 +284,7 @@ impl<'a> Parser<'a> {
284	284	}
285	285
286	286	/// Replace `self` with `snapshot.parser`.
287		pub(super) fn restore_snapshot(&mut self, snapshot: SnapshotParser<'a>) {
	287	pub fn restore_snapshot(&mut self, snapshot: SnapshotParser<'a>) {
288	288	*self = snapshot.parser;
289	289	}
290	290

compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs+1-1

...	...	@@ -21,7 +21,7 @@ pub(crate) fn target() -> Target {
21	21	max_atomic_width: Some(128),
22	22	// As documented in https://developer.android.com/ndk/guides/cpu-features.html
23	23	// the neon (ASIMD) and FP must exist on all android aarch64 targets.
24		features: "+v8a,+neon".into(),
	24	features: "+v8a,+neon,+outline-atomics".into(),
25	25	// the AAPCS64 expects use of non-leaf frame pointers per
26	26	// https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
27	27	// and we tend to encounter interesting bugs in AArch64 unwinding code if we do not

compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs+1-1

...	...	@@ -3,7 +3,7 @@ use crate::spec::{Arch, Cc, FramePointer, LinkerFlavor, Lld, Target, TargetMetad
3	3	pub(crate) fn target() -> Target {
4	4	let mut base = base::windows_gnullvm::opts();
5	5	base.max_atomic_width = Some(128);
6		base.features = "+v8a,+neon".into();
	6	base.features = "+v8a,+neon,+outline-atomics".into();
7	7	base.linker = Some("aarch64-w64-mingw32-clang".into());
8	8	base.add_pre_link_args(LinkerFlavor::Gnu(Cc::No, Lld::No), &["-m", "arm64pe"]);
9	9

compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs+1-1

...	...	@@ -3,7 +3,7 @@ use crate::spec::{Arch, FramePointer, Target, TargetMetadata, base};
3	3	pub(crate) fn target() -> Target {
4	4	let mut base = base::windows_msvc::opts();
5	5	base.max_atomic_width = Some(128);
6		base.features = "+v8a,+neon".into();
	6	base.features = "+v8a,+neon,+outline-atomics".into();
7	7
8	8	// Microsoft recommends enabling frame pointers on Arm64 Windows.
9	9	// From https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170#integer-registers

compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs+1-1

...	...	@@ -5,7 +5,7 @@ use crate::spec::{
5	5	pub(crate) fn target() -> Target {
6	6	let mut base = base::fuchsia::opts();
7	7	base.cpu = "generic".into();
8		base.features = "+v8a,+crc,+aes,+sha2,+neon".into();
	8	base.features = "+v8a,+crc,+aes,+sha2,+neon,+outline-atomics".into();
9	9	base.max_atomic_width = Some(128);
10	10	base.stack_probes = StackProbeType::Inline;
11	11	base.supported_sanitizers = SanitizerSet::ADDRESS

compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs+1-1

...	...	@@ -13,7 +13,7 @@ pub(crate) fn target() -> Target {
13	13	data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(),
14	14	arch: Arch::AArch64,
15	15	options: TargetOptions {
16		features: "+v8a".into(),
	16	features: "+v8a,+outline-atomics".into(),
17	17	max_atomic_width: Some(128),
18	18	stack_probes: StackProbeType::Inline,
19	19	..base::openbsd::opts()

library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs deleted-399

...	...	@@ -1,399 +0,0 @@
1		//! Aarch64 targets have two possible implementations for atomics:
2		//! 1. Load-Locked, Store-Conditional (LL/SC), older and slower.
3		//! 2. Large System Extensions (LSE), newer and faster.
4		//! To avoid breaking backwards compat, C toolchains introduced a concept of "outlined atomics",
5		//! where atomic operations call into the compiler runtime to dispatch between two depending on
6		//! which is supported on the current CPU.
7		//! See <https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10#:~:text=out%20of%20line%20atomics> for more discussion.
8		//!
9		//! Ported from `aarch64/lse.S` in LLVM's compiler-rt.
10		//!
11		//! Generate functions for each of the following symbols:
12		//! __aarch64_casM_ORDER
13		//! __aarch64_swpN_ORDER
14		//! __aarch64_ldaddN_ORDER
15		//! __aarch64_ldclrN_ORDER
16		//! __aarch64_ldeorN_ORDER
17		//! __aarch64_ldsetN_ORDER
18		//! for N = {1, 2, 4, 8}, M = {1, 2, 4, 8, 16}, ORDER = { relax, acq, rel, acq_rel }
19		//!
20		//! The original `lse.S` has some truly horrifying code that expects to be compiled multiple times with different constants.
21		//! We do something similar, but with macro arguments.
22		#![cfg_attr(feature = "c", allow(unused_macros))] // avoid putting the macros into a submodule
23
24		use core::sync::atomic::{AtomicU8, Ordering};
25
26		/// non-zero if the host supports LSE atomics.
27		static HAVE_LSE_ATOMICS: AtomicU8 = AtomicU8::new(0);
28
29		intrinsics! {
30		/// Call to enable LSE in outline atomic operations. The caller must verify
31		/// LSE operations are supported.
32		pub extern "C" fn __rust_enable_lse() {
33		HAVE_LSE_ATOMICS.store(1, Ordering::Relaxed);
34		}
35		}
36
37		/// Translate a byte size to a Rust type.
38		#[rustfmt::skip]
39		macro_rules! int_ty {
40		(1) => { i8 };
41		(2) => { i16 };
42		(4) => { i32 };
43		(8) => { i64 };
44		(16) => { i128 };
45		}
46
47		/// Given a byte size and a register number, return a register of the appropriate size.
48		///
49		/// See <https://developer.arm.com/documentation/102374/0101/Registers-in-AArch64---general-purpose-registers>.
50		#[rustfmt::skip]
51		macro_rules! reg {
52		(1, $num:literal) => { concat!("w", $num) };
53		(2, $num:literal) => { concat!("w", $num) };
54		(4, $num:literal) => { concat!("w", $num) };
55		(8, $num:literal) => { concat!("x", $num) };
56		(16, $num:literal) => { concat!("x", $num) };
57		}
58
59		/// Given an atomic ordering, translate it to the acquire suffix for the lxdr aarch64 ASM instruction.
60		#[rustfmt::skip]
61		macro_rules! acquire {
62		(Relaxed) => { "" };
63		(Acquire) => { "a" };
64		(Release) => { "" };
65		(AcqRel) => { "a" };
66		}
67
68		/// Given an atomic ordering, translate it to the release suffix for the stxr aarch64 ASM instruction.
69		#[rustfmt::skip]
70		macro_rules! release {
71		(Relaxed) => { "" };
72		(Acquire) => { "" };
73		(Release) => { "l" };
74		(AcqRel) => { "l" };
75		}
76
77		/// Given a size in bytes, translate it to the byte suffix for an aarch64 ASM instruction.
78		#[rustfmt::skip]
79		macro_rules! size {
80		(1) => { "b" };
81		(2) => { "h" };
82		(4) => { "" };
83		(8) => { "" };
84		(16) => { "" };
85		}
86
87		/// Given a byte size, translate it to an Unsigned eXTend instruction
88		/// with the correct semantics.
89		///
90		/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTB--Unsigned-Extend-Byte--an-alias-of-UBFM->
91		#[rustfmt::skip]
92		macro_rules! uxt {
93		(1) => { "uxtb" };
94		(2) => { "uxth" };
95		($_:tt) => { "mov" };
96		}
97
98		/// Given an atomic ordering and byte size, translate it to a LoaD eXclusive Register instruction
99		/// with the correct semantics.
100		///
101		/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDXR--Load-Exclusive-Register->.
102		macro_rules! ldxr {
103		($ordering:ident, $bytes:tt) => {
104		concat!("ld", acquire!($ordering), "xr", size!($bytes))
105		};
106		}
107
108		/// Given an atomic ordering and byte size, translate it to a STore eXclusive Register instruction
109		/// with the correct semantics.
110		///
111		/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STXR--Store-Exclusive-Register->.
112		macro_rules! stxr {
113		($ordering:ident, $bytes:tt) => {
114		concat!("st", release!($ordering), "xr", size!($bytes))
115		};
116		}
117
118		/// Given an atomic ordering and byte size, translate it to a LoaD eXclusive Pair of registers instruction
119		/// with the correct semantics.
120		///
121		/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDXP--Load-Exclusive-Pair-of-Registers->
122		macro_rules! ldxp {
123		($ordering:ident) => {
124		concat!("ld", acquire!($ordering), "xp")
125		};
126		}
127
128		/// Given an atomic ordering and byte size, translate it to a STore eXclusive Pair of registers instruction
129		/// with the correct semantics.
130		///
131		/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STXP--Store-Exclusive-Pair-of-registers->.
132		macro_rules! stxp {
133		($ordering:ident) => {
134		concat!("st", release!($ordering), "xp")
135		};
136		}
137
138		// If supported, perform the requested LSE op and return, or fallthrough.
139		macro_rules! try_lse_op {
140		($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => {
141		concat!(
142		".arch_extension lse; ",
143		"adrp x16, {have_lse}; ",
144		"ldrb w16, [x16, :lo12:{have_lse}]; ",
145		"cbz w16, 8f; ",
146		// LSE_OP s(reg),* [$mem]
147		concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]; ",),
148		"ret; ",
149		"8:"
150		)
151		};
152		}
153
154		// Translate memory ordering to the LSE suffix
155		#[rustfmt::skip]
156		macro_rules! lse_mem_sfx {
157		(Relaxed) => { "" };
158		(Acquire) => { "a" };
159		(Release) => { "l" };
160		(AcqRel) => { "al" };
161		}
162
163		// Generate the aarch64 LSE operation for memory ordering and width
164		macro_rules! lse {
165		($op:literal, $order:ident, 16) => {
166		concat!($op, "p", lse_mem_sfx!($order))
167		};
168		($op:literal, $order:ident, $bytes:tt) => {
169		concat!($op, lse_mem_sfx!($order), size!($bytes))
170		};
171		}
172
173		/// See <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.compare_and_swap>.
174		macro_rules! compare_and_swap {
175		($ordering:ident, $bytes:tt, $name:ident) => {
176		intrinsics! {
177		#[maybe_use_optimized_c_shim]
178		#[unsafe(naked)]
179		pub unsafe extern "C" fn $name (
180		expected: int_ty!($bytes), desired: int_ty!($bytes), ptr: *mut int_ty!($bytes)
181		) -> int_ty!($bytes) {
182		// We can't use `AtomicI8::compare_and_swap`; we are compare_and_swap.
183		core::arch::naked_asm! {
184		// CAS s(0), s(1), [x2]; if LSE supported.
185		try_lse_op!("cas", $ordering, $bytes, 0, 1, [x2]),
186		// UXT s(tmp0), s(0)
187		concat!(uxt!($bytes), " ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
188		"0:",
189		// LDXR s(0), [x2]
190		concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x2]"),
191		// cmp s(0), s(tmp0)
192		concat!("cmp ", reg!($bytes, 0), ", ", reg!($bytes, 16)),
193		"bne 1f",
194		// STXR w(tmp1), s(1), [x2]
195		concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"),
196		"cbnz w17, 0b",
197		"1:",
198		"ret",
199		have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
200		}
201		}
202		}
203		};
204		}
205
206		// i128 uses a completely different impl, so it has its own macro.
207		macro_rules! compare_and_swap_i128 {
208		($ordering:ident, $name:ident) => {
209		intrinsics! {
210		#[maybe_use_optimized_c_shim]
211		#[unsafe(naked)]
212		pub unsafe extern "C" fn $name (
213		expected: i128, desired: i128, ptr: *mut i128
214		) -> i128 {
215		core::arch::naked_asm! {
216		// CASP x0, x1, x2, x3, [x4]; if LSE supported.
217		try_lse_op!("cas", $ordering, 16, 0, 1, 2, 3, [x4]),
218		"mov x16, x0",
219		"mov x17, x1",
220		"0:",
221		// LDXP x0, x1, [x4]
222		concat!(ldxp!($ordering), " x0, x1, [x4]"),
223		"cmp x0, x16",
224		"ccmp x1, x17, #0, eq",
225		"bne 1f",
226		// STXP w(tmp2), x2, x3, [x4]
227		concat!(stxp!($ordering), " w15, x2, x3, [x4]"),
228		"cbnz w15, 0b",
229		"1:",
230		"ret",
231		have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
232		}
233		}
234		}
235		};
236		}
237
238		/// See <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.swap>.
239		macro_rules! swap {
240		($ordering:ident, $bytes:tt, $name:ident) => {
241		intrinsics! {
242		#[maybe_use_optimized_c_shim]
243		#[unsafe(naked)]
244		pub unsafe extern "C" fn $name (
245		left: int_ty!($bytes), right_ptr: *mut int_ty!($bytes)
246		) -> int_ty!($bytes) {
247		core::arch::naked_asm! {
248		// SWP s(0), s(0), [x1]; if LSE supported.
249		try_lse_op!("swp", $ordering, $bytes, 0, 0, [x1]),
250		// mov s(tmp0), s(0)
251		concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
252		"0:",
253		// LDXR s(0), [x1]
254		concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"),
255		// STXR w(tmp1), s(tmp0), [x1]
256		concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"),
257		"cbnz w17, 0b",
258		"ret",
259		have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
260		}
261		}
262		}
263		};
264		}
265
266		/// See (e.g.) <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.fetch_add>.
267		macro_rules! fetch_op {
268		($ordering:ident, $bytes:tt, $name:ident, $op:literal, $lse_op:literal) => {
269		intrinsics! {
270		#[maybe_use_optimized_c_shim]
271		#[unsafe(naked)]
272		pub unsafe extern "C" fn $name (
273		val: int_ty!($bytes), ptr: *mut int_ty!($bytes)
274		) -> int_ty!($bytes) {
275		core::arch::naked_asm! {
276		// LSEOP s(0), s(0), [x1]; if LSE supported.
277		try_lse_op!($lse_op, $ordering, $bytes, 0, 0, [x1]),
278		// mov s(tmp0), s(0)
279		concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
280		"0:",
281		// LDXR s(0), [x1]
282		concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"),
283		// OP s(tmp1), s(0), s(tmp0)
284		concat!($op, " ", reg!($bytes, 17), ", ", reg!($bytes, 0), ", ", reg!($bytes, 16)),
285		// STXR w(tmp2), s(tmp1), [x1]
286		concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"),
287		"cbnz w15, 0b",
288		"ret",
289		have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
290		}
291		}
292		}
293		}
294		}
295
296		// We need a single macro to pass to `foreach_ldadd`.
297		macro_rules! add {
298		($ordering:ident, $bytes:tt, $name:ident) => {
299		fetch_op! { $ordering, $bytes, $name, "add", "ldadd" }
300		};
301		}
302
303		macro_rules! and {
304		($ordering:ident, $bytes:tt, $name:ident) => {
305		fetch_op! { $ordering, $bytes, $name, "bic", "ldclr" }
306		};
307		}
308
309		macro_rules! xor {
310		($ordering:ident, $bytes:tt, $name:ident) => {
311		fetch_op! { $ordering, $bytes, $name, "eor", "ldeor" }
312		};
313		}
314
315		macro_rules! or {
316		($ordering:ident, $bytes:tt, $name:ident) => {
317		fetch_op! { $ordering, $bytes, $name, "orr", "ldset" }
318		};
319		}
320
321		#[macro_export]
322		macro_rules! foreach_ordering {
323		($macro:path, $bytes:tt, $name:ident) => {
324		$macro!( Relaxed, $bytes, ${concat($name, _relax)} );
325		$macro!( Acquire, $bytes, ${concat($name, _acq)} );
326		$macro!( Release, $bytes, ${concat($name, _rel)} );
327		$macro!( AcqRel, $bytes, ${concat($name, _acq_rel)} );
328		};
329		($macro:path, $name:ident) => {
330		$macro!( Relaxed, ${concat($name, _relax)} );
331		$macro!( Acquire, ${concat($name, _acq)} );
332		$macro!( Release, ${concat($name, _rel)} );
333		$macro!( AcqRel, ${concat($name, _acq_rel)} );
334		};
335		}
336
337		#[macro_export]
338		macro_rules! foreach_bytes {
339		($macro:path, $name:ident) => {
340		foreach_ordering!( $macro, 1, ${concat(__aarch64_, $name, "1")} );
341		foreach_ordering!( $macro, 2, ${concat(__aarch64_, $name, "2")} );
342		foreach_ordering!( $macro, 4, ${concat(__aarch64_, $name, "4")} );
343		foreach_ordering!( $macro, 8, ${concat(__aarch64_, $name, "8")} );
344		};
345		}
346
347		/// Generate different macros for cas/swp/add/clr/eor/set so that we can test them separately.
348		#[macro_export]
349		macro_rules! foreach_cas {
350		($macro:path) => {
351		foreach_bytes!($macro, cas);
352		};
353		}
354
355		/// Only CAS supports 16 bytes, and it has a different implementation that uses a different macro.
356		#[macro_export]
357		macro_rules! foreach_cas16 {
358		($macro:path) => {
359		foreach_ordering!($macro, __aarch64_cas16);
360		};
361		}
362		#[macro_export]
363		macro_rules! foreach_swp {
364		($macro:path) => {
365		foreach_bytes!($macro, swp);
366		};
367		}
368		#[macro_export]
369		macro_rules! foreach_ldadd {
370		($macro:path) => {
371		foreach_bytes!($macro, ldadd);
372		};
373		}
374		#[macro_export]
375		macro_rules! foreach_ldclr {
376		($macro:path) => {
377		foreach_bytes!($macro, ldclr);
378		};
379		}
380		#[macro_export]
381		macro_rules! foreach_ldeor {
382		($macro:path) => {
383		foreach_bytes!($macro, ldeor);
384		};
385		}
386		#[macro_export]
387		macro_rules! foreach_ldset {
388		($macro:path) => {
389		foreach_bytes!($macro, ldset);
390		};
391		}
392
393		foreach_cas!(compare_and_swap);
394		foreach_cas16!(compare_and_swap_i128);
395		foreach_swp!(swap);
396		foreach_ldadd!(add);
397		foreach_ldclr!(and);
398		foreach_ldeor!(xor);
399		foreach_ldset!(or);

library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs created+399

...	...	@@ -0,0 +1,399 @@
	1	//! Aarch64 targets have two possible implementations for atomics:
	2	//! 1. Load-Locked, Store-Conditional (LL/SC), older and slower.
	3	//! 2. Large System Extensions (LSE), newer and faster.
	4	//! To avoid breaking backwards compat, C toolchains introduced a concept of "outlined atomics",
	5	//! where atomic operations call into the compiler runtime to dispatch between two depending on
	6	//! which is supported on the current CPU.
	7	//! See <https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10#:~:text=out%20of%20line%20atomics> for more discussion.
	8	//!
	9	//! Ported from `aarch64/lse.S` in LLVM's compiler-rt.
	10	//!
	11	//! Generate functions for each of the following symbols:
	12	//! __aarch64_casM_ORDER
	13	//! __aarch64_swpN_ORDER
	14	//! __aarch64_ldaddN_ORDER
	15	//! __aarch64_ldclrN_ORDER
	16	//! __aarch64_ldeorN_ORDER
	17	//! __aarch64_ldsetN_ORDER
	18	//! for N = {1, 2, 4, 8}, M = {1, 2, 4, 8, 16}, ORDER = { relax, acq, rel, acq_rel }
	19	//!
	20	//! The original `lse.S` has some truly horrifying code that expects to be compiled multiple times with different constants.
	21	//! We do something similar, but with macro arguments.
	22	#![cfg_attr(feature = "c", allow(unused_macros))] // avoid putting the macros into a submodule
	23
	24	use core::sync::atomic::{AtomicU8, Ordering};
	25
	26	/// non-zero if the host supports LSE atomics.
	27	static HAVE_LSE_ATOMICS: AtomicU8 = AtomicU8::new(0);
	28
	29	intrinsics! {
	30	/// Call to enable LSE in outline atomic operations. The caller must verify
	31	/// LSE operations are supported.
	32	pub extern "C" fn __rust_enable_lse() {
	33	HAVE_LSE_ATOMICS.store(1, Ordering::Relaxed);
	34	}
	35	}
	36
	37	/// Translate a byte size to a Rust type.
	38	#[rustfmt::skip]
	39	macro_rules! int_ty {
	40	(1) => { i8 };
	41	(2) => { i16 };
	42	(4) => { i32 };
	43	(8) => { i64 };
	44	(16) => { i128 };
	45	}
	46
	47	/// Given a byte size and a register number, return a register of the appropriate size.
	48	///
	49	/// See <https://developer.arm.com/documentation/102374/0101/Registers-in-AArch64---general-purpose-registers>.
	50	#[rustfmt::skip]
	51	macro_rules! reg {
	52	(1, $num:literal) => { concat!("w", $num) };
	53	(2, $num:literal) => { concat!("w", $num) };
	54	(4, $num:literal) => { concat!("w", $num) };
	55	(8, $num:literal) => { concat!("x", $num) };
	56	(16, $num:literal) => { concat!("x", $num) };
	57	}
	58
	59	/// Given an atomic ordering, translate it to the acquire suffix for the lxdr aarch64 ASM instruction.
	60	#[rustfmt::skip]
	61	macro_rules! acquire {
	62	(Relaxed) => { "" };
	63	(Acquire) => { "a" };
	64	(Release) => { "" };
	65	(AcqRel) => { "a" };
	66	}
	67
	68	/// Given an atomic ordering, translate it to the release suffix for the stxr aarch64 ASM instruction.
	69	#[rustfmt::skip]
	70	macro_rules! release {
	71	(Relaxed) => { "" };
	72	(Acquire) => { "" };
	73	(Release) => { "l" };
	74	(AcqRel) => { "l" };
	75	}
	76
	77	/// Given a size in bytes, translate it to the byte suffix for an aarch64 ASM instruction.
	78	#[rustfmt::skip]
	79	macro_rules! size {
	80	(1) => { "b" };
	81	(2) => { "h" };
	82	(4) => { "" };
	83	(8) => { "" };
	84	(16) => { "" };
	85	}
	86
	87	/// Given a byte size, translate it to an Unsigned eXTend instruction
	88	/// with the correct semantics.
	89	///
	90	/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTB--Unsigned-Extend-Byte--an-alias-of-UBFM->
	91	#[rustfmt::skip]
	92	macro_rules! uxt {
	93	(1) => { "uxtb" };
	94	(2) => { "uxth" };
	95	($_:tt) => { "mov" };
	96	}
	97
	98	/// Given an atomic ordering and byte size, translate it to a LoaD eXclusive Register instruction
	99	/// with the correct semantics.
	100	///
	101	/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDXR--Load-Exclusive-Register->.
	102	macro_rules! ldxr {
	103	($ordering:ident, $bytes:tt) => {
	104	concat!("ld", acquire!($ordering), "xr", size!($bytes))
	105	};
	106	}
	107
	108	/// Given an atomic ordering and byte size, translate it to a STore eXclusive Register instruction
	109	/// with the correct semantics.
	110	///
	111	/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STXR--Store-Exclusive-Register->.
	112	macro_rules! stxr {
	113	($ordering:ident, $bytes:tt) => {
	114	concat!("st", release!($ordering), "xr", size!($bytes))
	115	};
	116	}
	117
	118	/// Given an atomic ordering and byte size, translate it to a LoaD eXclusive Pair of registers instruction
	119	/// with the correct semantics.
	120	///
	121	/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDXP--Load-Exclusive-Pair-of-Registers->
	122	macro_rules! ldxp {
	123	($ordering:ident) => {
	124	concat!("ld", acquire!($ordering), "xp")
	125	};
	126	}
	127
	128	/// Given an atomic ordering and byte size, translate it to a STore eXclusive Pair of registers instruction
	129	/// with the correct semantics.
	130	///
	131	/// See <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/STXP--Store-Exclusive-Pair-of-registers->.
	132	macro_rules! stxp {
	133	($ordering:ident) => {
	134	concat!("st", release!($ordering), "xp")
	135	};
	136	}
	137
	138	// If supported, perform the requested LSE op and return, or fallthrough.
	139	macro_rules! try_lse_op {
	140	($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => {
	141	concat!(
	142	".arch_extension lse; ",
	143	"adrp x16, {have_lse}; ",
	144	"ldrb w16, [x16, :lo12:{have_lse}]; ",
	145	"cbz w16, 8f; ",
	146	// LSE_OP s(reg),* [$mem]
	147	concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]; ",),
	148	"ret; ",
	149	"8:"
	150	)
	151	};
	152	}
	153
	154	// Translate memory ordering to the LSE suffix
	155	#[rustfmt::skip]
	156	macro_rules! lse_mem_sfx {
	157	(Relaxed) => { "" };
	158	(Acquire) => { "a" };
	159	(Release) => { "l" };
	160	(AcqRel) => { "al" };
	161	}
	162
	163	// Generate the aarch64 LSE operation for memory ordering and width
	164	macro_rules! lse {
	165	($op:literal, $order:ident, 16) => {
	166	concat!($op, "p", lse_mem_sfx!($order))
	167	};
	168	($op:literal, $order:ident, $bytes:tt) => {
	169	concat!($op, lse_mem_sfx!($order), size!($bytes))
	170	};
	171	}
	172
	173	/// See <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.compare_and_swap>.
	174	macro_rules! compare_and_swap {
	175	($ordering:ident, $bytes:tt, $name:ident) => {
	176	intrinsics! {
	177	#[maybe_use_optimized_c_shim]
	178	#[unsafe(naked)]
	179	pub unsafe extern "C" fn $name (
	180	expected: int_ty!($bytes), desired: int_ty!($bytes), ptr: *mut int_ty!($bytes)
	181	) -> int_ty!($bytes) {
	182	// We can't use `AtomicI8::compare_and_swap`; we are compare_and_swap.
	183	core::arch::naked_asm! {
	184	// CAS s(0), s(1), [x2]; if LSE supported.
	185	try_lse_op!("cas", $ordering, $bytes, 0, 1, [x2]),
	186	// UXT s(tmp0), s(0)
	187	concat!(uxt!($bytes), " ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
	188	"0:",
	189	// LDXR s(0), [x2]
	190	concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x2]"),
	191	// cmp s(0), s(tmp0)
	192	concat!("cmp ", reg!($bytes, 0), ", ", reg!($bytes, 16)),
	193	"bne 1f",
	194	// STXR w(tmp1), s(1), [x2]
	195	concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"),
	196	"cbnz w17, 0b",
	197	"1:",
	198	"ret",
	199	have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
	200	}
	201	}
	202	}
	203	};
	204	}
	205
	206	// i128 uses a completely different impl, so it has its own macro.
	207	macro_rules! compare_and_swap_i128 {
	208	($ordering:ident, $name:ident) => {
	209	intrinsics! {
	210	#[maybe_use_optimized_c_shim]
	211	#[unsafe(naked)]
	212	pub unsafe extern "C" fn $name (
	213	expected: i128, desired: i128, ptr: *mut i128
	214	) -> i128 {
	215	core::arch::naked_asm! {
	216	// CASP x0, x1, x2, x3, [x4]; if LSE supported.
	217	try_lse_op!("cas", $ordering, 16, 0, 1, 2, 3, [x4]),
	218	"mov x16, x0",
	219	"mov x17, x1",
	220	"0:",
	221	// LDXP x0, x1, [x4]
	222	concat!(ldxp!($ordering), " x0, x1, [x4]"),
	223	"cmp x0, x16",
	224	"ccmp x1, x17, #0, eq",
	225	"bne 1f",
	226	// STXP w(tmp2), x2, x3, [x4]
	227	concat!(stxp!($ordering), " w15, x2, x3, [x4]"),
	228	"cbnz w15, 0b",
	229	"1:",
	230	"ret",
	231	have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
	232	}
	233	}
	234	}
	235	};
	236	}
	237
	238	/// See <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.swap>.
	239	macro_rules! swap {
	240	($ordering:ident, $bytes:tt, $name:ident) => {
	241	intrinsics! {
	242	#[maybe_use_optimized_c_shim]
	243	#[unsafe(naked)]
	244	pub unsafe extern "C" fn $name (
	245	left: int_ty!($bytes), right_ptr: *mut int_ty!($bytes)
	246	) -> int_ty!($bytes) {
	247	core::arch::naked_asm! {
	248	// SWP s(0), s(0), [x1]; if LSE supported.
	249	try_lse_op!("swp", $ordering, $bytes, 0, 0, [x1]),
	250	// mov s(tmp0), s(0)
	251	concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
	252	"0:",
	253	// LDXR s(0), [x1]
	254	concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"),
	255	// STXR w(tmp1), s(tmp0), [x1]
	256	concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"),
	257	"cbnz w17, 0b",
	258	"ret",
	259	have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
	260	}
	261	}
	262	}
	263	};
	264	}
	265
	266	/// See (e.g.) <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.fetch_add>.
	267	macro_rules! fetch_op {
	268	($ordering:ident, $bytes:tt, $name:ident, $op:literal, $lse_op:literal) => {
	269	intrinsics! {
	270	#[maybe_use_optimized_c_shim]
	271	#[unsafe(naked)]
	272	pub unsafe extern "C" fn $name (
	273	val: int_ty!($bytes), ptr: *mut int_ty!($bytes)
	274	) -> int_ty!($bytes) {
	275	core::arch::naked_asm! {
	276	// LSEOP s(0), s(0), [x1]; if LSE supported.
	277	try_lse_op!($lse_op, $ordering, $bytes, 0, 0, [x1]),
	278	// mov s(tmp0), s(0)
	279	concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
	280	"0:",
	281	// LDXR s(0), [x1]
	282	concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"),
	283	// OP s(tmp1), s(0), s(tmp0)
	284	concat!($op, " ", reg!($bytes, 17), ", ", reg!($bytes, 0), ", ", reg!($bytes, 16)),
	285	// STXR w(tmp2), s(tmp1), [x1]
	286	concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"),
	287	"cbnz w15, 0b",
	288	"ret",
	289	have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
	290	}
	291	}
	292	}
	293	}
	294	}
	295
	296	// We need a single macro to pass to `foreach_ldadd`.
	297	macro_rules! add {
	298	($ordering:ident, $bytes:tt, $name:ident) => {
	299	fetch_op! { $ordering, $bytes, $name, "add", "ldadd" }
	300	};
	301	}
	302
	303	macro_rules! and {
	304	($ordering:ident, $bytes:tt, $name:ident) => {
	305	fetch_op! { $ordering, $bytes, $name, "bic", "ldclr" }
	306	};
	307	}
	308
	309	macro_rules! xor {
	310	($ordering:ident, $bytes:tt, $name:ident) => {
	311	fetch_op! { $ordering, $bytes, $name, "eor", "ldeor" }
	312	};
	313	}
	314
	315	macro_rules! or {
	316	($ordering:ident, $bytes:tt, $name:ident) => {
	317	fetch_op! { $ordering, $bytes, $name, "orr", "ldset" }
	318	};
	319	}
	320
	321	#[macro_export]
	322	macro_rules! foreach_ordering {
	323	($macro:path, $bytes:tt, $name:ident) => {
	324	$macro!( Relaxed, $bytes, ${concat($name, _relax)} );
	325	$macro!( Acquire, $bytes, ${concat($name, _acq)} );
	326	$macro!( Release, $bytes, ${concat($name, _rel)} );
	327	$macro!( AcqRel, $bytes, ${concat($name, _acq_rel)} );
	328	};
	329	($macro:path, $name:ident) => {
	330	$macro!( Relaxed, ${concat($name, _relax)} );
	331	$macro!( Acquire, ${concat($name, _acq)} );
	332	$macro!( Release, ${concat($name, _rel)} );
	333	$macro!( AcqRel, ${concat($name, _acq_rel)} );
	334	};
	335	}
	336
	337	#[macro_export]
	338	macro_rules! foreach_bytes {
	339	($macro:path, $name:ident) => {
	340	foreach_ordering!( $macro, 1, ${concat(__aarch64_, $name, "1")} );
	341	foreach_ordering!( $macro, 2, ${concat(__aarch64_, $name, "2")} );
	342	foreach_ordering!( $macro, 4, ${concat(__aarch64_, $name, "4")} );
	343	foreach_ordering!( $macro, 8, ${concat(__aarch64_, $name, "8")} );
	344	};
	345	}
	346
	347	/// Generate different macros for cas/swp/add/clr/eor/set so that we can test them separately.
	348	#[macro_export]
	349	macro_rules! foreach_cas {
	350	($macro:path) => {
	351	foreach_bytes!($macro, cas);
	352	};
	353	}
	354
	355	/// Only CAS supports 16 bytes, and it has a different implementation that uses a different macro.
	356	#[macro_export]
	357	macro_rules! foreach_cas16 {
	358	($macro:path) => {
	359	foreach_ordering!($macro, __aarch64_cas16);
	360	};
	361	}
	362	#[macro_export]
	363	macro_rules! foreach_swp {
	364	($macro:path) => {
	365	foreach_bytes!($macro, swp);
	366	};
	367	}
	368	#[macro_export]
	369	macro_rules! foreach_ldadd {
	370	($macro:path) => {
	371	foreach_bytes!($macro, ldadd);
	372	};
	373	}
	374	#[macro_export]
	375	macro_rules! foreach_ldclr {
	376	($macro:path) => {
	377	foreach_bytes!($macro, ldclr);
	378	};
	379	}
	380	#[macro_export]
	381	macro_rules! foreach_ldeor {
	382	($macro:path) => {
	383	foreach_bytes!($macro, ldeor);
	384	};
	385	}
	386	#[macro_export]
	387	macro_rules! foreach_ldset {
	388	($macro:path) => {
	389	foreach_bytes!($macro, ldset);
	390	};
	391	}
	392
	393	foreach_cas!(compare_and_swap);
	394	foreach_cas16!(compare_and_swap_i128);
	395	foreach_swp!(swap);
	396	foreach_ldadd!(add);
	397	foreach_ldclr!(and);
	398	foreach_ldeor!(xor);
	399	foreach_ldset!(or);

library/compiler-builtins/compiler-builtins/src/lib.rs+2-2

...	...	@@ -55,8 +55,8 @@ pub mod arm;
55	55	#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
56	56	pub mod aarch64;
57	57
58		#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
59		pub mod aarch64_linux;
	58	#[cfg(all(target_arch = "aarch64", target_feature = "outline-atomics"))]
	59	pub mod aarch64_outline_atomics;
60	60
61	61	#[cfg(all(
62	62	kernel_user_helpers,

library/proc_macro/src/lib.rs+14-13

...	...	@@ -1594,11 +1594,17 @@ impl fmt::Debug for Literal {
1594	1594	}
1595	1595	}
1596	1596
1597		/// Tracked access to environment variables.
1598		#[unstable(feature = "proc_macro_tracked_env", issue = "99515")]
1599		pub mod tracked_env {
	1597	#[unstable(
	1598	feature = "proc_macro_tracked_path",
	1599	issue = "99515",
	1600	implied_by = "proc_macro_tracked_env"
	1601	)]
	1602	/// Functionality for adding environment state to the build dependency info.
	1603	pub mod tracked {
	1604
1600	1605	use std::env::{self, VarError};
1601	1606	use std::ffi::OsStr;
	1607	use std::path::Path;
1602	1608
1603	1609	/// Retrieve an environment variable and add it to build dependency info.
1604	1610	/// The build system executing the compiler will know that the variable was accessed during
...	...	@@ -1606,25 +1612,20 @@ pub mod tracked_env {
1606	1612	/// Besides the dependency tracking this function should be equivalent to `env::var` from the
1607	1613	/// standard library, except that the argument must be UTF-8.
1608	1614	#[unstable(feature = "proc_macro_tracked_env", issue = "99515")]
1609		pub fn var<K: AsRef<OsStr> + AsRef<str>>(key: K) -> Result<String, VarError> {
	1615	pub fn env_var<K: AsRef<OsStr> + AsRef<str>>(key: K) -> Result<String, VarError> {
1610	1616	let key: &str = key.as_ref();
1611	1617	let value = crate::bridge::client::FreeFunctions::injected_env_var(key)
1612	1618	.map_or_else(\|\| env::var(key), Ok);
1613	1619	crate::bridge::client::FreeFunctions::track_env_var(key, value.as_deref().ok());
1614	1620	value
1615	1621	}
1616		}
1617
1618		/// Tracked access to additional files.
1619		#[unstable(feature = "track_path", issue = "99515")]
1620		pub mod tracked_path {
1621	1622
1622		/// Track a file explicitly.
	1623	/// Track a file or directory explicitly.
1623	1624	///
1624	1625	/// Commonly used for tracking asset preprocessing.
1625		#[unstable(feature = "track_path", issue = "99515")]
1626		pub fn path<P: AsRef<str>>(path: P) {
1627		let path: &str = path.as_ref();
	1626	#[unstable(feature = "proc_macro_tracked_path", issue = "99515")]
	1627	pub fn path<P: AsRef<Path>>(path: P) {
	1628	let path: &str = path.as_ref().to_str().unwrap();
1628	1629	crate::bridge::client::FreeFunctions::track_path(path);
1629	1630	}
1630	1631	}

library/std/src/sys/configure_builtins.rs+41-5

...	...	@@ -1,13 +1,49 @@
1		/// Hook into .init_array to enable LSE atomic operations at startup, if
2		/// supported.
3		#[cfg(all(target_arch = "aarch64", target_os = "linux", not(feature = "compiler-builtins-c")))]
	1	/// Enable LSE atomic operations at startup, if supported.
	2	///
	3	/// Linker sections are based on what [`ctor`] does, with priorities to run slightly before user
	4	/// code:
	5	///
	6	/// - Apple uses the section `__mod_init_func`, `mod_init_funcs` is needed to set
	7	/// `S_MOD_INIT_FUNC_POINTERS`. There doesn't seem to be a way to indicate priorities.
	8	/// - Windows uses `.CRT$XCT`, which is run before user constructors (these should use `.CRT$XCU`).
	9	/// - ELF uses `.init_array` with a priority of 90, which runs before our `ARGV_INIT_ARRAY`
	10	/// initializer (priority 99). Both are within the 0-100 implementation-reserved range, per docs
	11	/// for the [`prio-ctor-dtor`] warning, and this matches compiler-rt's `CONSTRUCTOR_PRIORITY`.
	12	///
	13	/// To save startup time, the initializer is only run if outline atomic routines from
	14	/// compiler-builtins may be used. If LSE is known to be available then the calls are never
	15	/// emitted, and if we build the C intrinsics then it has its own initializer using the symbol
	16	/// `__aarch64_have_lse_atomics`.
	17	///
	18	/// Initialization is done in a global constructor to so we get the same behavior regardless of
	19	/// whether Rust's `init` is used, or if we are in a `dylib` or `no_main` situation (as opposed
	20	/// to doing it as part of pre-main startup). This also matches C implementations.
	21	///
	22	/// Ideally `core` would have something similar, but detecting the CPU features requires the
	23	/// auxiliary vector from the OS. We do the initialization in `std` rather than as part of
	24	/// `compiler-builtins` because a builtins->std dependency isn't possible, and inlining parts of
	25	/// `std-detect` would be much messier.
	26	///
	27	/// [`ctor`]: https://github.com/mmastrac/rust-ctor/blob/63382b833ddcbfb8b064f4e86bfa1ed4026ff356/shared/src/macros/mod.rs#L522-L534
	28	/// [`prio-ctor-dtor`]: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html
	29	#[cfg(all(
	30	target_arch = "aarch64",
	31	target_feature = "outline-atomics",
	32	not(target_feature = "lse"),
	33	not(feature = "compiler-builtins-c"),
	34	))]
4	35	#[used]
5		#[unsafe(link_section = ".init_array.90")]
	36	#[cfg_attr(target_vendor = "apple", unsafe(link_section = "__DATA,__mod_init_func,mod_init_funcs"))]
	37	#[cfg_attr(target_os = "windows", unsafe(link_section = ".CRT$XCT"))]
	38	#[cfg_attr(
	39	not(any(target_vendor = "apple", target_os = "windows")),
	40	unsafe(link_section = ".init_array.90")
	41	)]
6	42	static RUST_LSE_INIT: extern "C" fn() = {
7	43	extern "C" fn init_lse() {
8	44	use crate::arch;
9	45
10		// This is provided by compiler-builtins::aarch64_linux.
	46	// This is provided by compiler-builtins::aarch64_outline_atomics.
11	47	unsafe extern "C" {
12	48	fn __rust_enable_lse();
13	49	}

tests/run-make/env-dep-info/macro_def.rs+2-2

...	...	@@ -6,7 +6,7 @@ use proc_macro::*;
6	6
7	7	#[proc_macro]
8	8	pub fn access_env_vars(_: TokenStream) -> TokenStream {
9		let _ = tracked_env::var("EXISTING_PROC_MACRO_ENV");
10		let _ = tracked_env::var("NONEXISTENT_PROC_MACEO_ENV");
	9	let _ = tracked::env_var("EXISTING_PROC_MACRO_ENV");
	10	let _ = tracked::env_var("NONEXISTENT_PROC_MACEO_ENV");
11	11	TokenStream::new()
12	12	}

tests/run-make/track-path-dep-info/macro_def.rs+2-2

...	...	@@ -1,4 +1,4 @@
1		#![feature(track_path)]
	1	#![feature(proc_macro_tracked_path)]
2	2	#![crate_type = "proc-macro"]
3	3
4	4	extern crate proc_macro;
...	...	@@ -6,6 +6,6 @@ use proc_macro::*;
6	6
7	7	#[proc_macro]
8	8	pub fn access_tracked_paths(_: TokenStream) -> TokenStream {
9		tracked_path::path("emojis.txt");
	9	tracked::path("emojis.txt");
10	10	TokenStream::new()
11	11	}

tests/ui/attributes/malformed-fn-align.rs+1-1

...	...	@@ -26,7 +26,7 @@ fn f3() {}
26	26	#[repr(align(16))] //~ ERROR `#[repr(align(...))]` is not supported on functions
27	27	fn f4() {}
28	28
29		#[rustc_align(-1)] //~ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `-`
	29	#[rustc_align(-1)] //~ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
30	30	fn f5() {}
31	31
32	32	#[rustc_align(3)] //~ ERROR invalid alignment value: not a power of two

tests/ui/attributes/malformed-fn-align.stderr+2-2

...	...	@@ -37,11 +37,11 @@ error[E0589]: invalid alignment value: not a power of two
37	37	LL \| #[rustc_align(0)]
38	38	\| ^
39	39
40		error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `-`
	40	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
41	41	--> $DIR/malformed-fn-align.rs:29:15
42	42	\|
43	43	LL \| #[rustc_align(-1)]
44		\| ^
	44	\| ^^ expressions are not allowed here
45	45	\|
46	46	help: negative numbers are not literals, try removing the `-` sign
47	47	\|

tests/ui/deprecation/issue-66340-deprecated-attr-non-meta-grammar.rs+1-1

...	...	@@ -7,5 +7,5 @@ fn main() {
7	7	}
8	8
9	9	#[deprecated(note = test)]
10		//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `test`
	10	//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
11	11	fn foo() {}

tests/ui/deprecation/issue-66340-deprecated-attr-non-meta-grammar.stderr+2-2

...	...	@@ -1,8 +1,8 @@
1		error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `test`
	1	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
2	2	--> $DIR/issue-66340-deprecated-attr-non-meta-grammar.rs:9:21
3	3	\|
4	4	LL \| #[deprecated(note = test)]
5		\| ^^^^
	5	\| ^^^^ expressions are not allowed here
6	6	\|
7	7	help: surround the identifier with quotation marks to make it into a string literal
8	8	\|

tests/ui/macros/cfg_select.rs+1-1

...	...	@@ -68,7 +68,7 @@ cfg_select! {
68	68
69	69	cfg_select! {
70	70	() => {}
71		//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `(`
	71	//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
72	72	}
73	73
74	74	cfg_select! {

tests/ui/macros/cfg_select.stderr+2-2

...	...	@@ -27,11 +27,11 @@ error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `=>`
27	27	LL \| => {}
28	28	\| ^^
29	29
30		error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `(`
	30	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
31	31	--> $DIR/cfg_select.rs:70:5
32	32	\|
33	33	LL \| () => {}
34		\| ^
	34	\| ^^ expressions are not allowed here
35	35
36	36	error[E0539]: malformed `cfg_select` macro input
37	37	--> $DIR/cfg_select.rs:75:5

tests/ui/parser/attribute/attr-bad-meta-4.rs+1-1

...	...	@@ -9,7 +9,7 @@ macro_rules! mac {
9	9	mac!(an(arbitrary token stream));
10	10
11	11	#[cfg(feature = -1)]
12		//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `-`
	12	//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
13	13	fn handler() {}
14	14
15	15	fn main() {}

tests/ui/parser/attribute/attr-bad-meta-4.stderr+2-2

...	...	@@ -1,8 +1,8 @@
1		error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `-`
	1	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
2	2	--> $DIR/attr-bad-meta-4.rs:11:17
3	3	\|
4	4	LL \| #[cfg(feature = -1)]
5		\| ^
	5	\| ^^ expressions are not allowed here
6	6	\|
7	7	help: negative numbers are not literals, try removing the `-` sign
8	8	\|

tests/ui/parser/attribute/attr-unquoted-ident.rs+13-6

...	...	@@ -4,14 +4,21 @@
4	4
5	5	fn main() {
6	6	#[cfg(key=foo)]
7		//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `foo`
8		//~\| HELP surround the identifier with quotation marks to make it into a string literal
	7	//~^ ERROR: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
	8	//~\| HELP: surround the identifier with quotation marks to make it into a string literal
	9	//~\| NOTE: expressions are not allowed here
9	10	println!();
10	11	#[cfg(key="bar")]
11	12	println!();
12	13	#[cfg(key=foo bar baz)]
13		//~^ ERROR expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `foo`
14		//~\| HELP surround the identifier with quotation marks to make it into a string literal
	14	//~^ ERROR: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
	15	//~\| HELP: surround the identifier with quotation marks to make it into a string literal
	16	//~\| NOTE: expressions are not allowed here
	17	println!();
	18	#[cfg(key=foo 1 bar 2.0 baz.)]
	19	//~^ ERROR: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
	20	//~\| HELP: surround the identifier with quotation marks to make it into a string literal
	21	//~\| NOTE: expressions are not allowed here
15	22	println!();
16	23	}
17	24
...	...	@@ -19,7 +26,7 @@ fn main() {
19	26
20	27	macro_rules! make {
21	28	($name:ident) => { #[doc(alias = $name)] pub struct S; }
22		//~^ ERROR expected unsuffixed literal, found identifier `nickname`
	29	//~^ ERROR: expected unsuffixed literal, found identifier `nickname`
23	30	}
24	31
25		make!(nickname); //~ NOTE in this expansion
	32	make!(nickname); //~ NOTE: in this expansion

tests/ui/parser/attribute/attr-unquoted-ident.stderr+18-7

...	...	@@ -1,27 +1,38 @@
1		error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `foo`
	1	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
2	2	--> $DIR/attr-unquoted-ident.rs:6:15
3	3	\|
4	4	LL \| #[cfg(key=foo)]
5		\| ^^^
	5	\| ^^^ expressions are not allowed here
6	6	\|
7	7	help: surround the identifier with quotation marks to make it into a string literal
8	8	\|
9	9	LL \| #[cfg(key="foo")]
10	10	\| + +
11	11
12		error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `foo`
13		--> $DIR/attr-unquoted-ident.rs:12:15
	12	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
	13	--> $DIR/attr-unquoted-ident.rs:13:15
14	14	\|
15	15	LL \| #[cfg(key=foo bar baz)]
16		\| ^^^
	16	\| ^^^ expressions are not allowed here
17	17	\|
18	18	help: surround the identifier with quotation marks to make it into a string literal
19	19	\|
20	20	LL \| #[cfg(key="foo bar baz")]
21	21	\| + +
22	22
	23	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
	24	--> $DIR/attr-unquoted-ident.rs:18:15
	25	\|
	26	LL \| #[cfg(key=foo 1 bar 2.0 baz.)]
	27	\| ^^^ expressions are not allowed here
	28	\|
	29	help: surround the identifier with quotation marks to make it into a string literal
	30	\|
	31	LL \| #[cfg(key="foo 1 bar 2.0 baz.")]
	32	\| + +
	33
23	34	error: expected unsuffixed literal, found identifier `nickname`
24		--> $DIR/attr-unquoted-ident.rs:21:38
	35	--> $DIR/attr-unquoted-ident.rs:28:38
25	36	\|
26	37	LL \| ($name:ident) => { #[doc(alias = $name)] pub struct S; }
27	38	\| ^^^^^
...	...	@@ -31,5 +42,5 @@ LL \| make!(nickname);
31	42	\|
32	43	= note: this error originates in the macro `make` (in Nightly builds, run with -Z macro-backtrace for more info)
33	44
34		error: aborting due to 3 previous errors
	45	error: aborting due to 4 previous errors
35	46

tests/ui/parser/macro/expr-in-attribute.rs created+9

...	...	@@ -0,0 +1,9 @@
	1	// Test for #146325.
	2	// Ensure that when we encounter an expr invocation in an attribute, we don't suggest nonsense.
	3
	4	#[deprecated(note = a!=b)]
	5	struct X;
	6	//~^^ ERROR: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
	7	//~\| NOTE: expressions are not allowed here
	8
	9	fn main() {}

tests/ui/parser/macro/expr-in-attribute.stderr created+8

...	...	@@ -0,0 +1,8 @@
	1	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
	2	--> $DIR/expr-in-attribute.rs:4:21
	3	\|
	4	LL \| #[deprecated(note = a!=b)]
	5	\| ^^^^ expressions are not allowed here
	6
	7	error: aborting due to 1 previous error
	8

tests/ui/parser/macro/macro-in-attribute.rs created+9

...	...	@@ -0,0 +1,9 @@
	1	// Test for #146325.
	2	// Ensure that when we encounter a macro invocation in an attribute, we don't suggest nonsense.
	3
	4	#[deprecated(note = concat!("a", "b"))]
	5	struct X;
	6	//~^^ ERROR: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found
	7	//~\| NOTE: macro calls are not allowed here
	8
	9	fn main() {}

tests/ui/parser/macro/macro-in-attribute.stderr created+8

...	...	@@ -0,0 +1,8 @@
	1	error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found macro call
	2	--> $DIR/macro-in-attribute.rs:4:21
	3	\|
	4	LL \| #[deprecated(note = concat!("a", "b"))]
	5	\| ^^^^^^^^^^^^^^^^^ macro calls are not allowed here
	6
	7	error: aborting due to 1 previous error
	8

tests/ui/proc-macro/auxiliary/env.rs+3-3

...	...	@@ -3,11 +3,11 @@
3	3	extern crate proc_macro;
4	4
5	5	use proc_macro::TokenStream;
6		use proc_macro::tracked_env::var;
	6	use proc_macro::tracked::env_var;
7	7
8	8	#[proc_macro]
9	9	pub fn generate_const(input: TokenStream) -> TokenStream {
10		let the_const = match var("THE_CONST") {
	10	let the_const = match env_var("THE_CONST") {
11	11	Ok(x) if x == "12" => {
12	12	"const THE_CONST: u32 = 12;"
13	13	}
...	...	@@ -15,7 +15,7 @@ pub fn generate_const(input: TokenStream) -> TokenStream {
15	15	"const THE_CONST: u32 = 0;"
16	16	}
17	17	};
18		let another = if var("ANOTHER").is_ok() {
	18	let another = if env_var("ANOTHER").is_ok() {
19	19	"const ANOTHER: u32 = 1;"
20	20	} else {
21	21	"const ANOTHER: u32 = 2;"

tests/ui/uninhabited/uninhabited-unreachable-warning-149571.rs created+10

...	...	@@ -0,0 +1,10 @@
	1	#![deny(unreachable_code)]
	2	//@ run-pass
	3
	4	use std::convert::Infallible;
	5
	6	pub fn foo(f: impl FnOnce() -> Infallible) -> Infallible {
	7	f()
	8	}
	9
	10	fn main() {}

author	bors <bors@rust-lang.org> 2025-12-09 22:05:52 UTC
committer	bors <bors@rust-lang.org> 2025-12-09 22:05:52 UTC
log	5f1173bb2b0a7012640bd5383c61b433b16a452d
tree	a5d452fec259462e17881bb74cd442ad0cc0e284
parent	c61a3a44d1a5bee35914cada6c788a05e0808f5b
parent	a00807ed93ddeeabf3e82f7a613e2849f24a944a

Auto merge of #149818 - matthiaskrgr:rollup-uw85yss, r=matthiaskrgr