Commit a6043039ad3aef48e08d72a3e32545accdee427a | rust

author	bors <bors@rust-lang.org> 2024-08-03 08:07:00 UTC
committer	bors <bors@rust-lang.org> 2024-08-03 08:07:00 UTC
log	a6043039ad3aef48e08d72a3e32545accdee427a
tree	3e2591cc4eeadef4926c2b13397cd2c3475d01b8
parent	eefd2eac52f50176a761d20f16dcf6434ecc8831
parent	290a260721591338e7acbd207c0c96283b5ec0fb

Auto merge of #128356 - Oneirical:real-estate-reaLTOr, r=jieyouxu

Migrate `cross-lang-lto-clang` and `cross-lang-lto-pgo-smoketest` `run-make` tests to rmake

Part of #121876 and the associated [Google Summer of Code project](https://blog.rust-lang.org/2024/05/01/gsoc-2024-selected-projects.html).

This has the same problem outlined by #126180, where the tests do not actually run as no test-running CI enviroment has `RUSTBUILD_FORCE_CLANG_BASED_TESTS` set.

However, I still find it interesting to turn the Makefiles into the rmake format until the Clang issue is fixed.

This should technically be tested on MSVC... if MSVC actually ran Clang tests.

try-job: x86_64-gnu-debug

17 files changed, 244 insertions(+), 165 deletions(-)

src/tools/run-make-support/src/external_deps/clang.rs+3-2

...	...	@@ -1,7 +1,7 @@
1	1	use std::path::Path;
2	2
3	3	use crate::command::Command;
4		use crate::{bin_name, env_var};
	4	use crate::{bin_name, cwd, env_var};
5	5
6	6	/// Construct a new `clang` invocation. `clang` is not always available for all targets.
7	7	#[track_caller]
...	...	@@ -23,7 +23,8 @@ impl Clang {
23	23	#[track_caller]
24	24	pub fn new() -> Self {
25	25	let clang = env_var("CLANG");
26		let cmd = Command::new(clang);
	26	let mut cmd = Command::new(clang);
	27	cmd.arg("-L").arg(cwd());
27	28	Self { cmd }
28	29	}
29	30

src/tools/run-make-support/src/external_deps/llvm.rs+6

...	...	@@ -246,6 +246,12 @@ impl LlvmObjdump {
246	246	self.cmd.arg(path.as_ref());
247	247	self
248	248	}
	249
	250	/// Disassemble all executable sections found in the input files.
	251	pub fn disassemble(&mut self) -> &mut Self {
	252	self.cmd.arg("-d");
	253	self
	254	}
249	255	}
250	256
251	257	impl LlvmAr {

src/tools/tidy/src/allowed_run_make_makefiles.txt-2

...	...	@@ -1,8 +1,6 @@
1	1	run-make/branch-protection-check-IBT/Makefile
2	2	run-make/cat-and-grep-sanity-check/Makefile
3	3	run-make/cdylib-dylib-linkage/Makefile
4		run-make/cross-lang-lto-clang/Makefile
5		run-make/cross-lang-lto-pgo-smoketest/Makefile
6	4	run-make/cross-lang-lto-upstream-rlibs/Makefile
7	5	run-make/dep-info-doesnt-run-much/Makefile
8	6	run-make/dep-info-spaces/Makefile

tests/run-make/cross-lang-lto-clang/Makefile deleted-25

...	...	@@ -1,25 +0,0 @@
1		# needs-force-clang-based-tests
2
3		# This test makes sure that cross-language inlining actually works by checking
4		# the generated machine code.
5
6		include ../tools.mk
7
8		all: cpp-executable rust-executable
9
10		cpp-executable:
11		$(RUSTC) -Clinker-plugin-lto=on -o $(TMPDIR)/librustlib-xlto.a -Copt-level=2 -Ccodegen-units=1 ./rustlib.rs
12		$(CLANG) -flto=thin -fuse-ld=lld -L $(TMPDIR) -lrustlib-xlto -o $(TMPDIR)/cmain ./cmain.c -O3
13		# Make sure we don't find a call instruction to the function we expect to
14		# always be inlined.
15		"$(LLVM_BIN_DIR)"/llvm-objdump -d $(TMPDIR)/cmain \| $(CGREP) -v -e "call.*rust_always_inlined"
16		# As a sanity check, make sure we do find a call instruction to a
17		# non-inlined function
18		"$(LLVM_BIN_DIR)"/llvm-objdump -d $(TMPDIR)/cmain \| $(CGREP) -e "call.*rust_never_inlined"
19
20		rust-executable:
21		$(CLANG) ./clib.c -flto=thin -c -o $(TMPDIR)/clib.o -O2
22		(cd $(TMPDIR); $(AR) crus ./libxyz.a ./clib.o)
23		$(RUSTC) -Clinker-plugin-lto=on -L$(TMPDIR) -Copt-level=2 -Clinker=$(CLANG) -Clink-arg=-fuse-ld=lld ./main.rs -o $(TMPDIR)/rsmain
24		"$(LLVM_BIN_DIR)"/llvm-objdump -d $(TMPDIR)/rsmain \| $(CGREP) -e "call.*c_never_inlined"
25		"$(LLVM_BIN_DIR)"/llvm-objdump -d $(TMPDIR)/rsmain \| $(CGREP) -v -e "call.*c_always_inlined"

tests/run-make/cross-lang-lto-clang/rmake.rs created+62

...	...	@@ -0,0 +1,62 @@
	1	// This test checks that cross-language inlining actually works by checking
	2	// the generated machine code.
	3	// See https://github.com/rust-lang/rust/pull/57514
	4
	5	//@ needs-force-clang-based-tests
	6	// NOTE(#126180): This test only runs on `x86_64-gnu-debug`, because that CI job sets
	7	// RUSTBUILD_FORCE_CLANG_BASED_TESTS and only runs tests which contain "clang" in their
	8	// name.
	9
	10	use run_make_support::{clang, env_var, llvm_ar, llvm_objdump, rustc, static_lib_name};
	11
	12	fn main() {
	13	rustc()
	14	.linker_plugin_lto("on")
	15	.output(static_lib_name("rustlib-xlto"))
	16	.opt_level("2")
	17	.codegen_units(1)
	18	.input("rustlib.rs")
	19	.run();
	20	clang()
	21	.lto("thin")
	22	.use_ld("lld")
	23	.arg("-lrustlib-xlto")
	24	.out_exe("cmain")
	25	.input("cmain.c")
	26	.arg("-O3")
	27	.run();
	28	// Make sure we don't find a call instruction to the function we expect to
	29	// always be inlined.
	30	llvm_objdump()
	31	.disassemble()
	32	.input("cmain")
	33	.run()
	34	.assert_stdout_not_contains_regex("call.*rust_always_inlined");
	35	// As a sanity check, make sure we do find a call instruction to a
	36	// non-inlined function
	37	llvm_objdump()
	38	.disassemble()
	39	.input("cmain")
	40	.run()
	41	.assert_stdout_contains_regex("call.*rust_never_inlined");
	42	clang().input("clib.c").lto("thin").arg("-c").out_exe("clib.o").arg("-O2").run();
	43	llvm_ar().obj_to_ar().output_input(static_lib_name("xyz"), "clib.o").run();
	44	rustc()
	45	.linker_plugin_lto("on")
	46	.opt_level("2")
	47	.linker(&env_var("CLANG"))
	48	.link_arg("-fuse-ld=lld")
	49	.input("main.rs")
	50	.output("rsmain")
	51	.run();
	52	llvm_objdump()
	53	.disassemble()
	54	.input("rsmain")
	55	.run()
	56	.assert_stdout_not_contains_regex("call.*c_always_inlined");
	57	llvm_objdump()
	58	.disassemble()
	59	.input("rsmain")
	60	.run()
	61	.assert_stdout_contains_regex("call.*c_never_inlined");
	62	}

tests/run-make/cross-lang-lto-pgo-smoketest-clang/clib.c created+9

...	...	@@ -0,0 +1,9 @@
	1	#include <stdint.h>
	2
	3	uint32_t c_always_inlined() {
	4	return 1234;
	5	}
	6
	7	__attribute__((noinline)) uint32_t c_never_inlined() {
	8	return 12345;
	9	}

tests/run-make/cross-lang-lto-pgo-smoketest-clang/cmain.c created+12

...	...	@@ -0,0 +1,12 @@
	1	#include <stdint.h>
	2
	3	// A trivial function defined in Rust, returning a constant value. This should
	4	// always be inlined.
	5	uint32_t rust_always_inlined();
	6
	7
	8	uint32_t rust_never_inlined();
	9
	10	int main(int argc, char** argv) {
	11	return (rust_never_inlined() + rust_always_inlined()) * 0;
	12	}

tests/run-make/cross-lang-lto-pgo-smoketest-clang/main.rs created+11

...	...	@@ -0,0 +1,11 @@
	1	#[link(name = "xyz")]
	2	extern "C" {
	3	fn c_always_inlined() -> u32;
	4	fn c_never_inlined() -> u32;
	5	}
	6
	7	fn main() {
	8	unsafe {
	9	println!("blub: {}", c_always_inlined() + c_never_inlined());
	10	}
	11	}

tests/run-make/cross-lang-lto-pgo-smoketest-clang/rmake.rs created+120

...	...	@@ -0,0 +1,120 @@
	1	// This test makes sure that cross-language inlining can be used in conjunction
	2	// with profile-guided optimization. The test only tests that the whole workflow
	3	// can be executed without anything crashing. It does not test whether PGO or
	4	// xLTO have any specific effect on the generated code.
	5	// See https://github.com/rust-lang/rust/pull/61036
	6
	7	//@ needs-force-clang-based-tests
	8	// NOTE(#126180): This test would only run on `x86_64-gnu-debug`, because that CI job sets
	9	// RUSTBUILD_FORCE_CLANG_BASED_TESTS and only runs tests which contain "clang" in their
	10	// name.
	11
	12	//@ needs-profiler-support
	13	// FIXME(Oneirical): Except that due to the reliance on llvm-profdata, this test
	14	// never runs, because `x86_64-gnu-debug` does not have the `profiler_builtins` crate.
	15
	16	//FIXME(Oneirical): There was a strange workaround for MSVC on this test
	17	// which added -C panic=abort to every RUSTC call. It was justified as follows:
	18
	19	// "LLVM doesn't support instrumenting binaries that use SEH:
	20	// https://bugs.llvm.org/show_bug.cgi?id=41279
	21	// Things work fine with -Cpanic=abort though."
	22
	23	// This isn't very pertinent, however, as the test does not get run on any
	24	// MSVC platforms.
	25
	26	use run_make_support::{
	27	clang, env_var, has_extension, has_prefix, llvm_ar, llvm_profdata, rfs, run, rustc,
	28	shallow_find_files, static_lib_name,
	29	};
	30
	31	fn main() {
	32	rustc()
	33	.linker_plugin_lto("on")
	34	.output(static_lib_name("rustlib-xlto"))
	35	.opt_level("3")
	36	.codegen_units(1)
	37	.input("rustlib.rs")
	38	.arg("-Cprofile-generate=cpp-profdata")
	39	.run();
	40	clang()
	41	.lto("thin")
	42	.arg("-fprofile-generate=cpp-profdata")
	43	.use_ld("lld")
	44	.arg("-lrustlib-xlto")
	45	.out_exe("cmain")
	46	.input("cmain.c")
	47	.arg("-O3")
	48	.run();
	49	run("cmain");
	50	// Postprocess the profiling data so it can be used by the compiler
	51	let profraw_files = shallow_find_files("cpp-profdata", \|path\| {
	52	has_prefix(path, "default") && has_extension(path, "profraw")
	53	});
	54	let profraw_file = profraw_files.get(0).unwrap();
	55	llvm_profdata().merge().output("cpp-profdata/merged.profdata").input(profraw_file).run();
	56	rustc()
	57	.linker_plugin_lto("on")
	58	.profile_use("cpp-profdata/merged.profdata")
	59	.output(static_lib_name("rustlib-xlto"))
	60	.opt_level("3")
	61	.codegen_units(1)
	62	.input("rustlib.rs")
	63	.run();
	64	clang()
	65	.lto("thin")
	66	.arg("-fprofile-use=cpp-profdata/merged.profdata")
	67	.use_ld("lld")
	68	.arg("-lrustlib-xlto")
	69	.out_exe("cmain")
	70	.input("cmain.c")
	71	.arg("-O3")
	72	.run();
	73
	74	clang()
	75	.input("clib.c")
	76	.arg("-fprofile-generate=rs-profdata")
	77	.lto("thin")
	78	.arg("-c")
	79	.out_exe("clib.o")
	80	.arg("-O3")
	81	.run();
	82	llvm_ar().obj_to_ar().output_input(static_lib_name("xyz"), "clib.o").run();
	83	rustc()
	84	.linker_plugin_lto("on")
	85	.opt_level("3")
	86	.codegen_units(1)
	87	.arg("-Cprofile-generate=rs-profdata")
	88	.linker(&env_var("CLANG"))
	89	.link_arg("-fuse-ld=lld")
	90	.input("main.rs")
	91	.output("rsmain")
	92	.run();
	93	run("rsmain");
	94	// Postprocess the profiling data so it can be used by the compiler
	95	let profraw_files = shallow_find_files("rs-profdata", \|path\| {
	96	has_prefix(path, "default") && has_extension(path, "profraw")
	97	});
	98	let profraw_file = profraw_files.get(0).unwrap();
	99	llvm_profdata().merge().output("rs-profdata/merged.profdata").input(profraw_file).run();
	100	clang()
	101	.input("clib.c")
	102	.arg("-fprofile-use=rs-profdata/merged.profdata")
	103	.arg("-c")
	104	.lto("thin")
	105	.out_exe("clib.o")
	106	.arg("-O3")
	107	.run();
	108	rfs::remove_file(static_lib_name("xyz"));
	109	llvm_ar().obj_to_ar().output_input(static_lib_name("xyz"), "clib.o").run();
	110	rustc()
	111	.linker_plugin_lto("on")
	112	.opt_level("3")
	113	.codegen_units(1)
	114	.arg("-Cprofile-use=rs-profdata/merged.profdata")
	115	.linker(&env_var("CLANG"))
	116	.link_arg("-fuse-ld=lld")
	117	.input("main.rs")
	118	.output("rsmain")
	119	.run();
	120	}

tests/run-make/cross-lang-lto-pgo-smoketest-clang/rustlib.rs created+12

...	...	@@ -0,0 +1,12 @@
	1	#![crate_type = "staticlib"]
	2
	3	#[no_mangle]
	4	pub extern "C" fn rust_always_inlined() -> u32 {
	5	42
	6	}
	7
	8	#[no_mangle]
	9	#[inline(never)]
	10	pub extern "C" fn rust_never_inlined() -> u32 {
	11	421
	12	}

tests/run-make/cross-lang-lto-pgo-smoketest/Makefile deleted-90

...	...	@@ -1,90 +0,0 @@
1		# needs-force-clang-based-tests
2
3		# FIXME(#126180): This test doesn't actually run anywhere, because the only
4		# CI job that sets RUSTBUILD_FORCE_CLANG_BASED_TESTS runs very few tests.
5
6		# This test makes sure that cross-language inlining can be used in conjunction
7		# with profile-guided optimization. The test only tests that the whole workflow
8		# can be executed without anything crashing. It does not test whether PGO or
9		# xLTO have any specific effect on the generated code.
10
11		include ../tools.mk
12
13		COMMON_FLAGS=-Copt-level=3 -Ccodegen-units=1
14
15		# LLVM doesn't support instrumenting binaries that use SEH:
16		# https://bugs.llvm.org/show_bug.cgi?id=41279
17		#
18		# Things work fine with -Cpanic=abort though.
19		ifdef IS_MSVC
20		COMMON_FLAGS+= -Cpanic=abort
21		endif
22
23		all: cpp-executable rust-executable
24
25		cpp-executable:
26		$(RUSTC) -Clinker-plugin-lto=on \
27		-Cprofile-generate="$(TMPDIR)"/cpp-profdata \
28		-o "$(TMPDIR)"/librustlib-xlto.a \
29		$(COMMON_FLAGS) \
30		./rustlib.rs
31		$(CLANG) -flto=thin \
32		-fprofile-generate="$(TMPDIR)"/cpp-profdata \
33		-fuse-ld=lld \
34		-L "$(TMPDIR)" \
35		-lrustlib-xlto \
36		-o "$(TMPDIR)"/cmain \
37		-O3 \
38		./cmain.c
39		$(TMPDIR)/cmain
40		# Postprocess the profiling data so it can be used by the compiler
41		"$(LLVM_BIN_DIR)"/llvm-profdata merge \
42		-o "$(TMPDIR)"/cpp-profdata/merged.profdata \
43		"$(TMPDIR)"/cpp-profdata/default_*.profraw
44		$(RUSTC) -Clinker-plugin-lto=on \
45		-Cprofile-use="$(TMPDIR)"/cpp-profdata/merged.profdata \
46		-o "$(TMPDIR)"/librustlib-xlto.a \
47		$(COMMON_FLAGS) \
48		./rustlib.rs
49		$(CLANG) -flto=thin \
50		-fprofile-use="$(TMPDIR)"/cpp-profdata/merged.profdata \
51		-fuse-ld=lld \
52		-L "$(TMPDIR)" \
53		-lrustlib-xlto \
54		-o "$(TMPDIR)"/cmain \
55		-O3 \
56		./cmain.c
57
58		rust-executable:
59		exit
60		$(CLANG) ./clib.c -fprofile-generate="$(TMPDIR)"/rs-profdata -flto=thin -c -o $(TMPDIR)/clib.o -O3
61		(cd $(TMPDIR); $(AR) crus ./libxyz.a ./clib.o)
62		$(RUSTC) -Clinker-plugin-lto=on \
63		-Cprofile-generate="$(TMPDIR)"/rs-profdata \
64		-L$(TMPDIR) \
65		$(COMMON_FLAGS) \
66		-Clinker=$(CLANG) \
67		-Clink-arg=-fuse-ld=lld \
68		-o $(TMPDIR)/rsmain \
69		./main.rs
70		$(TMPDIR)/rsmain
71		# Postprocess the profiling data so it can be used by the compiler
72		"$(LLVM_BIN_DIR)"/llvm-profdata merge \
73		-o "$(TMPDIR)"/rs-profdata/merged.profdata \
74		"$(TMPDIR)"/rs-profdata/default_*.profraw
75		$(CLANG) ./clib.c \
76		-fprofile-use="$(TMPDIR)"/rs-profdata/merged.profdata \
77		-flto=thin \
78		-c \
79		-o $(TMPDIR)/clib.o \
80		-O3
81		rm "$(TMPDIR)"/libxyz.a
82		(cd $(TMPDIR); $(AR) crus ./libxyz.a ./clib.o)
83		$(RUSTC) -Clinker-plugin-lto=on \
84		-Cprofile-use="$(TMPDIR)"/rs-profdata/merged.profdata \
85		-L$(TMPDIR) \
86		$(COMMON_FLAGS) \
87		-Clinker=$(CLANG) \
88		-Clink-arg=-fuse-ld=lld \
89		-o $(TMPDIR)/rsmain \
90		./main.rs

tests/run-make/cross-lang-lto-pgo-smoketest/clib.c deleted-9

...	...	@@ -1,9 +0,0 @@
1		#include <stdint.h>
2
3		uint32_t c_always_inlined() {
4		return 1234;
5		}
6
7		__attribute__((noinline)) uint32_t c_never_inlined() {
8		return 12345;
9		}

tests/run-make/cross-lang-lto-pgo-smoketest/cmain.c deleted-12

...	...	@@ -1,12 +0,0 @@
1		#include <stdint.h>
2
3		// A trivial function defined in Rust, returning a constant value. This should
4		// always be inlined.
5		uint32_t rust_always_inlined();
6
7
8		uint32_t rust_never_inlined();
9
10		int main(int argc, char** argv) {
11		return (rust_never_inlined() + rust_always_inlined()) * 0;
12		}

tests/run-make/cross-lang-lto-pgo-smoketest/main.rs deleted-11

...	...	@@ -1,11 +0,0 @@
1		#[link(name = "xyz")]
2		extern "C" {
3		fn c_always_inlined() -> u32;
4		fn c_never_inlined() -> u32;
5		}
6
7		fn main() {
8		unsafe {
9		println!("blub: {}", c_always_inlined() + c_never_inlined());
10		}
11		}

tests/run-make/cross-lang-lto-pgo-smoketest/rustlib.rs deleted-12

...	...	@@ -1,12 +0,0 @@
1		#![crate_type = "staticlib"]
2
3		#[no_mangle]
4		pub extern "C" fn rust_always_inlined() -> u32 {
5		42
6		}
7
8		#[no_mangle]
9		#[inline(never)]
10		pub extern "C" fn rust_never_inlined() -> u32 {
11		421
12		}

tests/run-make/cross-lang-lto-riscv-abi/rmake.rs+5-2

...	...	@@ -3,8 +3,11 @@
3	3	//@ needs-force-clang-based-tests
4	4	//@ needs-llvm-components riscv
5	5
6		// FIXME(#126180): This test doesn't actually run anywhere, because the only
7		// CI job that sets RUSTBUILD_FORCE_CLANG_BASED_TESTS runs very few tests.
	6	//@ needs-force-clang-based-tests
	7	// FIXME(#126180): This test can only run on `x86_64-gnu-debug`, because that CI job sets
	8	// RUSTBUILD_FORCE_CLANG_BASED_TESTS and only runs tests which contain "clang" in their
	9	// name.
	10	// However, this test does not run at all as its name does not contain "clang".
8	11
9	12	use std::path::PathBuf;
10	13	use std::process::{Command, Output};

tests/run-make/wasm-override-linker/rmake.rs+4

...	...	@@ -2,6 +2,10 @@
2	2	// $ RUSTBUILD_FORCE_CLANG_BASED_TESTS=1 ./x.py test tests/run-make/wasm-override-linker/
3	3
4	4	//@ needs-force-clang-based-tests
	5	// FIXME(#126180): This test can only run on `x86_64-gnu-debug`, because that CI job sets
	6	// RUSTBUILD_FORCE_CLANG_BASED_TESTS and only runs tests which contain "clang" in their
	7	// name.
	8	// However, this test does not run at all as its name does not contain "clang".
5	9
6	10	use run_make_support::{env_var, rustc, target};
7	11