authorbors <bors@rust-lang.org> 2025-03-01 08:22:18 UTC
committerbors <bors@rust-lang.org> 2025-03-01 08:22:18 UTC
log0c72c0d11adeba449886089c6bd5d48363f7a2cd
tree53caa967ed8e67b4eb3f41714c99a9ee76d7a043
parent002da76821d32c8807dc47da16660925d8cc9b62
parenta897cc03519c83e1c426be491c9a1bea63609e16

Auto merge of #133250 - DianQK:embed-bitcode-pgo, r=nikic

The embedded bitcode should always be prepared for LTO/ThinLTO Fixes #115344. Fixes #117220. There are currently two methods for generating bitcode that used for LTO. One method involves using `-C linker-plugin-lto` to emit object files as bitcode, which is the typical setting used by cargo. The other method is through `-C embed-bitcode=yes`. When using with `-C embed-bitcode=yes -C lto=no`, we run a complete non-LTO LLVM pipeline to obtain bitcode, then the bitcode is used for LTO. We run the Call Graph Profile Pass twice on the same module. This PR is doing something similar to LLVM's `buildFatLTODefaultPipeline`, obtaining the bitcode for embedding after running `buildThinLTOPreLinkDefaultPipeline`. r? nikic

20 files changed, 294 insertions(+), 101 deletions(-)

compiler/rustc_codegen_gcc/src/back/lto.rs+4-5
......@@ -632,17 +632,16 @@ pub unsafe fn optimize_thin_module(
632632 Arc::new(SyncContext::new(context))
633633 }
634634 };
635 let module = ModuleCodegen {
636 module_llvm: GccContext {
635 let module = ModuleCodegen::new_regular(
636 thin_module.name().to_string(),
637 GccContext {
637638 context,
638639 should_combine_object_files,
639640 // TODO(antoyo): use the correct relocation model here.
640641 relocation_model: RelocModel::Pic,
641642 temp_dir: None,
642643 },
643 name: thin_module.name().to_string(),
644 kind: ModuleKind::Regular,
645 };
644 );
646645 /*{
647646 let target = &*module.module_llvm.tm;
648647 let llmod = module.module_llvm.llmod();
compiler/rustc_codegen_gcc/src/base.rs+5-6
......@@ -4,10 +4,10 @@ use std::sync::Arc;
44use std::time::Instant;
55
66use gccjit::{CType, Context, FunctionType, GlobalKind};
7use rustc_codegen_ssa::ModuleCodegen;
78use rustc_codegen_ssa::base::maybe_create_entry_wrapper;
89use rustc_codegen_ssa::mono_item::MonoItemExt;
910use rustc_codegen_ssa::traits::DebugInfoCodegenMethods;
10use rustc_codegen_ssa::{ModuleCodegen, ModuleKind};
1111use rustc_middle::dep_graph;
1212use rustc_middle::mir::mono::Linkage;
1313#[cfg(feature = "master")]
......@@ -237,16 +237,15 @@ pub fn compile_codegen_unit(
237237 }
238238 }
239239
240 ModuleCodegen {
241 name: cgu_name.to_string(),
242 module_llvm: GccContext {
240 ModuleCodegen::new_regular(
241 cgu_name.to_string(),
242 GccContext {
243243 context: Arc::new(SyncContext::new(context)),
244244 relocation_model: tcx.sess.relocation_model(),
245245 should_combine_object_files: false,
246246 temp_dir: None,
247247 },
248 kind: ModuleKind::Regular,
249 }
248 )
250249 }
251250
252251 (module, cost)
compiler/rustc_codegen_gcc/src/lib.rs+1-1
......@@ -393,7 +393,7 @@ impl WriteBackendMethods for GccCodegenBackend {
393393 unsafe fn optimize(
394394 _cgcx: &CodegenContext<Self>,
395395 _dcx: DiagCtxtHandle<'_>,
396 module: &ModuleCodegen<Self::Module>,
396 module: &mut ModuleCodegen<Self::Module>,
397397 config: &ModuleConfig,
398398 ) -> Result<(), FatalError> {
399399 module.module_llvm.context.set_optimization_level(to_gcc_opt_level(config.opt_level));
compiler/rustc_codegen_llvm/src/back/lto.rs+15-12
......@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
22use std::ffi::{CStr, CString};
33use std::fs::File;
44use std::path::Path;
5use std::ptr::NonNull;
56use std::sync::Arc;
67use std::{io, iter, slice};
78
......@@ -305,11 +306,8 @@ fn fat_lto(
305306 assert!(!serialized_modules.is_empty(), "must have at least one serialized module");
306307 let (buffer, name) = serialized_modules.remove(0);
307308 info!("no in-memory regular modules to choose from, parsing {:?}", name);
308 ModuleCodegen {
309 module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?,
310 name: name.into_string().unwrap(),
311 kind: ModuleKind::Regular,
312 }
309 let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?;
310 ModuleCodegen::new_regular(name.into_string().unwrap(), llvm_module)
313311 }
314312 };
315313 {
......@@ -655,14 +653,14 @@ pub(crate) fn run_pass_manager(
655653 }
656654
657655 unsafe {
658 write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?;
656 write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?;
659657 }
660658
661659 if cfg!(llvm_enzyme) && enable_ad {
662660 let opt_stage = llvm::OptStage::FatLTO;
663661 let stage = write::AutodiffStage::PostAD;
664662 unsafe {
665 write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?;
663 write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?;
666664 }
667665
668666 // This is the final IR, so people should be able to inspect the optimized autodiff output.
......@@ -729,6 +727,11 @@ impl ThinBuffer {
729727 ThinBuffer(buffer)
730728 }
731729 }
730
731 pub unsafe fn from_raw_ptr(ptr: *mut llvm::ThinLTOBuffer) -> ThinBuffer {
732 let mut ptr = NonNull::new(ptr).unwrap();
733 ThinBuffer(unsafe { ptr.as_mut() })
734 }
732735}
733736
734737impl ThinBufferMethods for ThinBuffer {
......@@ -772,11 +775,11 @@ pub(crate) unsafe fn optimize_thin_module(
772775 // crates but for locally codegened modules we may be able to reuse
773776 // that LLVM Context and Module.
774777 let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx)?;
775 let mut module = ModuleCodegen {
776 module_llvm,
777 name: thin_module.name().to_string(),
778 kind: ModuleKind::Regular,
779 };
778 let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm);
779 // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here.
780 if cgcx.config(ModuleKind::Regular).embed_bitcode() {
781 module.thin_lto_buffer = Some(thin_module.data().to_vec());
782 }
780783 {
781784 let target = &*module.module_llvm.tm;
782785 let llmod = module.module_llvm.llmod();
compiler/rustc_codegen_llvm/src/back/write.rs+83-43
......@@ -1,6 +1,7 @@
11use std::ffi::{CStr, CString};
22use std::io::{self, Write};
33use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
45use std::sync::Arc;
56use std::{fs, slice, str};
67
......@@ -15,7 +16,7 @@ use rustc_codegen_ssa::back::write::{
1516 TargetMachineFactoryFn,
1617};
1718use rustc_codegen_ssa::traits::*;
18use rustc_codegen_ssa::{CompiledModule, ModuleCodegen};
19use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
1920use rustc_data_structures::profiling::SelfProfilerRef;
2021use rustc_data_structures::small_c_str::SmallCStr;
2122use rustc_errors::{DiagCtxtHandle, FatalError, Level};
......@@ -551,6 +552,7 @@ pub(crate) unsafe fn llvm_optimize(
551552 cgcx: &CodegenContext<LlvmCodegenBackend>,
552553 dcx: DiagCtxtHandle<'_>,
553554 module: &ModuleCodegen<ModuleLlvm>,
555 thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
554556 config: &ModuleConfig,
555557 opt_level: config::OptLevel,
556558 opt_stage: llvm::OptStage,
......@@ -584,7 +586,17 @@ pub(crate) unsafe fn llvm_optimize(
584586 vectorize_loop = config.vectorize_loop;
585587 }
586588 trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
587 let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed();
589 if thin_lto_buffer.is_some() {
590 assert!(
591 matches!(
592 opt_stage,
593 llvm::OptStage::PreLinkNoLTO
594 | llvm::OptStage::PreLinkFatLTO
595 | llvm::OptStage::PreLinkThinLTO
596 ),
597 "the bitcode for LTO can only be obtained at the pre-link stage"
598 );
599 }
588600 let pgo_gen_path = get_pgo_gen_path(config);
589601 let pgo_use_path = get_pgo_use_path(config);
590602 let pgo_sample_use_path = get_pgo_sample_use_path(config);
......@@ -644,7 +656,9 @@ pub(crate) unsafe fn llvm_optimize(
644656 config.no_prepopulate_passes,
645657 config.verify_llvm_ir,
646658 config.lint_llvm_ir,
647 using_thin_buffers,
659 thin_lto_buffer,
660 config.emit_thin_lto,
661 config.emit_thin_lto_summary,
648662 config.merge_functions,
649663 unroll_loops,
650664 vectorize_slp,
......@@ -675,7 +689,7 @@ pub(crate) unsafe fn llvm_optimize(
675689pub(crate) unsafe fn optimize(
676690 cgcx: &CodegenContext<LlvmCodegenBackend>,
677691 dcx: DiagCtxtHandle<'_>,
678 module: &ModuleCodegen<ModuleLlvm>,
692 module: &mut ModuleCodegen<ModuleLlvm>,
679693 config: &ModuleConfig,
680694) -> Result<(), FatalError> {
681695 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
......@@ -705,9 +719,53 @@ pub(crate) unsafe fn optimize(
705719 // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD).
706720 let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
707721 let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
708 return unsafe {
709 llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage)
722 // The embedded bitcode is used to run LTO/ThinLTO.
723 // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO.
724 // It may have undergone LTO due to ThinLocal, so we need to obtain the embedded bitcode at
725 // this point.
726 let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
727 && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
728 || config.emit_thin_lto_summary
729 {
730 Some(null_mut())
731 } else {
732 None
710733 };
734 unsafe {
735 llvm_optimize(
736 cgcx,
737 dcx,
738 module,
739 thin_lto_buffer.as_mut(),
740 config,
741 opt_level,
742 opt_stage,
743 autodiff_stage,
744 )
745 }?;
746 if let Some(thin_lto_buffer) = thin_lto_buffer {
747 let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
748 module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
749 let bc_summary_out =
750 cgcx.output_filenames.temp_path(OutputType::ThinLinkBitcode, module_name);
751 if config.emit_thin_lto_summary
752 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
753 {
754 let summary_data = thin_lto_buffer.thin_link_data();
755 cgcx.prof.artifact_size(
756 "llvm_bitcode_summary",
757 thin_link_bitcode_filename.to_string_lossy(),
758 summary_data.len() as u64,
759 );
760 let _timer = cgcx.prof.generic_activity_with_arg(
761 "LLVM_module_codegen_emit_bitcode_summary",
762 &*module.name,
763 );
764 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
765 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
766 }
767 }
768 }
711769 }
712770 Ok(())
713771}
......@@ -760,59 +818,41 @@ pub(crate) unsafe fn codegen(
760818 // otherwise requested.
761819
762820 let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name);
763 let bc_summary_out =
764 cgcx.output_filenames.temp_path(OutputType::ThinLinkBitcode, module_name);
765821 let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name);
766822
767823 if config.bitcode_needed() {
768 let _timer = cgcx
769 .prof
770 .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name);
771 let thin = ThinBuffer::new(llmod, config.emit_thin_lto, config.emit_thin_lto_summary);
772 let data = thin.data();
773
774 if let Some(bitcode_filename) = bc_out.file_name() {
775 cgcx.prof.artifact_size(
776 "llvm_bitcode",
777 bitcode_filename.to_string_lossy(),
778 data.len() as u64,
779 );
780 }
781
782 if config.emit_thin_lto_summary
783 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
784 {
785 let summary_data = thin.thin_link_data();
786 cgcx.prof.artifact_size(
787 "llvm_bitcode_summary",
788 thin_link_bitcode_filename.to_string_lossy(),
789 summary_data.len() as u64,
790 );
791
792 let _timer = cgcx.prof.generic_activity_with_arg(
793 "LLVM_module_codegen_emit_bitcode_summary",
794 &*module.name,
795 );
796 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
797 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
798 }
799 }
800
801824 if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
825 let thin = {
826 let _timer = cgcx.prof.generic_activity_with_arg(
827 "LLVM_module_codegen_make_bitcode",
828 &*module.name,
829 );
830 ThinBuffer::new(llmod, config.emit_thin_lto, false)
831 };
832 let data = thin.data();
802833 let _timer = cgcx
803834 .prof
804835 .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
836 if let Some(bitcode_filename) = bc_out.file_name() {
837 cgcx.prof.artifact_size(
838 "llvm_bitcode",
839 bitcode_filename.to_string_lossy(),
840 data.len() as u64,
841 );
842 }
805843 if let Err(err) = fs::write(&bc_out, data) {
806844 dcx.emit_err(WriteBytecode { path: &bc_out, err });
807845 }
808846 }
809847
810 if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
848 if config.embed_bitcode() && module.kind == ModuleKind::Regular {
811849 let _timer = cgcx
812850 .prof
813851 .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
852 let thin_bc =
853 module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
814854 unsafe {
815 embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data);
855 embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc);
816856 }
817857 }
818858 }
compiler/rustc_codegen_llvm/src/base.rs+2-6
......@@ -13,10 +13,10 @@
1313
1414use std::time::Instant;
1515
16use rustc_codegen_ssa::ModuleCodegen;
1617use rustc_codegen_ssa::base::maybe_create_entry_wrapper;
1718use rustc_codegen_ssa::mono_item::MonoItemExt;
1819use rustc_codegen_ssa::traits::*;
19use rustc_codegen_ssa::{ModuleCodegen, ModuleKind};
2020use rustc_data_structures::small_c_str::SmallCStr;
2121use rustc_middle::dep_graph;
2222use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
......@@ -133,11 +133,7 @@ pub(crate) fn compile_codegen_unit(
133133 }
134134 }
135135
136 ModuleCodegen {
137 name: cgu_name.to_string(),
138 module_llvm: llvm_module,
139 kind: ModuleKind::Regular,
140 }
136 ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module)
141137 }
142138
143139 (module, cost)
compiler/rustc_codegen_llvm/src/lib.rs+1-1
......@@ -194,7 +194,7 @@ impl WriteBackendMethods for LlvmCodegenBackend {
194194 unsafe fn optimize(
195195 cgcx: &CodegenContext<Self>,
196196 dcx: DiagCtxtHandle<'_>,
197 module: &ModuleCodegen<Self::Module>,
197 module: &mut ModuleCodegen<Self::Module>,
198198 config: &ModuleConfig,
199199 ) -> Result<(), FatalError> {
200200 unsafe { back::write::optimize(cgcx, dcx, module, config) }
compiler/rustc_codegen_llvm/src/llvm/ffi.rs+3-1
......@@ -2425,7 +2425,9 @@ unsafe extern "C" {
24252425 NoPrepopulatePasses: bool,
24262426 VerifyIR: bool,
24272427 LintIR: bool,
2428 UseThinLTOBuffers: bool,
2428 ThinLTOBuffer: Option<&mut *mut ThinLTOBuffer>,
2429 EmitThinLTO: bool,
2430 EmitThinLTOSummary: bool,
24292431 MergeFunctions: bool,
24302432 UnrollLoops: bool,
24312433 SLPVectorize: bool,
compiler/rustc_codegen_ssa/src/back/write.rs+6-2
......@@ -278,6 +278,10 @@ impl ModuleConfig {
278278 || self.emit_obj == EmitObj::Bitcode
279279 || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
280280 }
281
282 pub fn embed_bitcode(&self) -> bool {
283 self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
284 }
281285}
282286
283287/// Configuration passed to the function returned by the `target_machine_factory`.
......@@ -877,14 +881,14 @@ pub(crate) fn compute_per_cgu_lto_type(
877881
878882fn execute_optimize_work_item<B: ExtraBackendMethods>(
879883 cgcx: &CodegenContext<B>,
880 module: ModuleCodegen<B::Module>,
884 mut module: ModuleCodegen<B::Module>,
881885 module_config: &ModuleConfig,
882886) -> Result<WorkItemResult<B>, FatalError> {
883887 let dcx = cgcx.create_dcx();
884888 let dcx = dcx.handle();
885889
886890 unsafe {
887 B::optimize(cgcx, dcx, &module, module_config)?;
891 B::optimize(cgcx, dcx, &mut module, module_config)?;
888892 }
889893
890894 // After we've done the initial round of optimizations we need to
compiler/rustc_codegen_ssa/src/base.rs+1-1
......@@ -687,7 +687,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
687687 submit_codegened_module_to_llvm(
688688 &backend,
689689 &ongoing_codegen.coordinator.sender,
690 ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator },
690 ModuleCodegen::new_allocator(llmod_id, module_llvm),
691691 cost,
692692 );
693693 }
compiler/rustc_codegen_ssa/src/lib.rs+20
......@@ -75,9 +75,29 @@ pub struct ModuleCodegen<M> {
7575 pub name: String,
7676 pub module_llvm: M,
7777 pub kind: ModuleKind,
78 /// Saving the ThinLTO buffer for embedding in the object file.
79 pub thin_lto_buffer: Option<Vec<u8>>,
7880}
7981
8082impl<M> ModuleCodegen<M> {
83 pub fn new_regular(name: impl Into<String>, module: M) -> Self {
84 Self {
85 name: name.into(),
86 module_llvm: module,
87 kind: ModuleKind::Regular,
88 thin_lto_buffer: None,
89 }
90 }
91
92 pub fn new_allocator(name: impl Into<String>, module: M) -> Self {
93 Self {
94 name: name.into(),
95 module_llvm: module,
96 kind: ModuleKind::Allocator,
97 thin_lto_buffer: None,
98 }
99 }
100
81101 pub fn into_compiled_module(
82102 self,
83103 emit_obj: bool,
compiler/rustc_codegen_ssa/src/traits/write.rs+1-1
......@@ -40,7 +40,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
4040 unsafe fn optimize(
4141 cgcx: &CodegenContext<Self>,
4242 dcx: DiagCtxtHandle<'_>,
43 module: &ModuleCodegen<Self::Module>,
43 module: &mut ModuleCodegen<Self::Module>,
4444 config: &ModuleConfig,
4545 ) -> Result<(), FatalError>;
4646 fn optimize_fat(
compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp+50-16
......@@ -7,6 +7,7 @@
77#include "llvm/Analysis/Lint.h"
88#include "llvm/Analysis/TargetLibraryInfo.h"
99#include "llvm/Bitcode/BitcodeWriter.h"
10#include "llvm/Bitcode/BitcodeWriterPass.h"
1011#include "llvm/CodeGen/CommandFlags.h"
1112#include "llvm/IR/AssemblyAnnotationWriter.h"
1213#include "llvm/IR/AutoUpgrade.h"
......@@ -37,6 +38,7 @@
3738#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
3839#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
3940#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
41#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
4042#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
4143#include "llvm/Transforms/Utils/FunctionImportUtils.h"
4244#include "llvm/Transforms/Utils/NameAnonGlobals.h"
......@@ -195,6 +197,19 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char *FileName) {
195197GEN_SUBTARGETS
196198#undef SUBTARGET
197199
200// This struct and various functions are sort of a hack right now, but the
201// problem is that we've got in-memory LLVM modules after we generate and
202// optimize all codegen-units for one compilation in rustc. To be compatible
203// with the LTO support above we need to serialize the modules plus their
204// ThinLTO summary into memory.
205//
206// This structure is basically an owned version of a serialize module, with
207// a ThinLTO summary attached.
208struct LLVMRustThinLTOBuffer {
209 std::string data;
210 std::string thin_link_data;
211};
212
198213extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
199214 const char *Feature) {
200215 TargetMachine *Target = unwrap(TM);
......@@ -704,7 +719,8 @@ extern "C" LLVMRustResult LLVMRustOptimize(
704719 LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef,
705720 LLVMRustPassBuilderOptLevel OptLevelRust, LLVMRustOptStage OptStage,
706721 bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR,
707 bool LintIR, bool UseThinLTOBuffers, bool MergeFunctions, bool UnrollLoops,
722 bool LintIR, LLVMRustThinLTOBuffer **ThinLTOBufferRef, bool EmitThinLTO,
723 bool EmitThinLTOSummary, bool MergeFunctions, bool UnrollLoops,
708724 bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls,
709725 bool EmitLifetimeMarkers, bool RunEnzyme,
710726 LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath,
......@@ -952,7 +968,10 @@ extern "C" LLVMRustResult LLVMRustOptimize(
952968 }
953969
954970 ModulePassManager MPM;
955 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
971 bool NeedThinLTOBufferPasses = EmitThinLTO;
972 auto ThinLTOBuffer = std::make_unique<LLVMRustThinLTOBuffer>();
973 raw_string_ostream ThinLTODataOS(ThinLTOBuffer->data);
974 raw_string_ostream ThinLinkDataOS(ThinLTOBuffer->thin_link_data);
956975 if (!NoPrepopulatePasses) {
957976 // The pre-link pipelines don't support O0 and require using
958977 // buildO0DefaultPipeline() instead. At the same time, the LTO pipelines do
......@@ -976,7 +995,25 @@ extern "C" LLVMRustResult LLVMRustOptimize(
976995
977996 switch (OptStage) {
978997 case LLVMRustOptStage::PreLinkNoLTO:
979 MPM = PB.buildPerModuleDefaultPipeline(OptLevel);
998 if (ThinLTOBufferRef) {
999 // This is similar to LLVM's `buildFatLTODefaultPipeline`, where the
1000 // bitcode for embedding is obtained after performing
1001 // `ThinLTOPreLinkDefaultPipeline`.
1002 MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(OptLevel));
1003 if (EmitThinLTO) {
1004 MPM.addPass(ThinLTOBitcodeWriterPass(
1005 ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr));
1006 } else {
1007 MPM.addPass(BitcodeWriterPass(ThinLTODataOS));
1008 }
1009 *ThinLTOBufferRef = ThinLTOBuffer.release();
1010 MPM.addPass(PB.buildModuleOptimizationPipeline(
1011 OptLevel, ThinOrFullLTOPhase::None));
1012 MPM.addPass(
1013 createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));
1014 } else {
1015 MPM = PB.buildPerModuleDefaultPipeline(OptLevel);
1016 }
9801017 break;
9811018 case LLVMRustOptStage::PreLinkThinLTO:
9821019 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
......@@ -1022,6 +1059,16 @@ extern "C" LLVMRustResult LLVMRustOptimize(
10221059 MPM.addPass(CanonicalizeAliasesPass());
10231060 MPM.addPass(NameAnonGlobalPass());
10241061 }
1062 // For `-Copt-level=0`, ThinLTO, or LTO.
1063 if (ThinLTOBufferRef && *ThinLTOBufferRef == nullptr) {
1064 if (EmitThinLTO) {
1065 MPM.addPass(ThinLTOBitcodeWriterPass(
1066 ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr));
1067 } else {
1068 MPM.addPass(BitcodeWriterPass(ThinLTODataOS));
1069 }
1070 *ThinLTOBufferRef = ThinLTOBuffer.release();
1071 }
10251072
10261073 // now load "-enzyme" pass:
10271074#ifdef ENZYME
......@@ -1500,19 +1547,6 @@ extern "C" bool LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data,
15001547 return true;
15011548}
15021549
1503// This struct and various functions are sort of a hack right now, but the
1504// problem is that we've got in-memory LLVM modules after we generate and
1505// optimize all codegen-units for one compilation in rustc. To be compatible
1506// with the LTO support above we need to serialize the modules plus their
1507// ThinLTO summary into memory.
1508//
1509// This structure is basically an owned version of a serialize module, with
1510// a ThinLTO summary attached.
1511struct LLVMRustThinLTOBuffer {
1512 std::string data;
1513 std::string thin_link_data;
1514};
1515
15161550extern "C" LLVMRustThinLTOBuffer *
15171551LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin, bool emit_summary) {
15181552 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
compiler/rustc_session/src/config.rs+3
......@@ -539,7 +539,10 @@ impl FromStr for SplitDwarfKind {
539539#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord, HashStable_Generic)]
540540#[derive(Encodable, Decodable)]
541541pub enum OutputType {
542 /// This is the optimized bitcode, which could be either pre-LTO or non-LTO bitcode,
543 /// depending on the specific request type.
542544 Bitcode,
545 /// This is the summary or index data part of the ThinLTO bitcode.
543546 ThinLinkBitcode,
544547 Assembly,
545548 LlvmAssembly,
tests/codegen/overaligned-constant.rs+2-2
......@@ -9,14 +9,14 @@ struct S(i32);
99
1010struct SmallStruct(f32, Option<S>, &'static [f32]);
1111
12// CHECK: @0 = private unnamed_addr constant
12// CHECK: [[const:@.*]] = private unnamed_addr constant
1313// CHECK-SAME: , align 8
1414
1515#[no_mangle]
1616pub fn overaligned_constant() {
1717 // CHECK-LABEL: @overaligned_constant
1818 // CHECK: [[full:%_.*]] = alloca [32 x i8], align 8
19 // CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[full]], ptr align 8 @0, i64 32, i1 false)
19 // CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[full]], ptr align 8 [[const]], i64 32, i1 false)
2020 let mut s = S(1);
2121
2222 s.0 = 3;
tests/codegen/uninit-consts.rs+4-4
......@@ -11,15 +11,15 @@ pub struct PartiallyUninit {
1111 y: MaybeUninit<[u8; 10]>,
1212}
1313
14// CHECK: [[FULLY_UNINIT:@[0-9]+]] = private unnamed_addr constant <{ [10 x i8] }> undef
14// CHECK: [[FULLY_UNINIT:@.*]] = private unnamed_addr constant <{ [10 x i8] }> undef
1515
16// CHECK: [[PARTIALLY_UNINIT:@[0-9]+]] = private unnamed_addr constant <{ [4 x i8], [12 x i8] }> <{ [4 x i8] c"{{\\EF\\BE\\AD\\DE|\\DE\\AD\\BE\\EF}}", [12 x i8] undef }>, align 4
16// CHECK: [[PARTIALLY_UNINIT:@.*]] = private unnamed_addr constant <{ [4 x i8], [12 x i8] }> <{ [4 x i8] c"{{\\EF\\BE\\AD\\DE|\\DE\\AD\\BE\\EF}}", [12 x i8] undef }>, align 4
1717
1818// This shouldn't contain undef, since it contains more chunks
1919// than the default value of uninit_const_chunk_threshold.
20// CHECK: [[UNINIT_PADDING_HUGE:@[0-9]+]] = private unnamed_addr constant <{ [32768 x i8] }> <{ [32768 x i8] c"{{.+}}" }>, align 4
20// CHECK: [[UNINIT_PADDING_HUGE:@.*]] = private unnamed_addr constant <{ [32768 x i8] }> <{ [32768 x i8] c"{{.+}}" }>, align 4
2121
22// CHECK: [[FULLY_UNINIT_HUGE:@[0-9]+]] = private unnamed_addr constant <{ [16384 x i8] }> undef
22// CHECK: [[FULLY_UNINIT_HUGE:@.*]] = private unnamed_addr constant <{ [16384 x i8] }> undef
2323
2424// CHECK-LABEL: @fully_uninit
2525#[no_mangle]
tests/run-make/pgo-embed-bc-lto/interesting.rs created+16
......@@ -0,0 +1,16 @@
1#![crate_name = "interesting"]
2#![crate_type = "rlib"]
3
4extern crate opaque;
5
6#[no_mangle]
7#[inline(never)]
8pub fn function_called_once() {
9 opaque::foo();
10}
11
12// CHECK-LABEL: @function_called_once
13// CHECK-SAME: !prof [[function_called_once_id:![0-9]+]] {
14// CHECK: "CG Profile"
15// CHECK-NOT: "CG Profile"
16// CHECK-DAG: [[function_called_once_id]] = !{!"function_entry_count", i64 1}
tests/run-make/pgo-embed-bc-lto/main.rs created+5
......@@ -0,0 +1,5 @@
1extern crate interesting;
2
3fn main() {
4 interesting::function_called_once();
5}
tests/run-make/pgo-embed-bc-lto/opaque.rs created+5
......@@ -0,0 +1,5 @@
1#![crate_name = "opaque"]
2#![crate_type = "rlib"]
3
4#[inline(never)]
5pub fn foo() {}
tests/run-make/pgo-embed-bc-lto/rmake.rs created+67
......@@ -0,0 +1,67 @@
1// This test case verifies that we successfully complete an LTO build with PGO
2// using the embedded bitcode.
3// It also ensures that the generated IR correctly includes the call results.
4
5//@ needs-profiler-runtime
6//@ ignore-cross-compile
7
8use std::path::Path;
9
10use run_make_support::{
11 has_extension, has_prefix, llvm_filecheck, llvm_profdata, rfs, run, rustc, shallow_find_files,
12};
13
14fn run_test(cg_units: usize) {
15 let path_prof_data_dir = Path::new("prof_data_dir");
16 if path_prof_data_dir.exists() {
17 rfs::remove_dir_all(path_prof_data_dir);
18 }
19 rfs::create_dir_all(&path_prof_data_dir);
20 let path_merged_profdata = path_prof_data_dir.join("merged.profdata");
21 rustc().input("opaque.rs").codegen_units(1).run();
22 rustc()
23 .input("interesting.rs")
24 .profile_generate(&path_prof_data_dir)
25 .opt()
26 .crate_type("lib,cdylib")
27 .codegen_units(cg_units)
28 .run();
29 rustc()
30 .input("main.rs")
31 .arg("-Clto=thin")
32 .opt()
33 .codegen_units(cg_units)
34 .profile_generate(&path_prof_data_dir)
35 .opt()
36 .run();
37 run("main");
38 llvm_profdata().merge().output(&path_merged_profdata).input(path_prof_data_dir).run();
39 rustc()
40 .input("interesting.rs")
41 .profile_use(&path_merged_profdata)
42 .opt()
43 .crate_type("lib,cdylib")
44 .codegen_units(cg_units)
45 .emit("link")
46 .run();
47 rustc()
48 .input("main.rs")
49 .arg("-Clto=thin")
50 .opt()
51 .codegen_units(cg_units)
52 .profile_use(&path_merged_profdata)
53 .emit("llvm-ir,link")
54 .opt()
55 .run();
56 let files = shallow_find_files(".", |path| {
57 has_prefix(path, "main.interesting.interesting") && has_extension(path, "ll")
58 });
59 assert_eq!(files.len(), 1);
60 let llvm_ir = &files[0];
61 llvm_filecheck().patterns("interesting.rs").stdin_buf(rfs::read(llvm_ir)).run();
62}
63
64fn main() {
65 run_test(1);
66 run_test(16);
67}