Remove no-op cleanups on MIR post-monomorphization

This speeds up LLVM and improves codegen overall. As an example, for cargo this cuts ~5% of the LLVM IR lines we generate (measured with -Cno-prepopulate-passes).

Author: Mark-Simulacrum

Cargo.lock

@@ -3505,6 +3505,7 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
+ "rustc_mir_transform",
  "rustc_query_system",
  "rustc_serialize",
  "rustc_session",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -37,6 +37,7 @@ rustc_span = { path = "../rustc_span" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
+rustc_mir_transform = { path = "../rustc_mir_transform" }
 serde_json = "1.0.59"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tempfile = "3.2"

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -279,10 +279,14 @@ impl CleanupKind {
 /// MSVC requires unwinding code to be split to a tree of *funclets*, where each funclet can only
 /// branch to itself or to its parent. Luckily, the code we generates matches this pattern.
 /// Recover that structure in an analyze pass.
-pub(crate) fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKind> {
+pub(crate) fn cleanup_kinds(
+ mir: &mir::Body<'_>,
+ nop_landing_pads: &DenseBitSet<mir::BasicBlock>,
+) -> IndexVec<mir::BasicBlock, CleanupKind> {
  fn discover_masters<'tcx>(
  result: &mut IndexSlice<mir::BasicBlock, CleanupKind>,
  mir: &mir::Body<'tcx>,
+ nop_landing_pads: &DenseBitSet<mir::BasicBlock>,
  ) {
  for (bb, data) in mir.basic_blocks.iter_enumerated() {
  match data.terminator().kind {
@@ -301,7 +305,9 @@ pub(crate) fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, Cl
  | TerminatorKind::InlineAsm { unwind, .. }
  | TerminatorKind::Assert { unwind, .. }
  | TerminatorKind::Drop { unwind, .. } => {
- if let mir::UnwindAction::Cleanup(unwind) = unwind {
+ if let mir::UnwindAction::Cleanup(unwind) = unwind
+ && !nop_landing_pads.contains(unwind)
+ {
  debug!(
  "cleanup_kinds: {:?}/{:?} registering {:?} as funclet",
  bb, data, unwind
@@ -382,7 +388,7 @@ pub(crate) fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, Cl
 
  let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, &mir.basic_blocks);
 
- discover_masters(&mut result, mir);
+ discover_masters(&mut result, mir, &nop_landing_pads);
  propagate(&mut result, mir);
  debug!("cleanup_kinds: result={:?}", result);
  result

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -200,7 +200,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
  }
 
  let unwind_block = match unwind {
- mir::UnwindAction::Cleanup(cleanup) => Some(self.llbb_with_cleanup(fx, cleanup)),
+ mir::UnwindAction::Cleanup(cleanup) => {
+ if !fx.nop_landing_pads.contains(cleanup) {
+ Some(self.llbb_with_cleanup(fx, cleanup))
+ } else {
+ None
+ }
+ }
  mir::UnwindAction::Continue => None,
  mir::UnwindAction::Unreachable => None,
  mir::UnwindAction::Terminate(reason) => {
@@ -286,7 +292,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
  mergeable_succ: bool,
  ) -> MergingSucc {
  let unwind_target = match unwind {
- mir::UnwindAction::Cleanup(cleanup) => Some(self.llbb_with_cleanup(fx, cleanup)),
+ mir::UnwindAction::Cleanup(cleanup) => {
+ if !fx.nop_landing_pads.contains(cleanup) {
+ Some(self.llbb_with_cleanup(fx, cleanup))
+ } else {
+ None
+ }
+ }
  mir::UnwindAction::Terminate(reason) => Some(fx.terminate_block(reason)),
  mir::UnwindAction::Continue => None,
  mir::UnwindAction::Unreachable => None,

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -97,6 +97,8 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
  /// A cold block is a block that is unlikely to be executed at runtime.
  cold_blocks: IndexVec<mir::BasicBlock, bool>,
 
+ nop_landing_pads: DenseBitSet<mir::BasicBlock>,
+
  /// The location where each MIR arg/var/tmp/ret is stored. This is
  /// usually an `PlaceRef` representing an alloca, but not always:
  /// sometimes we can skip the alloca and just store the value
@@ -176,8 +178,14 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
  let mut mir = tcx.instance_mir(instance.def);
 
- let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
- debug!("fn_abi: {:?}", fn_abi);
+ let nop_landing_pads = rustc_mir_transform::remove_noop_landing_pads::find_noop_landing_pads(
+ mir,
+ Some(rustc_mir_transform::remove_noop_landing_pads::ExtraInfo {
+ tcx,
+ instance,
+ typing_env: cx.typing_env(),
+ }),
+ );
 
  if tcx.features().ergonomic_clones() {
  let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions(
@@ -188,19 +196,23 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  mir = tcx.arena.alloc(optimize_use_clone::<Bx>(cx, monomorphized_mir));
  }
 
+ let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
+ debug!("fn_abi: {:?}", fn_abi);
+
  let debug_context = cx.create_function_debug_context(instance, fn_abi, llfn, &mir);
 
  let start_llbb = Bx::append_block(cx, llfn, "start");
  let mut start_bx = Bx::build(cx, start_llbb);
 
- if mir.basic_blocks.iter().any(|bb| {
- bb.is_cleanup || matches!(bb.terminator().unwind(), Some(mir::UnwindAction::Terminate(_)))
+ if mir::traversal::mono_reachable(&mir, tcx, instance).any(|(bb, block)| {
+ (block.is_cleanup && !nop_landing_pads.contains(bb))
+ || matches!(block.terminator().unwind(), Some(mir::UnwindAction::Terminate(_)))
  }) {
  start_bx.set_personality_fn(cx.eh_personality());
  }
 
- let cleanup_kinds =
- base::wants_new_eh_instructions(tcx.sess).then(|| analyze::cleanup_kinds(&mir));
+ let cleanup_kinds = base::wants_new_eh_instructions(tcx.sess)
+ .then(|| analyze::cleanup_kinds(&mir, &nop_landing_pads));
 
  let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> =
  mir.basic_blocks
@@ -228,6 +240,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  debug_context,
  per_local_var_debug_info: None,
  caller_location: None,
+ nop_landing_pads,
  };
 
  // It may seem like we should iterate over `required_consts` to ensure they all successfully
@@ -239,7 +252,36 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
  fx.compute_per_local_var_debug_info(&mut start_bx).unzip();
  fx.per_local_var_debug_info = per_local_var_debug_info;
 
- let traversal_order = traversal::mono_reachable_reverse_postorder(mir, tcx, instance);
+ let mut traversal_order = traversal::mono_reachable_reverse_postorder(mir, tcx, instance);
+
+ // Filter out blocks that won't be codegen'd because of nop_landing_pads optimization.
+ // FIXME: We might want to integrate the nop_landing_pads analysis into mono reachability.
+ {
+ let mut reachable = DenseBitSet::new_empty(mir.basic_blocks.len());
+ let mut to_visit = vec![mir::START_BLOCK];
+ while let Some(next) = to_visit.pop() {
+ if !reachable.insert(next) {
+ continue;
+ }
+
+ let block = &mir.basic_blocks[next];
+ if let Some(mir::UnwindAction::Cleanup(target)) = block.terminator().unwind()
+ && fx.nop_landing_pads.contains(*target)
+ {
+ // This edge will not be followed when we actually codegen, so skip generating it here.
+ //
+ // It's guaranteed that the cleanup block (`target`) occurs only in
+ // UnwindAction::Cleanup(...) -- i.e., we can't incorrectly filter too much here --
+ // because cleanup transitions must happen via UnwindAction::Cleanup.
+ to_visit.extend(block.terminator().successors().filter(|s| s != target));
+ } else {
+ to_visit.extend(block.terminator().successors());
+ }
+ }
+
+ traversal_order.retain(|bb| reachable.contains(*bb));
+ }
+
  let memory_locals = analyze::non_ssa_locals(&fx, &traversal_order);
 
  // Allocate variable and temp allocas

compiler/rustc_mir_transform/src/lib.rs

@@ -158,7 +158,7 @@ declare_passes! {
  mod prettify : ReorderBasicBlocks, ReorderLocals;
  mod promote_consts : PromoteTemps;
  mod ref_prop : ReferencePropagation;
- mod remove_noop_landing_pads : RemoveNoopLandingPads;
+ pub mod remove_noop_landing_pads : RemoveNoopLandingPads;
  mod remove_place_mention : RemovePlaceMention;
  mod remove_storage_markers : RemoveStorageMarkers;
  mod remove_uninit_drops : RemoveUninitDrops;