Break critical edges in inline asm before code generation

An inline asm terminator defines outputs along its target edges -- a fallthrough target and labeled targets. Code generation implements this by inserting code directly into the target blocks. This approach works only if the target blocks don't have other predecessors. Establish required invariant by extending existing code that breaks critical edges before code generation.

Author: tmiasko

compiler/rustc_mir_transform/src/add_call_guards.rs

@@ -65,6 +65,32 @@ impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
  // It's a critical edge, break it
  *destination = new_block(source_info, block.is_cleanup, *destination);
  }
+ Some(Terminator {
+ kind:
+ TerminatorKind::InlineAsm {
+ asm_macro: InlineAsmMacro::Asm,
+ ref mut targets,
+ ref operands,
+ unwind,
+ ..
+ },
+ source_info,
+ }) if self == &CriticalCallEdges => {
+ let has_outputs = operands.iter().any(|op| {
+ matches!(op, InlineAsmOperand::InOut { .. } | InlineAsmOperand::Out { .. })
+ });
+ let has_labels =
+ operands.iter().any(|op| matches!(op, InlineAsmOperand::Label { .. }));
+ let invoke =
+ matches!(unwind, UnwindAction::Cleanup(_) | UnwindAction::Terminate(_));
+ if has_outputs && (has_labels || invoke) {
+ for target in targets.iter_mut() {
+ if pred_count[*target] > 1 {
+ *target = new_block(source_info, block.is_cleanup, *target);
+ }
+ }
+ }
+ }
  _ => {}
  }
  }

tests/codegen/asm/critical.rs

@@ -0,0 +1,27 @@
+//@ only-x86_64
+//@ compile-flags: -C no-prepopulate-passes
+#![feature(asm_goto)]
+#![feature(asm_goto_with_outputs)]
+#![crate_type = "lib"]
+use std::arch::asm;
+
+// Regression test for #137867. Check that critical edges have been split before code generation,
+// and so all labels in callbr lead to distinct blocks (CHECK-DAG matches are non-overlapping).
+//
+// CHECK-LABEL: @f(
+// CHECK: callbr i32 asm
+// CHECK-NEXT: to label %[[BB0:.*]] [label %[[BB1:.*]], label %[[BB2:.*]]],
+// CHECK-DAG: [[BB0]]:
+// CHECK-DAG: [[BB1]]:
+// CHECK-DAG: [[BB2]]:
+#[unsafe(no_mangle)]
+pub unsafe fn f(mut a: u32) -> u32 {
+ asm!(
+ "jmp {}
+ jmp {}",
+ label {},
+ label {},
+ inout("eax") a,
+ );
+ a
+}