Implement asm goto for LLVM and GCC backend

Author: nbdd0121

compiler/rustc_codegen_gcc/src/asm.rs

@@ -107,7 +107,7 @@ enum ConstraintOrRegister {
 
 
 impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
- fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
+ fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option<Self::BasicBlock>, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) {
  if options.contains(InlineAsmOptions::MAY_UNWIND) {
  self.sess().dcx()
  .create_err(UnwindingInlineAsm { span: span[0] })
@@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
  // added to `outputs.len()`
  let mut inputs = vec![];
 
+ // GCC index of a label equals its position in the array added to
+ // `outputs.len() + inputs.len()`.
+ let mut labels = vec![];
+
  // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _`
  let mut clobbers = vec![];
 
@@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
  // some targets to add a leading underscore (Mach-O).
  constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
  }
+
+ InlineAsmOperandRef::Label { label } => {
+ labels.push(label);
+ }
  }
  }
 
@@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
  InlineAsmOperandRef::Const { .. } => {
  // processed in the previous pass
  }
+
+ InlineAsmOperandRef::Label { .. } => {
+ // processed in the previous pass
+ }
  }
  }
 
@@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
  InlineAsmOperandRef::Const { ref string } => {
  template_str.push_str(string);
  }
+
+ InlineAsmOperandRef::Label { label } => {
+ let label_gcc_index = labels.iter()
+ .position(|&l| l == label)
+ .expect("wrong rust index");
+ let gcc_index = label_gcc_index + outputs.len() + inputs.len();
+ push_to_template(Some('l'), gcc_index);
+ }
  }
  }
  }
@@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
  // 4. Generate Extended Asm block
 
  let block = self.llbb();
- let extended_asm = block.add_extended_asm(None, &template_str);
+ let extended_asm = if let Some(dest) = dest {
+ assert!(!labels.is_empty());
+ block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest))
+ } else {
+ block.add_extended_asm(None, &template_str)
+ };
 
  for op in &outputs {
  extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var);
@@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
  if !options.contains(InlineAsmOptions::NOSTACK) {
  // TODO(@Commeownist): figure out how to align stack
  }
- if options.contains(InlineAsmOptions::NORETURN) {
+ if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) {
  let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
  let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) };
  self.call(self.type_void(), None, None, builtin_unreachable, &[], None);

compiler/rustc_codegen_llvm/src/asm.rs

@@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
  options: InlineAsmOptions,
  line_spans: &[Span],
  instance: Instance<'_>,
- dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+ dest: Option<Self::BasicBlock>,
+ catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
  ) {
  let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
  }
 
  // Build the template string
+ let mut labels = vec![];
  let mut template_str = String::new();
  for piece in template {
  match *piece {
@@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
  // Only emit the raw symbol name
  template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
  }
+ InlineAsmOperandRef::Label { label } => {
+ template_str.push_str(&format!("${{{}:l}}", constraints.len()));
+ constraints.push("!i".to_owned());
+ labels.push(label);
+ }
  }
  }
  }
@@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
  &constraints.join(","),
  &inputs,
  output_type,
+ &labels,
  volatile,
  alignstack,
  dialect,
  line_spans,
  options.contains(InlineAsmOptions::MAY_UNWIND),
- dest_catch_funclet,
+ dest,
+ catch_funclet,
  )
  .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
 
@@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
  attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
 
  // Switch to the 'normal' basic block if we did an `invoke` instead of a `call`
- if let Some((dest, _, _)) = dest_catch_funclet {
+ if let Some(dest) = dest {
  self.switch_to_block(dest);
  }
 
@@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>(
  cons: &str,
  inputs: &[&'ll Value],
  output: &'ll llvm::Type,
+ labels: &[&'ll llvm::BasicBlock],
  volatile: bool,
  alignstack: bool,
  dia: llvm::AsmDialect,
  line_spans: &[Span],
  unwind: bool,
- dest_catch_funclet: Option<(
- &'ll llvm::BasicBlock,
- &'ll llvm::BasicBlock,
- Option<&Funclet<'ll>>,
- )>,
+ dest: Option<&'ll llvm::BasicBlock>,
+ catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
 ) -> Option<&'ll Value> {
  let volatile = if volatile { llvm::True } else { llvm::False };
  let alignstack = if alignstack { llvm::True } else { llvm::False };
@@ -457,8 +464,10 @@ pub(crate) fn inline_asm_call<'ll>(
  can_throw,
  );
 
- let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
- bx.invoke(fty, None, None, v, inputs, dest, catch, funclet)
+ let call = if !labels.is_empty() {
+ bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None)
+ } else if let Some((catch, funclet)) = catch_funclet {
+ bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet)
  } else {
  bx.call(fty, None, None, v, inputs, None)
  };

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -377,12 +377,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
  constraint,
  inputs,
  self.type_void(),
+ &[],
  true,
  false,
  llvm::AsmDialect::Att,
  &[span],
  false,
  None,
+ None,
  )
  .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -264,7 +264,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
  mir::UnwindAction::Unreachable => None,
  };
 
- if let Some(cleanup) = unwind_target {
+ if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) {
  let ret_llbb = if let Some(target) = destination {
  fx.llbb(target)
  } else {
@@ -277,11 +277,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
  options,
  line_spans,
  instance,
- Some((ret_llbb, cleanup, self.funclet(fx))),
+ Some(ret_llbb),
+ None,
+ );
+ MergingSucc::False
+ } else if let Some(cleanup) = unwind_target {
+ let ret_llbb = if let Some(target) = destination {
+ fx.llbb(target)
+ } else {
+ fx.unreachable_block()
+ };
+
+ bx.codegen_inline_asm(
+ template,
+ operands,
+ options,
+ line_spans,
+ instance,
+ Some(ret_llbb),
+ Some((cleanup, self.funclet(fx))),
  );
  MergingSucc::False
  } else {
- bx.codegen_inline_asm(template, operands, options, line_spans, instance, None);
+ bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None);
 
  if let Some(target) = destination {
  self.funclet_br(fx, bx, target, mergeable_succ)
@@ -1067,7 +1085,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  operands: &[mir::InlineAsmOperand<'tcx>],
  options: ast::InlineAsmOptions,
  line_spans: &[Span],
- destination: Option<mir::BasicBlock>,
+ targets: &[mir::BasicBlock],
  unwind: mir::UnwindAction,
  instance: Instance<'_>,
  mergeable_succ: bool,
@@ -1119,8 +1137,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  mir::InlineAsmOperand::SymStatic { def_id } => {
  InlineAsmOperandRef::SymStatic { def_id }
  }
- mir::InlineAsmOperand::Label { target_index: _ } => {
- todo!();
+ mir::InlineAsmOperand::Label { target_index } => {
+ InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) }
  }
  })
  .collect();
@@ -1132,7 +1150,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  &operands,
  options,
  line_spans,
- destination,
+ if options.contains(InlineAsmOptions::NORETURN) {
+ None
+ } else {
+ targets.get(0).copied()
+ },
  unwind,
  instance,
  mergeable_succ,
@@ -1298,7 +1320,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  operands,
  options,
  line_spans,
- targets.get(0).copied(),
+ targets,
  unwind,
  self.instance,
  mergeable_succ(),

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
  SymStatic {
  def_id: DefId,
  },
+ Label {
+ label: B::BasicBlock,
+ },
 }
 
 #[derive(Debug)]
@@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
  options: InlineAsmOptions,
  line_spans: &[Span],
  instance: Instance<'_>,
- dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+ dest: Option<Self::BasicBlock>,
+ catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
  );
 }