Validate CopyForDeref and DerefTemps better and remove them from runtime MIR

compiler/rustc_borrowck/src/lib.rs

@@ -1539,24 +1539,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
  self.consume_operand(location, (operand, span), state)
  }
 
- &Rvalue::CopyForDeref(place) => {
- self.access_place(
- location,
- (place, span),
- (Deep, Read(ReadKind::Copy)),
- LocalMutationIsAllowed::No,
- state,
- );
-
- // Finally, check if path was already moved.
- self.check_if_path_or_subpath_is_moved(
- location,
- InitializationRequiringAction::Use,
- (place.as_ref(), span),
- state,
- );
- }
-
  &Rvalue::Discriminant(place) => {
  let af = match *rvalue {
  Rvalue::Discriminant(..) => None,
@@ -1618,6 +1600,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
  Rvalue::WrapUnsafeBinder(op, _) => {
  self.consume_operand(location, (op, span), state);
  }
+
+ Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in borrowck"),
  }
  }
 

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -301,11 +301,6 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> {
  | Rvalue::Cast(_ /*cast_kind*/, operand, _ /*ty*/)
  | Rvalue::ShallowInitBox(operand, _ /*ty*/) => self.consume_operand(location, operand),
 
- &Rvalue::CopyForDeref(place) => {
- let op = &Operand::Copy(place);
- self.consume_operand(location, op);
- }
-
  &Rvalue::Discriminant(place) => {
  self.access_place(
  location,
@@ -331,6 +326,8 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> {
  Rvalue::WrapUnsafeBinder(op, _) => {
  self.consume_operand(location, op);
  }
+
+ Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in borrowck"),
  }
  }
 

compiler/rustc_codegen_cranelift/src/base.rs

@@ -600,11 +600,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
  let val = codegen_operand(fx, operand);
  lval.write_cvalue(fx, val);
  }
- Rvalue::CopyForDeref(place) => {
- let cplace = codegen_place(fx, place);
- let val = cplace.to_cvalue(fx);
- lval.write_cvalue(fx, val)
- }
  Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) => {
  let place = codegen_place(fx, place);
  let ref_ = place.place_ref(fx, lval.layout());
@@ -928,6 +923,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
  let operand = codegen_operand(fx, operand);
  lval.write_cvalue_transmute(fx, operand);
  }
+ Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"),
  }
  }
  StatementKind::StorageLive(_)

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -504,9 +504,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  self.codegen_place_to_pointer(bx, place, mk_ref)
  }
 
- mir::Rvalue::CopyForDeref(place) => {
- self.codegen_operand(bx, &mir::Operand::Copy(place))
- }
  mir::Rvalue::RawPtr(kind, place) => {
  let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
  Ty::new_ptr(tcx, ty, kind.to_mutbl_lossy())
@@ -742,6 +739,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
  let layout = bx.cx().layout_of(binder_ty);
  OperandRef { val: operand.val, layout }
  }
+ mir::Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"),
  }
  }
 

compiler/rustc_const_eval/src/check_consts/qualifs.rs

@@ -234,7 +234,7 @@ where
 
  Rvalue::Discriminant(place) => in_place::<Q, _>(cx, in_local, place.as_ref()),
 
- Rvalue::CopyForDeref(place) => in_place::<Q, _>(cx, in_local, place.as_ref()),
+ Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"),
 
  Rvalue::Use(operand)
  | Rvalue::Repeat(operand, _)

compiler/rustc_const_eval/src/interpret/step.rs

@@ -188,10 +188,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
  self.copy_op(&op, &dest)?;
  }
 
- CopyForDeref(place) => {
- let op = self.eval_place_to_op(place, Some(dest.layout))?;
- self.copy_op(&op, &dest)?;
- }
+ CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"),
 
  BinaryOp(bin_op, box (ref left, ref right)) => {
  let layout = util::binop_left_homogeneous(bin_op).then_some(dest.layout);

compiler/rustc_middle/src/mir/mod.rs

@@ -1065,7 +1065,11 @@ pub enum LocalInfo<'tcx> {
  /// A temporary created during evaluating `if` predicate, possibly for pattern matching for `let`s,
  /// and subject to Edition 2024 temporary lifetime rules
  IfThenRescopeTemp { if_then: HirId },
- /// A temporary created during the pass `Derefer` to avoid it's retagging
+ /// A temporary created during the pass `Derefer` treated as a transparent alias
+ /// for the place its copied from by analysis passes such as `AddRetag` and `ElaborateDrops`.
+ ///
+ /// It may only be written to by a `CopyForDeref` and otherwise only accessed through a deref.
+ /// In runtime MIR, it is replaced with a normal `Boring` local.
  DerefTemp,
  /// A temporary created for borrow checking.
  FakeBorrow,

compiler/rustc_middle/src/mir/syntax.rs

@@ -130,7 +130,9 @@ pub enum RuntimePhase {
  /// * [`TerminatorKind::Yield`]
  /// * [`TerminatorKind::CoroutineDrop`]
  /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
+ /// * [`Rvalue::CopyForDeref`]
  /// * [`PlaceElem::OpaqueCast`]
+ /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)
  ///
  /// And the following variants are allowed:
  /// * [`StatementKind::Retag`]
@@ -1460,11 +1462,13 @@ pub enum Rvalue<'tcx> {
  /// A CopyForDeref is equivalent to a read from a place at the
  /// codegen level, but is treated specially by drop elaboration. When such a read happens, it
  /// is guaranteed (via nature of the mir_opt `Derefer` in rustc_mir_transform/src/deref_separator)
- /// that the only use of the returned value is a deref operation, immediately
- /// followed by one or more projections. Drop elaboration treats this rvalue as if the
+ /// that the returned value is written into a `DerefTemp` local and that its only use is a deref operation,
+ /// immediately followed by one or more projections. Drop elaboration treats this rvalue as if the
  /// read never happened and just projects further. This allows simplifying various MIR
  /// optimizations and codegen backends that previously had to handle deref operations anywhere
  /// in a place.
+ ///
+ /// Disallowed in runtime MIR and is replaced by normal copies.
  CopyForDeref(Place<'tcx>),
 
  /// Wraps a value in an unsafe binder.

compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs

@@ -253,7 +253,6 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
  Ok(Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, offset))))
  },
  @call(mir_ptr_metadata, args) => Ok(Rvalue::UnaryOp(UnOp::PtrMetadata, self.parse_operand(args[0])?)),
- @call(mir_copy_for_deref, args) => Ok(Rvalue::CopyForDeref(self.parse_place(args[0])?)),
  ExprKind::Borrow { borrow_kind, arg } => Ok(
  Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
  ),

compiler/rustc_mir_transform/src/copy_prop.rs

@@ -74,9 +74,7 @@ fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> DenseBitSet<Local> {
  let mut fully_moved = DenseBitSet::new_filled(body.local_decls.len());
 
  for (_, rvalue, _) in ssa.assignments(body) {
- let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place))
- | Rvalue::CopyForDeref(place)) = rvalue
- else {
+ let Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) = rvalue else {
  continue;
  };
 
@@ -85,7 +83,7 @@ fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> DenseBitSet<Local> {
  continue;
  }
 
- if let Rvalue::Use(Operand::Copy(_)) | Rvalue::CopyForDeref(_) = rvalue {
+ if let Rvalue::Use(Operand::Copy(_)) = rvalue {
  fully_moved.remove(rhs);
  }
  }
@@ -146,8 +144,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
 
  // Do not leave tautological assignments around.
  if let StatementKind::Assign(box (lhs, ref rhs)) = stmt.kind
- && let Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)) | Rvalue::CopyForDeref(rhs) =
- *rhs
+ && let Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)) = *rhs
  && lhs == rhs
  {
  stmt.make_nop(true);

compiler/rustc_mir_transform/src/coroutine.rs

@@ -1625,27 +1625,27 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform {
  let mut drop_shim =
  create_coroutine_drop_shim_async(tcx, &transform, body, drop_clean, can_unwind);
  // Run derefer to fix Derefs that are not in the first place
- deref_finder(tcx, &mut drop_shim);
+ deref_finder(tcx, &mut drop_shim, false);
  body.coroutine.as_mut().unwrap().coroutine_drop_async = Some(drop_shim);
  } else {
  // If coroutine has no async drops, generating sync drop shim
  let mut drop_shim =
  create_coroutine_drop_shim(tcx, &transform, coroutine_ty, body, drop_clean);
  // Run derefer to fix Derefs that are not in the first place
- deref_finder(tcx, &mut drop_shim);
+ deref_finder(tcx, &mut drop_shim, false);
  body.coroutine.as_mut().unwrap().coroutine_drop = Some(drop_shim);
 
  // For coroutine with sync drop, generating async proxy for `future_drop_poll` call
  let mut proxy_shim = create_coroutine_drop_shim_proxy_async(tcx, body);
- deref_finder(tcx, &mut proxy_shim);
+ deref_finder(tcx, &mut proxy_shim, false);
  body.coroutine.as_mut().unwrap().coroutine_drop_proxy_async = Some(proxy_shim);
  }
 
  // Create the Coroutine::resume / Future::poll function
  create_coroutine_resume_function(tcx, transform, body, can_return, can_unwind);
 
  // Run derefer to fix Derefs that are not in the first place
- deref_finder(tcx, body);
+ deref_finder(tcx, body, false);
  }
 
  fn is_required(&self) -> bool {

compiler/rustc_mir_transform/src/dataflow_const_prop.rs

@@ -309,12 +309,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
  self.assign_operand(state, target, operand);
  }
  }
- Rvalue::CopyForDeref(rhs) => {
- state.flood(target.as_ref(), &self.map);
- if let Some(target) = self.map.find(target.as_ref()) {
- self.assign_operand(state, target, &Operand::Copy(*rhs));
- }
- }
+ Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"),
  Rvalue::Aggregate(kind, operands) => {
  // If we assign `target = Enum::Variant#0(operand)`,
  // we must make sure that all `target as Variant#i` are `Top`.
@@ -488,9 +483,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
  }
  Rvalue::Discriminant(place) => state.get_discr(place.as_ref(), &self.map),
  Rvalue::Use(operand) => return self.handle_operand(operand, state),
- Rvalue::CopyForDeref(place) => {
- return self.handle_operand(&Operand::Copy(*place), state);
- }
+ Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"),
  Rvalue::Ref(..) | Rvalue::RawPtr(..) => {
  // We don't track such places.
  return ValueOrPlace::TOP;

compiler/rustc_mir_transform/src/deref_separator.rs

@@ -11,6 +11,7 @@ struct DerefChecker<'a, 'tcx> {
  tcx: TyCtxt<'tcx>,
  patcher: MirPatch<'tcx>,
  local_decls: &'a LocalDecls<'tcx>,
+ add_deref_metadata: bool,
 }
 
 impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> {
@@ -39,7 +40,11 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> {
  let temp = self.patcher.new_local_with_info(
  ty,
  self.local_decls[p_ref.local].source_info.span,
- LocalInfo::DerefTemp,
+ if self.add_deref_metadata {
+ LocalInfo::DerefTemp
+ } else {
+ LocalInfo::Boring
+ },
  );
 
  // We are adding current p_ref's projections to our
@@ -50,7 +55,11 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> {
  self.patcher.add_assign(
  loc,
  Place::from(temp),
- Rvalue::CopyForDeref(deref_place),
+ if self.add_deref_metadata {
+ Rvalue::CopyForDeref(deref_place)
+ } else {
+ Rvalue::Use(Operand::Copy(deref_place))
+ },
  );
  place_local = temp;
  last_len = p_ref.projection.len();
@@ -67,9 +76,14 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> {
  }
 }
 
-pub(super) fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+pub(super) fn deref_finder<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ body: &mut Body<'tcx>,
+ add_deref_metadata: bool,
+) {
  let patch = MirPatch::new(body);
- let mut checker = DerefChecker { tcx, patcher: patch, local_decls: &body.local_decls };
+ let mut checker =
+ DerefChecker { tcx, patcher: patch, local_decls: &body.local_decls, add_deref_metadata };
 
  for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
  checker.visit_basic_block_data(bb, data);
@@ -80,7 +94,7 @@ pub(super) fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 
 impl<'tcx> crate::MirPass<'tcx> for Derefer {
  fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
- deref_finder(tcx, body);
+ deref_finder(tcx, body, true);
  }
 
  fn is_required(&self) -> bool {

compiler/rustc_mir_transform/src/dest_prop.rs

@@ -285,8 +285,7 @@ impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> {
  match &statement.kind {
  StatementKind::Assign(box (dest, rvalue)) => {
  match rvalue {
- Rvalue::CopyForDeref(place)
- | Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) => {
+ Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) => {
  // These might've been turned into self-assignments by the replacement
  // (this includes the original statement we wanted to eliminate).
  if dest == place {
@@ -400,7 +399,7 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> {
  fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
  if let StatementKind::Assign(box (
  lhs,
- Rvalue::CopyForDeref(rhs) | Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)),
+ Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)),
  )) = &statement.kind
  && let Some(src) = lhs.as_local()
  && let Some(dest) = rhs.as_local()

compiler/rustc_mir_transform/src/elaborate_drops.rs

@@ -14,7 +14,6 @@ use rustc_mir_dataflow::{
 use rustc_span::Span;
 use tracing::{debug, instrument};
 
-use crate::deref_separator::deref_finder;
 use crate::elaborate_drop::{DropElaborator, DropFlagMode, DropStyle, Unwind, elaborate_drop};
 use crate::patch::MirPatch;
 
@@ -87,7 +86,6 @@ impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops {
  .elaborate()
  };
  elaborate_patch.apply(body);
- deref_finder(tcx, body);
  }
 
  fn is_required(&self) -> bool {

compiler/rustc_mir_transform/src/erase_deref_temps.rs

@@ -0,0 +1,41 @@
+//! This pass converts all `DerefTemp` locals into normal temporaries
+//! and turns their `CopyForDeref` rvalues into normal copies.
+
+use rustc_middle::mir::visit::MutVisitor;
+use rustc_middle::mir::*;
+use rustc_middle::ty::TyCtxt;
+
+struct EraseDerefTempsVisitor<'tcx> {
+ tcx: TyCtxt<'tcx>,
+}
+
+impl<'tcx> MutVisitor<'tcx> for EraseDerefTempsVisitor<'tcx> {
+ fn tcx(&self) -> TyCtxt<'tcx> {
+ self.tcx
+ }
+
+ fn visit_rvalue(&mut self, rvalue: &mut Rvalue<'tcx>, _: Location) {
+ if let &mut Rvalue::CopyForDeref(place) = rvalue {
+ *rvalue = Rvalue::Use(Operand::Copy(place))
+ }
+ }
+
+ fn visit_local_decl(&mut self, _: Local, local_decl: &mut LocalDecl<'tcx>) {
+ if local_decl.is_deref_temp() {
+ let info = local_decl.local_info.as_mut().unwrap_crate_local();
+ **info = LocalInfo::Boring;
+ }
+ }
+}
+
+pub(super) struct EraseDerefTemps;
+
+impl<'tcx> crate::MirPass<'tcx> for EraseDerefTemps {
+ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ EraseDerefTempsVisitor { tcx }.visit_body_preserves_cfg(body);
+ }
+
+ fn is_required(&self) -> bool {
+ true
+ }
+}