Safe Transmute: Refactor error handling and Answer type

- Create `Answer` type that is not just a type alias of `Result` - Remove a usage of `map_layouts` to make the code easier to read - Don't hide errors related to Unknown Layout when computing transmutability

Author: bryangarza

compiler/rustc_trait_selection/src/solve/eval_ctxt.rs

@@ -668,18 +668,16 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
  scope: Ty<'tcx>,
  assume: rustc_transmute::Assume,
  ) -> Result<Certainty, NoSolution> {
+ use rustc_transmute::Answer;
  // FIXME(transmutability): This really should be returning nested goals for `Answer::If*`
  match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable(
  ObligationCause::dummy(),
  src_and_dst,
  scope,
  assume,
  ) {
- Ok(None) => Ok(Certainty::Yes),
- Err(_)
- | Ok(Some(rustc_transmute::Condition::IfTransmutable { .. }))
- | Ok(Some(rustc_transmute::Condition::IfAll(_)))
- | Ok(Some(rustc_transmute::Condition::IfAny(_))) => Err(NoSolution),
+ Answer::Yes => Ok(Certainty::Yes),
+ Answer::No(_) | Answer::If(_) => Err(NoSolution),
  }
  }
 }

compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs

@@ -65,6 +65,11 @@ pub struct ImplCandidate<'tcx> {
  pub similarity: CandidateSimilarity,
 }
 
+enum GetSafeTransmuteErrorAndReason {
+ Silent,
+ Error { err_msg: String, safe_transmute_explanation: String },
+}
+
 pub trait InferCtxtExt<'tcx> {
  /// Given some node representing a fn-like thing in the HIR map,
  /// returns a span and `ArgKind` information that describes the
@@ -724,11 +729,17 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
  == self.tcx.lang_items().transmute_trait()
  {
  // Recompute the safe transmute reason and use that for the error reporting
- self.get_safe_transmute_error_and_reason(
+ match self.get_safe_transmute_error_and_reason(
  obligation.clone(),
  trait_ref,
  span,
- )
+ ) {
+ GetSafeTransmuteErrorAndReason::Silent => return,
+ GetSafeTransmuteErrorAndReason::Error {
+ err_msg,
+ safe_transmute_explanation,
+ } => (err_msg, Some(safe_transmute_explanation)),
+ }
  } else {
  (err_msg, None)
  };
@@ -1292,7 +1303,7 @@ trait InferCtxtPrivExt<'tcx> {
  obligation: PredicateObligation<'tcx>,
  trait_ref: ty::PolyTraitRef<'tcx>,
  span: Span,
- ) -> (String, Option<String>);
+ ) -> GetSafeTransmuteErrorAndReason;
 
  fn add_tuple_trait_message(
  &self,
@@ -2738,7 +2749,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
  obligation: PredicateObligation<'tcx>,
  trait_ref: ty::PolyTraitRef<'tcx>,
  span: Span,
- ) -> (String, Option<String>) {
+ ) -> GetSafeTransmuteErrorAndReason {
+ use rustc_transmute::Answer;
+
  // Erase regions because layout code doesn't particularly care about regions.
  let trait_ref = self.tcx.erase_regions(self.tcx.erase_late_bound_regions(trait_ref));
 
@@ -2758,13 +2771,13 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
  scope,
  assume,
  ) {
- Err(reason) => {
+ Answer::No(reason) => {
  let dst = trait_ref.substs.type_at(0);
  let src = trait_ref.substs.type_at(1);
- let custom_err_msg = format!(
+ let err_msg = format!(
  "`{src}` cannot be safely transmuted into `{dst}` in the defining scope of `{scope}`"
  );
- let reason_msg = match reason {
+ let safe_transmute_explanation = match reason {
  rustc_transmute::Reason::SrcIsUnspecified => {
  format!("`{src}` does not have a well-specified layout")
  }
@@ -2794,11 +2807,21 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
  rustc_transmute::Reason::DstIsMoreUnique => {
  format!("`{src}` is a shared reference, but `{dst}` is a unique reference")
  }
+ // Already reported by rustc
+ rustc_transmute::Reason::TypeError => {
+ return GetSafeTransmuteErrorAndReason::Silent;
+ }
+ rustc_transmute::Reason::SrcLayoutUnknown => {
+ format!("`{src}` has an unknown layout")
+ }
+ rustc_transmute::Reason::DstLayoutUnknown => {
+ format!("`{dst}` has an unknown layout")
+ }
  };
- (custom_err_msg, Some(reason_msg))
+ GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation }
  }
  // Should never get a Yes at this point! We already ran it before, and did not get a Yes.
- Ok(None) => span_bug!(
+ Answer::Yes => span_bug!(
  span,
  "Inconsistent rustc_transmute::is_transmutable(...) result, got Yes",
  ),

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -285,28 +285,22 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
  &mut self,
  obligation: &TraitObligation<'tcx>,
  ) -> Result<ImplSourceBuiltinData<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
+ use rustc_transmute::{Answer, Condition};
  #[instrument(level = "debug", skip(tcx, obligation, predicate))]
  fn flatten_answer_tree<'tcx>(
  tcx: TyCtxt<'tcx>,
  obligation: &TraitObligation<'tcx>,
  predicate: TraitPredicate<'tcx>,
- answer: rustc_transmute::Condition<rustc_transmute::layout::rustc::Ref<'tcx>>,
+ cond: Condition<rustc_transmute::layout::rustc::Ref<'tcx>>,
  ) -> Vec<PredicateObligation<'tcx>> {
- match answer {
+ match cond {
  // FIXME(bryangarza): Add separate `IfAny` case, instead of treating as `IfAll`
  // Not possible until the trait solver supports disjunctions of obligations
- rustc_transmute::Condition::IfAll(answers)
- | rustc_transmute::Condition::IfAny(answers) => {
- let mut nested = vec![];
- for flattened in answers
- .into_iter()
- .map(|answer| flatten_answer_tree(tcx, obligation, predicate, answer))
- {
- nested.extend(flattened);
- }
- nested
- }
- rustc_transmute::Condition::IfTransmutable { src, dst } => {
+ Condition::IfAll(conds) | Condition::IfAny(conds) => conds
+ .into_iter()
+ .flat_map(|cond| flatten_answer_tree(tcx, obligation, predicate, cond))
+ .collect(),
+ Condition::IfTransmutable { src, dst } => {
  let trait_def_id = obligation.predicate.def_id();
  let scope = predicate.trait_ref.substs.type_at(2);
  let assume_const = predicate.trait_ref.substs.const_at(3);
@@ -333,11 +327,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
  // If Dst is mutable, check bidirectionally.
  // For example, transmuting bool -> u8 is OK as long as you can't update that u8
  // to be > 1, because you could later transmute the u8 back to a bool and get UB.
- let mut obligations = vec![make_obl(src.ty, dst.ty)];
- if dst.mutability == Mutability::Mut {
- obligations.push(make_obl(dst.ty, src.ty));
+ match dst.mutability {
+  Mutability::Not => vec![make_obl(src.ty, dst.ty)],
+ Mutability::Mut => vec![make_obl(src.ty, dst.ty), make_obl(dst.ty, src.ty)],
  }
- obligations
  }
  }
  }
@@ -370,9 +363,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
  );
 
  let fully_flattened = match maybe_transmutable {
- Err(_) => Err(Unimplemented)?,
- Ok(Some(mt)) => flatten_answer_tree(self.tcx(), obligation, predicate, mt),
- Ok(None) => vec![],
+ Answer::No(_) => Err(Unimplemented)?,
+ Answer::If(cond) => flatten_answer_tree(self.tcx(), obligation, predicate, cond),
+ Answer::Yes => vec![],
  };
 
  debug!(?fully_flattened);

compiler/rustc_transmute/src/layout/tree.rs

@@ -188,14 +188,14 @@ pub(crate) mod rustc {
  /// The layout of the type is unspecified.
  Unspecified,
  /// This error will be surfaced elsewhere by rustc, so don't surface it.
- Unknown,
+ UnknownLayout,
  TypeError(ErrorGuaranteed),
  }
 
  impl<'tcx> From<LayoutError<'tcx>> for Err {
  fn from(err: LayoutError<'tcx>) -> Self {
  match err {
- LayoutError::Unknown(..) => Self::Unknown,
+ LayoutError::Unknown(..) => Self::UnknownLayout,
  err => unimplemented!("{:?}", err),
  }
  }

compiler/rustc_transmute/src/lib.rs

@@ -19,10 +19,16 @@ pub struct Assume {
  pub validity: bool,
 }
 
-/// Either we have an error, or we have an optional Condition that must hold.
-pub type Answer<R> = Result<Option<Condition<R>>, Reason>;
+/// Either we have an error, transmutation is allowed, or we have an optional
+/// Condition that must hold.
+#[derive(Debug, Hash, Eq, PartialEq, Clone)]
+pub enum Answer<R> {
+ Yes,
+ No(Reason),
+ If(Condition<R>),
+}
 
-/// A condition which must hold for safe transmutation to be possible
+/// A condition which must hold for safe transmutation to be possible.
 #[derive(Debug, Hash, Eq, PartialEq, Clone)]
 pub enum Condition<R> {
  /// `Src` is transmutable into `Dst`, if `src` is transmutable into `dst`.
@@ -35,7 +41,7 @@ pub enum Condition<R> {
  IfAny(Vec<Condition<R>>),
 }
 
-/// Answers: Why wasn't the source type transmutable into the destination type?
+/// Answers "why wasn't the source type transmutable into the destination type?"
 #[derive(Debug, Hash, Eq, PartialEq, PartialOrd, Ord, Clone)]
 pub enum Reason {
  /// The layout of the source type is unspecified.
@@ -52,6 +58,12 @@ pub enum Reason {
  DstHasStricterAlignment { src_min_align: usize, dst_min_align: usize },
  /// Can't go from shared pointer to unique pointer
  DstIsMoreUnique,
+ /// Encountered a type error
+ TypeError,
+ /// The layout of src is unknown
+ SrcLayoutUnknown,
+ /// The layout of dst is unknown
+ DstLayoutUnknown,
 }
 
 #[cfg(feature = "rustc")]