Implement profiling for compiler-generated move/copy operations

As part of Rust's move semantics, the compiler will generate memory copy operations to move objects about. These are generally pretty small, and the backend is good at optimizing them. But sometimes, if the type is large, they can end up being surprisingly expensive. In such cases, you might want to pass them by reference, or Box them up. However, these moves are also invisible to profiling. At best they appear as a `memcpy`, but one memcpy is basically indistinguishable from another, and its very hard to know that 1) it's actually a compiler-generated copy, and 2) what type it pertains to. This PR adds two new pseudo-functions in `core::profiling`: ``` pub fn compiler_move<T, const SIZE: usize>(_src: *const T, _dst: *mut T); pub fn compiler_copy<T, const SIZE: usize>(_src: *const T, _dst: *mut T); ``` These functions are never actually called however. A MIR transform pass -- `instrument_moves.rs` -- will locate all `Operand::Move`/`Copy` operations, and modify their source location to make them appear as if they had been inlined from `compiler_move`/`_copy`. These functions have two generic parameters: the type being copied, and its size in bytes. This should make it very easy to identify which types are being expensive in your program (both in aggregate, and at specific hotspots). The size isn't strictly necessary since you can derive it from the type, but it's small and it makes it easier to understand what you're looking at. This functionality is only enabled if you have debug info generation enabled, and also set the `-Zinstrument-moves` option. It does not instrument all moves. By default it will only annotate ones for types over 64 bytes. The `-Zinstrument-moves-size-limit` specifies the size in bytes to start instrumenting for. This has minimal impact on the size of debugging info. For rustc itself, the overall increase in librustc_driver*.so size is around .05% for 65 byte limit, 0.004% for 1025 byte limit, and a worst case of 0.6% for an 8 byte limit. There's no effect on generated code, it only adds debug info. As an example of a backtrace: ``` Breakpoint 1.3, __memcpy_avx512_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:255 255	ENTRY_P2ALIGN (MEMMOVE_SYMBOL (__memmove, unaligned_erms), 6) (gdb) bt # 0 __memcpy_avx512_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:255 # 1 0x0000555555590e7e in core::profiling::compiler_copy<[u64; 1000], 8000> () at library/core/src/profiling.rs:27 # 2 t::main () at t.rs:10 ```

Author: jsgf

compiler/rustc_mir_transform/src/instrument_moves.rs

@@ -0,0 +1,260 @@
+//! Instrumentation pass for move/copy operations.
+//!
+//! This pass modifies the source scopes of statements containing `Operand::Move` and `Operand::Copy`
+//! to make them appear as if they were inlined from `compiler_move()` and `compiler_copy()` intrinsic
+//! functions. This creates the illusion that moves/copies are function calls in debuggers and
+//! profilers, making them visible for performance analysis.
+//!
+//! The pass leverages the existing inlining infrastructure by creating synthetic `SourceScopeData`
+//! with the `inlined` field set to point to the appropriate intrinsic function.
+
+use rustc_index::IndexVec;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypingEnv};
+use rustc_session::config::DebugInfo;
+use rustc_span::sym;
+
+/// Default minimum size in bytes for move/copy operations to be instrumented. Set to 64+1 bytes
+/// (typical cache line size) to focus on potentially expensive operations.
+const DEFAULT_INSTRUMENT_MOVES_SIZE_LIMIT: u64 = 65;
+
+#[derive(Copy, Clone, Debug)]
+enum Operation {
+ Move,
+ Copy,
+}
+
+/// Bundle up parameters into a structure to make repeated calling neater
+struct Params<'a, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ source_scopes: &'a mut IndexVec<SourceScope, SourceScopeData<'tcx>>,
+ local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>,
+ typing_env: TypingEnv<'tcx>,
+ size_limit: u64,
+}
+
+/// MIR transform that instruments move/copy operations for profiler visibility.
+pub(crate) struct InstrumentMoves;
+
+impl<'tcx> crate::MirPass<'tcx> for InstrumentMoves {
+ fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
+ sess.opts.unstable_opts.instrument_moves && sess.opts.debuginfo != DebugInfo::None
+ }
+
+ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ // Skip promoted MIR bodies to avoid recursion
+ if body.source.promoted.is_some() {
+ return;
+ }
+
+ let typing_env = body.typing_env(tcx);
+ let size_limit = tcx
+ .sess
+ .opts
+ .unstable_opts
+ .instrument_moves_size_limit
+ .unwrap_or(DEFAULT_INSTRUMENT_MOVES_SIZE_LIMIT);
+
+ // Common params, including selectively borrowing the bits of Body we need to avoid
+ // mut/non-mut aliasing problems.
+ let mut params = Params {
+ tcx,
+ source_scopes: &mut body.source_scopes,
+ local_decls: &body.local_decls,
+ typing_env,
+ size_limit,
+ };
+
+ // Process each basic block
+ for block_data in body.basic_blocks.as_mut() {
+ for stmt in &mut block_data.statements {
+ let source_info = &mut stmt.source_info;
+
+ if let StatementKind::Assign(box (_, rvalue)) = &stmt.kind {
+ match rvalue {
+ Rvalue::Use(op)
+ | Rvalue::Repeat(op, _)
+ | Rvalue::Cast(_, op, _)
+ | Rvalue::UnaryOp(_, op) => {
+ self.annotate_move(&mut params, source_info, op);
+ }
+ Rvalue::BinaryOp(_, box (lop, rop)) => {
+ self.annotate_move(&mut params, source_info, lop);
+ self.annotate_move(&mut params, source_info, rop);
+ }
+ Rvalue::Aggregate(_, ops) => {
+ for op in ops {
+ self.annotate_move(&mut params, source_info, op);
+ }
+ }
+ Rvalue::Ref(..)
+ | Rvalue::ThreadLocalRef(..)
+ | Rvalue::RawPtr(..)
+ | Rvalue::NullaryOp(..)
+ | Rvalue::Discriminant(..)
+ | Rvalue::CopyForDeref(..)
+ | Rvalue::ShallowInitBox(..)
+ | Rvalue::WrapUnsafeBinder(..) => {} // No operands to instrument
+ }
+ }
+ }
+
+ // Process terminator operands
+ if let Some(terminator) = &mut block_data.terminator {
+ let source_info = &mut terminator.source_info;
+ match &terminator.kind {
+ TerminatorKind::Call { func, args, .. } => {
+ // Instrument the function operand
+ self.annotate_move(&mut params, source_info, func);
+ // Instrument each argument
+ for arg in &*args {
+ self.annotate_move(&mut params, source_info, &arg.node);
+ }
+ }
+ TerminatorKind::SwitchInt { discr, .. } => {
+ self.annotate_move(&mut params, source_info, discr);
+ }
+ _ => {} // Other terminators don't have operands
+ }
+ }
+ }
+ }
+
+ fn is_required(&self) -> bool {
+ false // Optional optimization/instrumentation pass
+ }
+}
+
+impl InstrumentMoves {
+ /// If this is a Move or Copy of a concrete type, update its debug info to make it look like it
+ /// was inlined from `core::profiling::compiler_move`/`compiler_copy`.
+ fn annotate_move<'tcx>(
+ &self,
+ params: &mut Params<'_, 'tcx>,
+ source_info: &mut SourceInfo,
+ op: &Operand<'tcx>,
+ ) {
+ let (place, operation) = match op {
+ Operand::Move(place) => (place, Operation::Move),
+ Operand::Copy(place) => (place, Operation::Copy),
+ _ => return,
+ };
+ let Params { tcx, typing_env, local_decls, size_limit, source_scopes } = params;
+
+ if let Some(type_size) =
+ self.should_instrument_operation(*tcx, *typing_env, local_decls, place, *size_limit)
+ {
+ let ty = place.ty(*local_decls, *tcx).ty;
+ source_info.scope = self.create_inlined_scope(
+ *tcx,
+ *typing_env,
+ source_scopes,
+ source_info,
+ operation,
+ ty,
+ type_size,
+ );
+ }
+ }
+
+ /// Determines if an operation should be instrumented based on type characteristics.
+ /// Returns Some(size) if it should be instrumented, None otherwise.
+ fn should_instrument_operation<'tcx>(
+ &self,
+ tcx: TyCtxt<'tcx>,
+ typing_env: ty::TypingEnv<'tcx>,
+ local_decls: &rustc_index::IndexVec<Local, LocalDecl<'tcx>>,
+ place: &Place<'tcx>,
+ size_limit: u64,
+ ) -> Option<u64> {
+ let ty = place.ty(local_decls, tcx).ty;
+ let layout = match tcx.layout_of(typing_env.as_query_input(ty)) {
+ Ok(layout) => layout,
+ Err(err) => {
+ tracing::info!("Failed to get layout of {ty:?}: {err}");
+ return None;
+ }
+ };
+
+ let size = layout.size.bytes();
+
+ // 1. Skip ZST types (no actual move/copy happens)
+ if layout.is_zst() {
+ return None;
+ }
+
+ // 2. Check size threshold (only instrument large moves/copies)
+ if size < size_limit {
+ return None;
+ }
+
+ // 3. Skip scalar/vector types that won't generate memcpy
+ match layout.layout.backend_repr {
+ rustc_abi::BackendRepr::Scalar(_)
+ | rustc_abi::BackendRepr::ScalarPair(_, _)
+ | rustc_abi::BackendRepr::SimdVector { .. } => None,
+ _ => Some(size),
+ }
+ }
+
+ /// Creates an inlined scope that makes operations appear to come from
+ /// the specified compiler intrinsic function.
+ fn create_inlined_scope<'tcx>(
+ &self,
+ tcx: TyCtxt<'tcx>,
+ typing_env: TypingEnv<'tcx>,
+ source_scopes: &mut IndexVec<SourceScope, SourceScopeData<'tcx>>,
+ original_source_info: &SourceInfo,
+ operation: Operation,
+ ty: Ty<'tcx>,
+ type_size: u64,
+ ) -> SourceScope {
+ let intrinsic_def_id = match operation {
+ Operation::Move => tcx.get_diagnostic_item(sym::compiler_move),
+ Operation::Copy => tcx.get_diagnostic_item(sym::compiler_copy),
+ };
+
+ let Some(intrinsic_def_id) = intrinsic_def_id else {
+ // Shouldn't happen, but just return original scope if it does
+ return original_source_info.scope;
+ };
+
+ // Monomorphize the intrinsic for the actual type being moved/copied + size const parameter
+ // compiler_move<T, const SIZE: usize> or compiler_copy<T, const SIZE: usize>
+ let size_const = ty::Const::from_target_usize(tcx, type_size);
+ let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+ let intrinsic_instance = Instance::expect_resolve(
+ tcx,
+ typing_env,
+ intrinsic_def_id,
+ generic_args,
+ original_source_info.span,
+ );
+
+ // Create new inlined scope that makes the operation appear to come from the intrinsic
+ let inlined_scope_data = SourceScopeData {
+ span: original_source_info.span,
+ parent_scope: Some(original_source_info.scope),
+
+ // Pretend this op is inlined from the intrinsic
+ inlined: Some((intrinsic_instance, original_source_info.span)),
+
+ // Proper inlined scope chaining to maintain debug info hierarchy
+ inlined_parent_scope: {
+ let parent_scope = &source_scopes[original_source_info.scope];
+ if parent_scope.inlined.is_some() {
+ // If parent is already inlined, chain through it
+ Some(original_source_info.scope)
+ } else {
+ // Otherwise, use the parent's inlined_parent_scope
+ parent_scope.inlined_parent_scope
+ }
+ },
+
+ local_data: ClearCrossCrate::Clear,
+ };
+
+ // Add the new scope
+ source_scopes.push(inlined_scope_data)
+ }
+}

compiler/rustc_mir_transform/src/lib.rs

@@ -147,6 +147,7 @@ declare_passes! {
  // by custom rustc drivers, running all the steps by themselves. See #114628.
  pub mod inline : Inline, ForceInline;
  mod impossible_predicates : ImpossiblePredicates;
+ mod instrument_moves : InstrumentMoves;
  mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg };
  mod jump_threading : JumpThreading;
  mod known_panics_lint : KnownPanicsLint;
@@ -730,6 +731,9 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'
  // Cleanup for human readability, off by default.
  &prettify::ReorderBasicBlocks,
  &prettify::ReorderLocals,
+ // Instrument move/copy operations for profiler visibility.
+ // Late so we're instrumenting any Move/Copy that survived all the previous passes.
+ &instrument_moves::InstrumentMoves,
  // Dump the end result for testing and debugging purposes.
  &dump_mir::Marker("PreCodegen"),
  ],

compiler/rustc_session/src/options.rs

@@ -2384,6 +2384,12 @@ options! {
  "print some statistics about AST and HIR (default: no)"),
  instrument_mcount: bool = (false, parse_bool, [TRACKED],
  "insert function instrument code for mcount-based tracing (default: no)"),
+ instrument_moves: bool = (false, parse_bool, [TRACKED],
+ "emit debug info for compiler-generated move and copy operations \
+ to make them visible in profilers (default: no)"),
+ instrument_moves_size_limit: Option<u64> = (None, parse_opt_number, [TRACKED],
+ "the minimum size object to instrument move/copy operations \
+ (default: 65 bytes)"),
  instrument_xray: Option<InstrumentXRay> = (None, parse_instrument_xray, [TRACKED],
  "insert function instrument code for XRay-based tracing (default: no)
  Optional extra settings:

compiler/rustc_span/src/symbol.rs

@@ -691,7 +691,9 @@ symbols! {
  compile_error,
  compiler,
  compiler_builtins,
+ compiler_copy,
  compiler_fence,
+ compiler_move,
  concat,
  concat_bytes,
  concat_idents,

library/core/src/lib.rs

@@ -279,6 +279,7 @@ pub mod num;
 pub mod hint;
 pub mod intrinsics;
 pub mod mem;
+pub mod profiling;
 pub mod ptr;
 #[unstable(feature = "ub_checks", issue = "none")]
 pub mod ub_checks;

library/core/src/profiling.rs

@@ -0,0 +1,29 @@
+//! Profiling markers for compiler instrumentation.
+
+/// Profiling marker for move operations.
+///
+/// This function is never called at runtime. When `-Z instrument-moves` is enabled,
+/// the compiler creates synthetic debug info that makes move operations appear as
+/// calls to this function in profilers.
+///
+/// The `SIZE` parameter encodes the size of the type being moved.
+#[unstable(feature = "profiling_marker_api", issue = "none")]
+#[rustc_force_inline]
+#[rustc_diagnostic_item = "compiler_move"]
+pub fn compiler_move<T, const SIZE: usize>(_src: *const T, _dst: *mut T) {
+ unreachable!("compiler_move should never be called - it's only for debug info")
+}
+
+/// Profiling marker for copy operations.
+///
+/// This function is never called at runtime. When `-Z instrument-moves` is enabled,
+/// the compiler creates synthetic debug info that makes copy operations appear as
+/// calls to this function in profilers.
+///
+/// The `SIZE` parameter encodes the size of the type being copied.
+#[unstable(feature = "profiling_marker_api", issue = "none")]
+#[rustc_force_inline]
+#[rustc_diagnostic_item = "compiler_copy"]
+pub fn compiler_copy<T, const SIZE: usize>(_src: *const T, _dst: *mut T) {
+ unreachable!("compiler_copy should never be called - it's only for debug info")
+}