Add progress info to LeaderRetentionManager::delete_all_no_checkpoint (#42576)

GitOrigin-RevId: f30801c6f5bc38380f3451268917f32ad2b3e452

Author: goffrie

crates/database/src/retention.rs

@@ -12,7 +12,10 @@ use std::{
  Hasher,
  },
  sync::Arc,
- time::Duration,
+ time::{
+ Duration,
+ Instant,
+ },
 };
 
 use anyhow::Context;
@@ -723,6 +726,7 @@ impl<RT: Runtime> LeaderRetentionManager<RT> {
  all_indexes: &BTreeMap<IndexId, (GenericIndexName<TabletId>, IndexedFields)>,
  retention_validator: Arc<dyn RetentionValidator>,
  ) -> anyhow::Result<()> {
+ let mut last_logged = Instant::now();
  while cursor_ts.succ()? < *min_snapshot_ts {
  let (new_cursor_ts, _) = Self::delete(
  min_snapshot_ts,
@@ -732,10 +736,16 @@ impl<RT: Runtime> LeaderRetentionManager<RT> {
  retention_validator.clone(),
  )
  .await?;
+ let now = Instant::now();
+ let duration = now.saturating_duration_since(last_logged).as_secs_f64();
+ let catchup_rate = new_cursor_ts.secs_since_f64(*cursor_ts) / duration;
+ let lag = min_snapshot_ts.secs_since_f64(*new_cursor_ts);
  tracing::info!(
- "custom index retention completed between ts {cursor_ts} and {new_cursor_ts}"
+ "custom index retention completed between ts {cursor_ts} and {new_cursor_ts}; \
+ catchup rate: {catchup_rate:.1}s/s, {lag:.1}s behind snapshot"
  );
  cursor_ts = new_cursor_ts;
+ last_logged = now;
  }
  Ok(())
  }