chore(internal logs): Disable rate limiting for critical internal error logs (#24192)

* Improve debug log in aws_ecs_metrics * Add internal_log_rate_limit = false to config reload errors * Disable rate limiting for lua build failure * Disable rate limiting for amqp build failure * Add internal_log_rate_limit = false to important failures during reload * Disable log rate limit for repeated events in process.rs * Update src/config/watcher.rs * Fix message format

Author: thomasqueirozb

src/config/watcher.rs

@@ -147,19 +147,38 @@ pub fn spawn_thread<'a>(
  .all(|(_, t)| *t == ComponentType::EnrichmentTable)
  {
  info!("Only enrichment tables have changed.");
- _ = signal_tx.send(crate::signal::SignalTo::ReloadEnrichmentTables).map_err(|error| {
- error!(message = "Unable to reload enrichment tables.", cause = %error)
- });
+ _ = signal_tx
+ .send(crate::signal::SignalTo::ReloadEnrichmentTables)
+ .map_err(|error| {
+ error!(
+ message = "Unable to reload enrichment tables.",
+ cause = %error,
+ internal_log_rate_limit = false,
+ )
+ });
  } else {
- _ = signal_tx.send(crate::signal::SignalTo::ReloadComponents(changed_components.into_keys().collect())).map_err(|error| {
- error!(message = "Unable to reload component configuration. Restart Vector to reload it.", cause = %error)
- });
+ _ = signal_tx
+ .send(crate::signal::SignalTo::ReloadComponents(
+ changed_components.into_keys().collect(),
+ ))
+ .map_err(|error| {
+ error!(
+ message = "Unable to reload component configuration. Restart Vector to reload it.",
+ cause = %error,
+ internal_log_rate_limit = false,
+ )
+ });
  }
  } else {
- _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk)
- .map_err(|error| {
- error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error)
- });
+ _ = signal_tx
+ .send(crate::signal::SignalTo::ReloadFromDisk)
+ .map_err(|error| {
+ error!(
+ message = "Unable to reload configuration file. Restart Vector to reload it.",
+ cause = %error,
+ internal_log_rate_limit = false,
+ )
+ });
  }
  } else {
  debug!(message = "Ignoring event.", event = ?event)

src/internal_events/aws_ecs_metrics.rs

@@ -52,8 +52,9 @@ impl InternalEvent for AwsEcsMetricsParseError<'_> {
  error_type = error_type::PARSER_FAILED,
  );
  debug!(
- message = %format!("Failed to parse response:\\n\\n{}\\n\\n", self.body.escape_debug()),
+ response = %self.body.escape_debug(),
  endpoint = %self.endpoint,
+ "Failed to parse response.",
  );
  counter!("parse_errors_total").increment(1);
  counter!(

src/internal_events/config.rs

@@ -14,6 +14,7 @@ impl InternalEvent for ConfigReloadRejected {
  message = "Config reload rejected due to non-reloadable global options.",
  reason = %self.reason.as_str(),
  changed_fields = %fields.join(", "),
+ internal_log_rate_limit = false,
  );
 
  counter!(
@@ -27,6 +28,7 @@ impl InternalEvent for ConfigReloadRejected {
  message = "Config reload rejected due to failed to compute global diff.",
  reason = %self.reason.as_str(),
  error = %err,
+ internal_log_rate_limit = false,
  );
 
  counter!(

src/internal_events/lua.rs

@@ -58,6 +58,7 @@ impl InternalEvent for LuaBuildError {
  error_type = error_type::SCRIPT_FAILED,
  error_code = lua_build_error_code(&self.error),
  stage = error_stage::PROCESSING,
+ internal_log_rate_limit = false,
  );
  counter!(
  "component_errors_total",

src/internal_events/process.rs

@@ -30,7 +30,8 @@ impl InternalEvent for VectorReloaded<'_> {
  info!(
  target: "vector",
  message = "Vector has reloaded.",
- path = ?self.config_paths
+ path = ?self.config_paths,
+ internal_log_rate_limit = false,
  );
  counter!("reloaded_total").increment(1);
  }
@@ -43,7 +44,7 @@ impl InternalEvent for VectorStopped {
  fn emit(self) {
  info!(
  target: "vector",
- message = "Vector has stopped."
+ message = "Vector has stopped.",
  );
  counter!("stopped_total").increment(1);
  }
@@ -56,7 +57,7 @@ impl InternalEvent for VectorQuit {
  fn emit(self) {
  info!(
  target: "vector",
- message = "Vector has quit."
+ message = "Vector has quit.",
  );
  counter!("quit_total").increment(1);
  }
@@ -72,6 +73,7 @@ impl InternalEvent for VectorReloadError {
  error_code = "reload",
  error_type = error_type::CONFIGURATION_FAILED,
  stage = error_stage::PROCESSING,
+ internal_log_rate_limit = false,
  );
  counter!(
  "component_errors_total",
@@ -93,6 +95,7 @@ impl InternalEvent for VectorConfigLoadError {
  error_code = "config_load",
  error_type = error_type::CONFIGURATION_FAILED,
  stage = error_stage::PROCESSING,
+ internal_log_rate_limit = false,
  );
  counter!(
  "component_errors_total",
@@ -114,6 +117,7 @@ impl InternalEvent for VectorRecoveryError {
  error_code = "recovery",
  error_type = error_type::CONFIGURATION_FAILED,
  stage = error_stage::PROCESSING,
+ internal_log_rate_limit = false,
  );
  counter!(
  "component_errors_total",

src/sinks/amqp/config.rs

@@ -45,6 +45,7 @@ impl AmqpPropertiesConfig {
  error = %error,
  error_type = error_type::TEMPLATE_FAILED,
  stage = error_stage::PROCESSING,
+ internal_log_rate_limit = false,
  );
  Default::default()
  });