Tell collector to watch the .log.1 files to avoid missing records around rotation time.

Add comments around each components filelog receiver settings.

Author: dsessler7

internal/collector/patroni.go

@@ -39,7 +39,14 @@ func EnablePatroniLogging(ctx context.Context,
 // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 outConfig.Receivers["filelog/patroni_jsonlog"] = map[string]any{
 // Read the JSON files and keep track of what has been processed.
-"include": []string{directory + "/*.log"},
+// When patroni rotates its log files, it renames the old .log file
+// to .log.1. We want the collector to ingest logs from both files
+// as it is possible that patroni will continue to write a log
+// record or two to the old file while rotation is occurring. The
+// collector knows not to create duplicate logs.
+"include": []string{
+directory + "/*.log", directory + "/*.log.1",
+},
 "storage": "file_storage/patroni_logs",
 
 "operators": []map[string]any{

internal/collector/patroni_test.go

@@ -88,6 +88,7 @@ receivers:
  filelog/patroni_jsonlog:
  include:
  - /pgdata/patroni/log/*.log
+ - /pgdata/patroni/log/*.log.1
  operators:
  - from: body
  to: body.original
@@ -183,6 +184,7 @@ receivers:
  filelog/patroni_jsonlog:
  include:
  - /pgdata/patroni/log/*.log
+ - /pgdata/patroni/log/*.log.1
  operators:
  - from: body
  to: body.original

internal/collector/pgadmin.go

@@ -29,6 +29,11 @@ func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec
 "fsync": true,
 }
 
+// PgAdmin/gunicorn logs are rotated by python -- python tries to emit a log
+// and if the file needs to rotate, it rotates first and then emits the log.
+// The collector therefore only needs to watch the single active log for
+// each component.
+// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 otelConfig.Receivers["filelog/pgadmin"] = map[string]any{
 "include": []string{"/var/lib/pgadmin/logs/pgadmin.log"},
 "storage": "file_storage/pgadmin_data_logs",

internal/collector/pgbackrest.go

@@ -55,8 +55,13 @@ func NewConfigForPgBackrestRepoHostPod(
 // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 config.Receivers["filelog/pgbackrest_log"] = map[string]any{
 // Read the files and keep track of what has been processed.
+// We use logrotate to rotate the pgbackrest logs which renames the
+// old .log file to .log.1. We want the collector to ingest logs from
+// both files as it is possible that pgbackrest will continue to write
+// a log record or two to the old file while rotation is occurring.
+// The collector knows not to create duplicate logs.
 "include": []string{
-directory + "/*.log",
+directory + "/*.log", directory + "/*.log.1",
 },
 "storage": "file_storage/pgbackrest_logs",
 // pgBackRest prints logs with a log prefix, which includes a timestamp

internal/collector/pgbackrest_test.go

@@ -95,6 +95,7 @@ receivers:
  filelog/pgbackrest_log:
  include:
  - /pgbackrest/repo1/log/*.log
+ - /pgbackrest/repo1/log/*.log.1
  multiline:
  line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
  storage: file_storage/pgbackrest_logs
@@ -195,6 +196,7 @@ receivers:
  filelog/pgbackrest_log:
  include:
  - /pgbackrest/repo1/log/*.log
+ - /pgbackrest/repo1/log/*.log.1
  multiline:
  line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
  storage: file_storage/pgbackrest_logs

internal/collector/postgres.go

@@ -142,6 +142,7 @@ func EnablePostgresLogging(
 // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 outConfig.Receivers["filelog/postgres_csvlog"] = map[string]any{
 // Read the CSV files and keep track of what has been processed.
+// The wildcard covers all potential log file names.
 "include": []string{directory + "/*.csv"},
 "storage": "file_storage/postgres_logs",
 
@@ -173,6 +174,7 @@ func EnablePostgresLogging(
 // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 outConfig.Receivers["filelog/postgres_jsonlog"] = map[string]any{
 // Read the JSON files and keep track of what has been processed.
+// The wildcard covers all potential log file names.
 "include": []string{directory + "/*.json"},
 "storage": "file_storage/postgres_logs",
 
@@ -238,8 +240,17 @@ func EnablePostgresLogging(
 "fsync": true,
 }
 
+// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 outConfig.Receivers["filelog/pgbackrest_log"] = map[string]any{
-"include": []string{naming.PGBackRestPGDataLogPath + "/*.log"},
+// We use logrotate to rotate the pgbackrest logs which renames the
+// old .log file to .log.1. We want the collector to ingest logs from
+// both files as it is possible that pgbackrest will continue to write
+// a log record or two to the old file while rotation is occurring.
+// The collector knows not to create duplicate logs.
+"include": []string{
+naming.PGBackRestPGDataLogPath + "/*.log",
+naming.PGBackRestPGDataLogPath + "/*.log.1",
+},
 "storage": "file_storage/pgbackrest_logs",
 
 // pgBackRest prints logs with a log prefix, which includes a timestamp

internal/collector/postgres_test.go

@@ -197,6 +197,7 @@ receivers:
  filelog/pgbackrest_log:
  include:
  - /pgdata/pgbackrest/log/*.log
+ - /pgdata/pgbackrest/log/*.log.1
  multiline:
  line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
  storage: file_storage/pgbackrest_logs
@@ -438,6 +439,7 @@ receivers:
  filelog/pgbackrest_log:
  include:
  - /pgdata/pgbackrest/log/*.log
+ - /pgdata/pgbackrest/log/*.log.1
  multiline:
  line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
  storage: file_storage/pgbackrest_logs