CLOUDP-127672 - add logRotate support for automation (#1363)

Author: nammn

api/v1/mongodbcommunity_types.go

@@ -130,7 +130,7 @@ type MongoDBCommunitySpec struct {
 AdditionalConnectionStringConfig MapWrapper `json:"additionalConnectionStringConfig,omitempty"`
 }
 
-// Wrapper for a map to be used by other structs.
+// MapWrapper is a wrapper for a map to be used by other structs.
 // The CRD generator does not support map[string]interface{}
 // on the top level and hence we need to work around this with
 // a wrapping struct.
@@ -327,8 +327,9 @@ type AutomationConfigOverride struct {
 
 // OverrideProcess contains fields that we can override on the AutomationConfig processes.
 type OverrideProcess struct {
-Name string `json:"name"`
-Disabled bool `json:"disabled"`
+Name string `json:"name"`
+Disabled bool `json:"disabled"`
+LogRotate *automationconfig.CrdLogRotate `json:"logRotate,omitempty"`
 }
 
 // StatefulSetConfiguration holds the optional custom StatefulSet
@@ -355,6 +356,12 @@ type AgentConfiguration struct {
 LogLevel LogLevel `json:"logLevel"`
 // +optional
 MaxLogFileDurationHours int `json:"maxLogFileDurationHours"`
+// +optional
+// LogRotate if enabled, will enable LogRotate for all processes.
+LogRotate *automationconfig.CrdLogRotate `json:"logRotate,omitempty"`
+// +optional
+// SystemLog configures system log of mongod
+SystemLog *automationconfig.SystemLog `json:"systemLog,omitempty"`
 }
 
 // StatefulSetSpecWrapper is a wrapper around StatefulSetSpec with a custom implementation

api/v1/zz_generated.deepcopy.go

@@ -71,8 +71,54 @@ spec:
  properties:
  logLevel:
  type: string
+ logRotate:
+ description: LogRotate if enabled, will enable LogRotate for all
+ processes.
+ properties:
+ includeAuditLogsWithMongoDBLogs:
+ description: set to 'true' to have the Automation Agent rotate
+ the audit files along with mongodb log files
+ type: boolean
+ numTotal:
+ description: maximum number of log files to have total
+ type: integer
+ numUncompressed:
+ description: maximum number of log files to leave uncompressed
+ type: integer
+ percentOfDiskspace:
+ description: Maximum percentage of the total disk space these
+ log files should take up. The string needs to be able to
+ be converted to float64
+ type: string
+ sizeThresholdMB:
+ description: Maximum size for an individual log file before
+ rotation. The string needs to be able to be converted to
+ float64. Fractional values of MB are supported.
+ type: string
+ timeThresholdHrs:
+ description: maximum hours for an individual log file before
+ rotation
+ type: integer
+ required:
+ - sizeThresholdMB
+ - timeThresholdHrs
+ type: object
  maxLogFileDurationHours:
  type: integer
+ systemLog:
+ description: SystemLog configures system log of mongod
+ properties:
+ destination:
+ type: string
+ logAppend:
+ type: boolean
+ path:
+ type: string
+ required:
+ - destination
+ - logAppend
+ - path
+ type: object
  type: object
  arbiters:
  description: 'Arbiters is the number of arbiters to add to the Replica
@@ -91,6 +137,40 @@ spec:
  properties:
  disabled:
  type: boolean
+ logRotate:
+ description: CrdLogRotate is the crd definition of LogRotate
+ including fields in strings while the agent supports them
+ as float64
+ properties:
+ includeAuditLogsWithMongoDBLogs:
+ description: set to 'true' to have the Automation Agent
+ rotate the audit files along with mongodb log files
+ type: boolean
+ numTotal:
+ description: maximum number of log files to have total
+ type: integer
+ numUncompressed:
+ description: maximum number of log files to leave uncompressed
+ type: integer
+ percentOfDiskspace:
+ description: Maximum percentage of the total disk space
+ these log files should take up. The string needs to
+ be able to be converted to float64
+ type: string
+ sizeThresholdMB:
+ description: Maximum size for an individual log file
+ before rotation. The string needs to be able to be
+ converted to float64. Fractional values of MB are
+ supported.
+ type: string
+ timeThresholdHrs:
+ description: maximum hours for an individual log file
+ before rotation
+ type: integer
+ required:
+ - sizeThresholdMB
+ - timeThresholdHrs
+ type: object
  name:
  type: string
  required:

config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml

@@ -318,6 +318,17 @@ func TestAutomationConfigIsCorrectlyConfiguredWithTLS(t *testing.T) {
 }
 })
 
+t.Run("With logRotate and SystemLog enabled", func(t *testing.T) {
+mdb := newTestReplicaSetWithSystemLogAndLogRotate()
+ac := createAC(mdb)
+
+for _, process := range ac.Processes {
+assert.Equal(t, "/tmp/test", process.Args26.Get("systemLog.path").String())
+assert.Equal(t, "file", process.Args26.Get("systemLog.destination").String())
+assert.Equal(t, process.LogRotate, automationconfig.ConvertCrdLogRotateToAC(mdb.Spec.AgentConfiguration.LogRotate))
+}
+})
+
 t.Run("With TLS enabled and required, rollout completed", func(t *testing.T) {
 mdb := newTestReplicaSetWithTLS()
 ac := createAC(mdb)

controllers/mongodb_tls_test.go

@@ -557,6 +557,9 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut
 SetDataDir(mdb.GetMongodConfiguration().GetDBDataDir()).
 AddModifications(getMongodConfigModification(mdb)).
 AddModifications(modifications...).
+AddProcessModification(func(_ int, p *automationconfig.Process) {
+automationconfig.ConfigureAgentConfiguration(mdb.Spec.AgentConfiguration.SystemLog, mdb.Spec.AgentConfiguration.LogRotate, p)
+}).
 Build()
 }
 
@@ -703,24 +706,25 @@ func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity)
 }
 
 if mdb.Spec.AutomationConfigOverride != nil {
-automationConfig = merge.AutomationConfigs(automationConfig, overrideToAutomationConfig(*mdb.Spec.AutomationConfigOverride))
+automationConfig = merge.AutomationConfigs(automationConfig, OverrideToAutomationConfig(*mdb.Spec.AutomationConfigOverride))
 }
 
 return automationConfig, nil
 }
 
-// overrideToAutomationConfig turns an automation config override from the resource spec into an automation config
+// OverrideToAutomationConfig turns an automation config override from the resource spec into an automation config
 // which can be used to merge.
-func overrideToAutomationConfig(override mdbv1.AutomationConfigOverride) automationconfig.AutomationConfig {
+func OverrideToAutomationConfig(override mdbv1.AutomationConfigOverride) automationconfig.AutomationConfig {
 var processes []automationconfig.Process
-for _, p := range override.Processes {
-processes = append(processes, automationconfig.Process{
-Name: p.Name,
-Disabled: p.Disabled,
-})
+for _, o := range override.Processes {
+p := automationconfig.Process{
+Name: o.Name,
+Disabled: o.Disabled,
+LogRotate: automationconfig.ConvertCrdLogRotateToAC(o.LogRotate),
+}
+processes = append(processes, p)
 }
 
-// TODO: currently we are just merging processes. Other fields can be added here.
 return automationconfig.AutomationConfig{
 Processes: processes,
 }

controllers/replica_set_controller.go

@@ -68,6 +68,34 @@ func newTestReplicaSet() mdbv1.MongoDBCommunity {
 }
 }
 
+func newTestReplicaSetWithSystemLogAndLogRotate() mdbv1.MongoDBCommunity {
+return mdbv1.MongoDBCommunity{
+ObjectMeta: metav1.ObjectMeta{
+Name: "my-rs",
+Namespace: "my-ns",
+Annotations: map[string]string{},
+},
+Spec: mdbv1.MongoDBCommunitySpec{
+Members: 3,
+Version: "6.0.5",
+Security: mdbv1.Security{
+Authentication: mdbv1.Authentication{
+Modes: []mdbv1.AuthMode{"SCRAM"},
+},
+},
+AgentConfiguration: mdbv1.AgentConfiguration{
+LogRotate: &automationconfig.CrdLogRotate{
+SizeThresholdMB: "1",
+},
+SystemLog: &automationconfig.SystemLog{
+Destination: automationconfig.File,
+Path: "/tmp/test",
+},
+},
+},
+}
+}
+
 func newScramReplicaSet(users ...mdbv1.MongoDBUser) mdbv1.MongoDBCommunity {
 return mdbv1.MongoDBCommunity{
 ObjectMeta: metav1.ObjectMeta{

controllers/replicaset_controller_test.go

@@ -13,6 +13,8 @@
  - For these users, `scramCredentialsSecretName` and `passwordSecretRef` should **not** be set.
  - Sample resource [yaml](config/samples/mongodb.com_v1_mongodbcommunity_x509.yaml)
  - Sample agent certificate [yaml](config/samples/external_access/agent-certificate.yaml)
+ - Add support for configuring [logRotate](https://www.mongodb.com/docs/ops-manager/current/reference/cluster-configuration/#mongodb-instances) on the automation-agent. The settings can be found under `processes[n].logRotate.<setting>`.
+ - Additionally, [systemLog](https://www.mongodb.com/docs/manual/reference/configuration-options/#systemlog-options) can now be configured. In particular the settings: `path`, `destination` and `logAppend`.
 
 # MongoDB Kubernetes Operator 0.8.2