[CNS] add config snapshot event metrics at an interval (Azure#3140)

* feat: add and expose new ConfigSnapshotIntervalInMins config * feat: add interval event emitting of CNS config snapshot * lint: address lint errors * chore: send CNS config snapshot immediately * test: refactor test to be less coupled * lint * fix: remove setting default ConfigSnapshotIntervalInMins * chore: remove unnecessary comment

Author: jackieluc

cns/configuration/cns_config.json

@@ -5,6 +5,7 @@
  "HeartBeatIntervalInMins": 30,
  "RefreshIntervalInSecs": 15,
  "SnapshotIntervalInMins": 60,
+ "ConfigSnapshotIntervalInMins": 60,
  "TelemetryBatchIntervalInSecs": 15,
  "TelemetryBatchSizeBytes": 16384
  },

cns/configuration/configuration.go

@@ -80,6 +80,8 @@ type TelemetrySettings struct {
 DebugMode bool
 // Interval for sending snapshot events.
 SnapshotIntervalInMins int
+// Interval for sending config snapshot events.
+ConfigSnapshotIntervalInMins int
 // AppInsightsInstrumentationKey allows the user to override the default appinsights ikey
 AppInsightsInstrumentationKey string
 }

cns/configuration/configuration_test.go

@@ -80,6 +80,7 @@ func TestReadConfigFromFile(t *testing.T) {
 HeartBeatIntervalInMins: 30,
 RefreshIntervalInSecs: 15,
 SnapshotIntervalInMins: 60,
+ConfigSnapshotIntervalInMins: 60,
 TelemetryBatchIntervalInSecs: 15,
 TelemetryBatchSizeBytes: 16384,
 },

cns/configuration/testdata/good.json

@@ -26,6 +26,7 @@
  "HeartBeatIntervalInMins": 30,
  "RefreshIntervalInSecs": 15,
  "SnapshotIntervalInMins": 60,
+ "ConfigSnapshotIntervalInMins": 60,
  "TelemetryBatchIntervalInSecs": 15,
  "TelemetryBatchSizeBytes": 16384
  },

cns/logger/constants.go

@@ -3,17 +3,20 @@ package logger
 
 const (
 // Metrics
-HeartBeatMetricStr = "HeartBeat"
+HeartBeatMetricStr = "HeartBeat"
+ConfigSnapshotMetricsStr = "ConfigSnapshot"
 
 // Dimensions
-OrchestratorTypeStr = "OrchestratorType"
-NodeIDStr = "NodeID"
-HomeAZStr = "HomeAZ"
-IsAZRSupportedStr = "IsAZRSupported"
-HomeAZErrorCodeStr = "HomeAZErrorCode"
-HomeAZErrorMsgStr = "HomeAZErrorMsg"
+OrchestratorTypeStr = "OrchestratorType"
+NodeIDStr = "NodeID"
+HomeAZStr = "HomeAZ"
+IsAZRSupportedStr = "IsAZRSupported"
+HomeAZErrorCodeStr = "HomeAZErrorCode"
+HomeAZErrorMsgStr = "HomeAZErrorMsg"
+CNSConfigPropertyStr = "CNSConfiguration"
+CNSConfigMD5CheckSumPropertyStr = "CNSConfigurationMD5Checksum"
 
-// CNS Snspshot properties
+// CNS NC Snspshot properties
 CnsNCSnapshotEventStr = "CNSNCSnapshot"
 IpConfigurationStr = "IPConfiguration"
 LocalIPConfigurationStr = "LocalIPConfiguration"

cns/metric/configsnapshot.go

@@ -0,0 +1,58 @@
+package metric
+
+import (
+"context"
+"crypto/md5" //nolint:gosec // used for checksum
+"encoding/json"
+"time"
+
+"github.com/Azure/azure-container-networking/aitelemetry"
+"github.com/Azure/azure-container-networking/cns/configuration"
+"github.com/Azure/azure-container-networking/cns/logger"
+"github.com/pkg/errors"
+)
+
+// SendCNSConfigSnapshot emits CNS config periodically
+func SendCNSConfigSnapshot(ctx context.Context, config *configuration.CNSConfig) {
+ticker := time.NewTicker(time.Minute * time.Duration(config.TelemetrySettings.ConfigSnapshotIntervalInMins))
+defer ticker.Stop()
+
+event, err := createCNSConfigSnapshotEvent(config)
+if err != nil {
+logger.Errorf("[Azure CNS] SendCNSConfigSnapshot: %v", err)
+return
+}
+
+// Log the first event immediately
+logger.LogEvent(event)
+
+for {
+select {
+case <-ctx.Done():
+return
+case <-ticker.C:
+logger.LogEvent(event)
+}
+}
+}
+
+func createCNSConfigSnapshotEvent(config *configuration.CNSConfig) (aitelemetry.Event, error) {
+bb, err := json.Marshal(config) //nolint:musttag // no tag needed for config
+if err != nil {
+return aitelemetry.Event{}, errors.Wrap(err, "failed to marshal config")
+}
+
+cs := md5.Sum(bb) //nolint:gosec // used for checksum
+csStr := string(cs[:])
+
+event := aitelemetry.Event{
+EventName: logger.ConfigSnapshotMetricsStr,
+ResourceID: csStr, // not guaranteed unique, instead use VM ID and Subscription to correlate
+Properties: map[string]string{
+logger.CNSConfigPropertyStr: string(bb),
+logger.CNSConfigMD5CheckSumPropertyStr: csStr,
+},
+}
+
+return event, nil
+}

cns/metric/configsnapshot_test.go

@@ -0,0 +1,30 @@
+package metric
+
+import (
+"encoding/json"
+"testing"
+
+"github.com/Azure/azure-container-networking/cns/configuration"
+"github.com/Azure/azure-container-networking/cns/logger"
+"github.com/stretchr/testify/assert"
+"github.com/stretchr/testify/require"
+)
+
+func TestCreateCNSConfigSnapshotEvent(t *testing.T) {
+logger.InitLogger("testlogs", 0, 0, "./")
+
+config, err := configuration.ReadConfig("../configuration/testdata/good.json")
+require.NoError(t, err)
+
+event, err := createCNSConfigSnapshotEvent(config)
+require.NoError(t, err)
+
+assert.Equal(t, logger.ConfigSnapshotMetricsStr, event.EventName)
+assert.NotEmpty(t, event.ResourceID)
+assert.Contains(t, event.Properties[logger.CNSConfigPropertyStr], "\"TLSPort\":\"10091\"")
+
+eventConfig := &configuration.CNSConfig{}
+err = json.Unmarshal([]byte(event.Properties[logger.CNSConfigPropertyStr]), eventConfig) //nolint:musttag // no tag needed for config
+require.NoError(t, err)
+assert.EqualValues(t, config, eventConfig)
+}

cns/service/main.go

@@ -578,6 +578,10 @@ func main() {
 } else {
 logger.InitAI(aiConfig, ts.DisableTrace, ts.DisableMetric, ts.DisableEvent)
 }
+
+if cnsconfig.TelemetrySettings.ConfigSnapshotIntervalInMins > 0 {
+go metric.SendCNSConfigSnapshot(rootCtx, cnsconfig)
+}
 }
 logger.Printf("[Azure CNS] Using config: %+v", cnsconfig)