Set fullnameOverride and validate Otel kube-stack deployment datastreams are being written (metrics, traces) (#7754)

Author: rogercoll

changelog/fragments/1744384247-set-collector's-fullnameOverride-for-edot-kube-stack-values.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: set collectors fullnameOverride for edot kube-stack values
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/7754
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/7381

deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml

@@ -33,6 +33,7 @@ collectors:
  # Cluster is a K8s deployment EDOT collector focused on gathering telemetry
  # at the cluster level (Kubernetes Events and cluster metrics).
  cluster:
+ fullnameOverride: "opentelemetry-kube-stack-cluster-stats"
  env:
  - name: ELASTIC_AGENT_OTEL
  value: '"true"'
@@ -184,6 +185,7 @@ collectors:
  # node level and exposing an OTLP endpoint for data ingestion.
  # Auto-instrumentation SDKs will use this endpoint.
  daemon:
+ fullnameOverride: "opentelemetry-kube-stack-daemon"
  env:
  # Work around for open /mounts error: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35990
  - name: HOST_PROC_MOUNTINFO
@@ -526,6 +528,7 @@ collectors:
  # Gateway is a K8s deployment EDOT collector focused on processing and
  # forwarding telemetry to an Elasticsearch endpoint.
  gateway:
+ fullnameOverride: "opentelemetry-kube-stack-gateway"
  suffix: gateway
  replicas: 2
  autoscaler:

deploy/helm/edot-collector/kube-stack/values.yaml

@@ -33,6 +33,7 @@ collectors:
  # Cluster is a K8s deployment EDOT collector focused on gathering telemetry
  # at the cluster level (Kubernetes Events and cluster metrics).
  cluster:
+ fullnameOverride: "opentelemetry-kube-stack-cluster-stats"
  env:
  - name: ELASTIC_AGENT_OTEL
  value: '"true"'
@@ -184,6 +185,7 @@ collectors:
  # node level and exposing an OTLP endpoint for data ingestion.
  # Auto-instrumentation SDKs will use this endpoint.
  daemon:
+ fullnameOverride: "opentelemetry-kube-stack-daemon"
  env:
  # Work around for open /mounts error: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35990
  - name: HOST_PROC_MOUNTINFO
@@ -526,6 +528,7 @@ collectors:
  # Gateway is a K8s deployment EDOT collector focused on processing and
  # forwarding telemetry to an Elasticsearch endpoint.
  gateway:
+ fullnameOverride: "opentelemetry-kube-stack-gateway"
  resources:
  limits:
  cpu: 1500m

testing/integration/kubernetes_agent_standalone_test.go

@@ -29,6 +29,7 @@ import (
 "github.com/stretchr/testify/require"
 
 "github.com/elastic/elastic-agent-libs/kibana"
+"github.com/elastic/elastic-agent-libs/testing/estools"
 "github.com/elastic/go-elasticsearch/v8"
 
 appsv1 "k8s.io/api/apps/v1"
@@ -864,7 +865,8 @@ func k8sCheckAgentStatus(ctx context.Context, client klient.Client, stdout *byte
 
 // k8sGetAgentID returns the agent ID for the given agent pod
 func k8sGetAgentID(ctx context.Context, client klient.Client, stdout *bytes.Buffer, stderr *bytes.Buffer,
-namespace string, agentPodName string, containerName string) (string, error) {
+namespace string, agentPodName string, containerName string,
+) (string, error) {
 command := []string{"elastic-agent", "status", "--output=json"}
 
 status := atesting.AgentStatusOutput{} // clear status output
@@ -997,7 +999,7 @@ func k8sDumpPods(t *testing.T, ctx context.Context, client klient.Client, testNa
 header := &tar.Header{
 Name: logFileName,
 Size: int64(len(b)),
-Mode: 0600,
+Mode: 0o600,
 ModTime: time.Now(),
 AccessTime: time.Now(),
 ChangeTime: time.Now(),
@@ -1024,7 +1026,7 @@ func k8sDumpPods(t *testing.T, ctx context.Context, client klient.Client, testNa
 header := &tar.Header{
 Name: statesDumpFile,
 Size: int64(len(b)),
-Mode: 0600,
+Mode: 0o600,
 ModTime: time.Now(),
 AccessTime: time.Now(),
 ChangeTime: time.Now(),
@@ -1102,35 +1104,8 @@ func k8sRenderKustomize(kustomizePath string) ([]byte, error) {
 }
 
 // generateESAPIKey generates an API key for the given Elasticsearch.
-func generateESAPIKey(esClient *elasticsearch.Client, keyName string) (string, error) {
-apiKeyReqBody := fmt.Sprintf(`{
-"name": "%s",
-"expiration": "1d"
-}`, keyName)
-
-resp, err := esClient.Security.CreateAPIKey(strings.NewReader(apiKeyReqBody))
-if err != nil {
-return "", err
-}
-defer resp.Body.Close()
-
-response := make(map[string]interface{})
-err = json.NewDecoder(resp.Body).Decode(&response)
-if err != nil {
-return "", err
-}
-
-keyToken := response["api_key"].(string)
-if keyToken == "" {
-return "", fmt.Errorf("key token is empty")
-}
-
-keyID := response["id"].(string)
-if keyID == "" {
-return "", fmt.Errorf("key ID is empty")
-}
-
-return fmt.Sprintf("%s:%s", keyID, keyToken), nil
+func generateESAPIKey(esClient *elasticsearch.Client, keyName string) (estools.APIKeyResponse, error) {
+return estools.CreateAPIKey(context.Background(), esClient, estools.APIKeyRequest{Name: keyName, Expiration: "1d"})
 }
 
 // k8sDeleteOpts contains options for deleting k8s objects
@@ -1330,6 +1305,8 @@ type k8sContext struct {
 esHost string
 // esAPIKey is the API key of the elasticsearch to use in the test
 esAPIKey string
+// esEncodedAPIKey is the encoded API key of the elasticsearch to use in the test
+esEncodedAPIKey string
 // enrollParams contains the information needed to enroll an agent with Fleet in the test
 enrollParams *fleettools.EnrollParams
 // createdAt is the time when the k8sContext was created
@@ -1415,16 +1392,17 @@ func k8sGetContext(t *testing.T, info *define.Info) k8sContext {
 require.NoError(t, err, "failed to create fleet enroll params")
 
 return k8sContext{
-client: client,
-clientSet: clientSet,
-agentImage: agentImage,
-agentImageRepo: agentImageRepo,
-agentImageTag: agentImageTag,
-logsBasePath: testLogsBasePath,
-esHost: esHost,
-esAPIKey: esAPIKey,
-enrollParams: enrollParams,
-createdAt: time.Now(),
+client: client,
+clientSet: clientSet,
+agentImage: agentImage,
+agentImageRepo: agentImageRepo,
+agentImageTag: agentImageTag,
+logsBasePath: testLogsBasePath,
+esHost: esHost,
+esAPIKey: esAPIKey.APIKey,
+esEncodedAPIKey: esAPIKey.Encoded,
+enrollParams: enrollParams,
+createdAt: time.Now(),
 }
 }