Support policy tests in input packages with input type otelcol

internal/stack/_static/kibana.yml.tmpl

@@ -41,9 +41,18 @@ xpack.fleet.registryUrl: "https://package-registry:8080"
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.fleet_server.hosts: ["{{ fact "fleet_url" }}"]
 
-{{ if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }}
-xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI
-{{ end }}
+{{/* Feature flags introduced in 8.3.0 https://github.com/elastic/kibana/pull/130253 */}}
+{{- if (not (semverLessThan $version "8.3.0")) }}
+xpack.fleet.enableExperimental: [
+{{- if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }}
+ # Enable experimental toggles in Fleet UI
+ "experimentalDataStreamSettings"
+{{- else if (not (semverLessThan $version "9.2.0-SNAPSHOT")) }}
+ # Support for otelcol input
+ "enableOtelIntegrations"
+{{- end }}
+]
+{{- end }}
 
 xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 

internal/testrunner/runners/policy/policy.go

@@ -5,6 +5,7 @@
 package policy
 
 import (
+"bufio"
 "bytes"
 "context"
 "errors"
@@ -13,13 +14,15 @@ import (
 "path/filepath"
 "regexp"
 "slices"
+"strconv"
 "strings"
 
 "github.com/pmezard/go-difflib/difflib"
 "gopkg.in/yaml.v3"
 
 "github.com/elastic/elastic-package/internal/common"
 "github.com/elastic/elastic-package/internal/kibana"
+"github.com/elastic/elastic-package/internal/logger"
 )
 
 func dumpExpectedAgentPolicy(ctx context.Context, kibanaClient *kibana.Client, testPath string, policyID string) error {
@@ -71,6 +74,9 @@ func comparePolicies(expected, found []byte) (string, error) {
 if err != nil {
 return "", fmt.Errorf("failed to prepare found policy: %w", err)
 }
+logger.Tracef("expected policy after cleaning:\n%s", want)
+logger.Tracef("found policy after cleaning:\n%s", got)
+
 if bytes.Equal(want, got) {
 return "", nil
 }
@@ -166,10 +172,35 @@ var policyEntryFilters = []policyEntryFilter{
 }},
 }
 
+var uniqueOtelComponentIDReplace = policyEntryReplace{
+regexp: regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s\{\}|\s*)$`),
+replace: "$1$2/componentid-%s:$4",
+}
+
+// otelComponentIDsRegexp is the regex to find otel components sections and their IDs to replace them with controlled values.
+// It matches sections like:
+//
+// extensions:
+// health_check/4391d954-1ffe-4014-a256-5eda78a71828: {}
+//
+// receivers:
+// httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7:
+// collection_interval: 1m
+// targets:
+// - endpoints:
+// - https://epr.elastic.co
+// method: GET
+var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|connectors|exporters):(?:\s\{\}\n|\n(?:\s{2,}.+\n)+)`)
+
 // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other
 // policies. This preparation is based on removing contents that are generated, or replace them
 // by controlled values.
 func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, error) {
+// Replacement of the OTEL component IDs needs to be done before unmarshalling the YAML.
+// The OTEL IDs are keys in maps, and using the policyEntryFilter with memberReplace does
+// not ensure to keep the same ordering.
+policy = replaceOtelComponentIDs(policy)
+
 var policyMap common.MapStr
 err := yaml.Unmarshal(policy, &policyMap)
 if err != nil {
@@ -184,6 +215,47 @@ func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, err
 return yaml.Marshal(policyMap)
 }
 
+// replaceOtelComponentIDs finds OTel Collector component IDs in the policy and replaces them with controlled values.
+// It also replaces references to those IDs in service.extensions and service.pipelines.
+func replaceOtelComponentIDs(policy []byte) []byte {
+replacementsDone := map[string]string{}
+
+policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte {
+count := 0
+scanner := bufio.NewScanner(bytes.NewReader(match))
+var section strings.Builder
+for scanner.Scan() {
+line := scanner.Text()
+if uniqueOtelComponentIDReplace.regexp.MatchString(line) {
+originalOtelID, _, _ := strings.Cut(strings.TrimSpace(line), ":")
+
+replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count))
+count++
+line = uniqueOtelComponentIDReplace.regexp.ReplaceAllString(line, replacement)
+
+// store the otel ID replaced without the space indentation and the colon to be replaced later
+// (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0)
+replacementsDone[originalOtelID], _, _ = strings.Cut(strings.TrimSpace(string(line)), ":")
+}
+section.WriteString(line + "\n")
+}
+
+return []byte(section.String())
+})
+
+// Replace references in arrays to the otel component IDs replaced before.
+// These references can be in:
+// service.extensions
+// service.pipelines.<signal>.(receivers|processors|exporters)
+for original, replacement := range replacementsDone {
+originalArrayItem := fmt.Sprintf("- %s", original)
+replacedArrayItem := fmt.Sprintf("- %s", replacement)
+
+policy = bytes.ReplaceAll(policy, []byte(originalArrayItem), []byte(replacedArrayItem))
+}
+return policy
+}
+
 func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (common.MapStr, error) {
 for _, entry := range entries {
 v, err := policyMap.GetValue(entry.name)
@@ -218,10 +290,14 @@ func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (commo
 if !ok {
 return nil, fmt.Errorf("expected map, found %T", v)
 }
+regexp := entry.memberReplace.regexp
+replacement := entry.memberReplace.replace
 for k, e := range m {
-if entry.memberReplace.regexp.MatchString(k) {
+key := k
+if regexp.MatchString(k) {
 delete(m, k)
-m[entry.memberReplace.replace] = e
+key = regexp.ReplaceAllString(k, replacement)
+m[key] = e
 }
 }
 default:

internal/testrunner/runners/policy/policy_test.go

@@ -121,7 +121,7 @@ output_permissions:
  - monitor
  _elastic_agent_monitoring:
  indices: []
- uuid-for-permissions-on-related-indices:
+ 8d024b11-4e82-4192-8e7f-be71d1b13aac:
  indices:
  - names:
  - metrics-*-*
@@ -221,7 +221,7 @@ output_permissions:
  - monitor
  _elastic_agent_monitoring:
  indices: []
- uuid-for-permissions-on-related-indices:
+ bfe4f402-df02-4673-8a71-fd5b29f1e2f3:
  indices:
  - names:
  - metrics-*-*
@@ -285,6 +285,194 @@ secret_references:
 `,
 equal: false,
 },
+{
+title: "otel ids",
+expected: `
+inputs: []
+output_permissions:
+ default:
+ _elastic_agent_checks:
+ cluster:
+ - monitor
+ _elastic_agent_monitoring:
+ indices: []
+ 05c98f91-203c-44a9-bee7-dd621c9bd37e:
+ indices:
+ - names:
+ - logs-*-*
+ privileges:
+ - auto_configure
+ - create_doc
+extensions:
+ health_check/31c94f44-214a-4778-8a36-acc2634096f7: {}
+processors:
+ batch/11c35ad0-4351-49d4-9c78-fa679ce9d950:
+ send_batch_size: 10
+ timeout: 1s
+ batch/e6e379c5-6446-4090-af10-a9e5f8fc4640:
+ send_batch_size: 10000
+ timeout: 10s
+receivers:
+ httpcheck/4bae34b3-8f66-49c1-b04f-d58af1b5f743:
+ collection_interval: 1m
+ targets:
+ - endpoints:
+ - https://epr.elastic.co
+ method: GET
+secret_references: []
+service:
+ extensions:
+ - health_check/31c94f44-214a-4778-8a36-acc2634096f7
+ pipelines:
+ logs:
+ receivers:
+ - httpcheck/4bae34b3-8f66-49c1-b04f-d58af1b5f743
+ processors:
+ - batch/11c35ad0-4351-49d4-9c78-fa679ce9d950
+ - batch/e6e379c5-6446-4090-af10-a9e5f8fc4640
+
+`,
+found: `
+inputs: []
+output_permissions:
+ default:
+ _elastic_agent_checks:
+ cluster:
+ - monitor
+ aeb4d606-2d90-4b41-b231-27bfad6dea09:
+ indices:
+ - names:
+ - logs-*-*
+ privileges:
+ - auto_configure
+ - create_doc
+ _elastic_agent_monitoring:
+ indices: []
+extensions:
+ health_check/4391d954-1ffe-4014-a256-5eda78a71829: {}
+processors:
+ batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b:
+ send_batch_size: 10
+ timeout: 1s
+ batch/8ec6ee99-2176-4231-9668-908069c77784:
+ send_batch_size: 10000
+ timeout: 10s
+receivers:
+ httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7:
+ collection_interval: 1m
+ targets:
+ - endpoints:
+ - https://epr.elastic.co
+ method: GET
+secret_references: []
+service:
+ extensions:
+ - health_check/4391d954-1ffe-4014-a256-5eda78a71829
+ pipelines:
+ logs:
+ receivers:
+ - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7
+ processors:
+ - batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b
+ - batch/8ec6ee99-2176-4231-9668-908069c77784
+
+`,
+equal: true,
+},
+{
+title: "otel hardcode expected ids",
+expected: `
+inputs: []
+output_permissions:
+ default:
+ _elastic_agent_checks:
+ cluster:
+ - monitor
+ _elastic_agent_monitoring:
+ indices: []
+ 05c98f91-203c-44a9-bee7-dd621c9bd37e:
+ indices:
+ - names:
+ - logs-*-*
+ privileges:
+ - auto_configure
+ - create_doc
+extensions:
+ health_check/componentid-0: {}
+processors:
+ batch/componentid-0:
+ send_batch_size: 10
+ timeout: 1s
+ batch/componentid-1:
+ send_batch_size: 10000
+ timeout: 10s
+receivers:
+ httpcheck/componentid-0:
+ collection_interval: 1m
+ targets:
+ - endpoints:
+ - https://epr.elastic.co
+ method: GET
+secret_references: []
+service:
+ extensions:
+ - health_check/componentid-0
+ pipelines:
+ logs:
+ receivers:
+ - httpcheck/componentid-0
+ processors:
+ - batch/componentid-0
+ - batch/componentid-1
+
+`,
+found: `
+inputs: []
+output_permissions:
+ default:
+ _elastic_agent_checks:
+ cluster:
+ - monitor
+ aeb4d606-2d90-4b41-b231-27bfad6dea09:
+ indices:
+ - names:
+ - logs-*-*
+ privileges:
+ - auto_configure
+ - create_doc
+ _elastic_agent_monitoring:
+ indices: []
+extensions:
+ health_check/4391d954-1ffe-4014-a256-5eda78a71828: {}
+processors:
+ batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b:
+ send_batch_size: 10
+ timeout: 1s
+ batch/8ec6ee99-2176-4231-9668-908069c77784:
+ send_batch_size: 10000
+ timeout: 10s
+receivers:
+ httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7:
+ collection_interval: 1m
+ targets:
+ - endpoints:
+ - https://epr.elastic.co
+ method: GET
+secret_references: []
+service:
+ extensions:
+ - health_check/4391d954-1ffe-4014-a256-5eda78a71828
+ pipelines:
+ logs:
+ receivers:
+ - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7
+ processors:
+ - batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b
+ - batch/8ec6ee99-2176-4231-9668-908069c77784
+
+`,
+equal: true,
+},
 }
 
 for _, c := range cases {

test/packages/parallel/httpcheck.stack_version

@@ -0,0 +1 @@
+9.2.0-SNAPSHOT