add support for minimal field retention

The fields were identified by running the following shell script in the the security_detection_engine/kibana/security_rule directory. for f in *; do jq 'select(.attributes.required_fields != null)|.attributes.required_fields|.[]|select(.name != null)|select(.name|contains("cloudtrail.flattened"))|.name'<$f done|sort|uniq The test for this is derived from the test-copy-object-json.log test case which includes one of the required fields and a number of other fields under cloudtrail.flattened. So comparing the test added here to that demonstrates whether is works.

Author: efd6

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-copy-object-json-minimal.log

@@ -0,0 +1 @@
+{"eventVersion":"1.09","userIdentity":{"type":"IAMUser","principalId":"ACCESSKEYID","arn":"arn:aws:iam::000000000:user/test@elastic.co","accountId":"000000000","accessKeyId":"ACCESSKEYID","userName":"test@elastic.co"},"eventTime":"2024-10-08T12:24:16Z","eventSource":"s3.amazonaws.com","eventName":"CopyObject","awsRegion":"us-east-1","sourceIPAddress":"216.160.83.56","userAgent":"[aws-cli/2.17.60 md/awscrt#0.21.2 ua/2.0 os/macos#23.6.0 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython cfg/retry-mode#standard md/installer#exe md/prompt#off md/command#s3api.copy-object]","requestParameters":{"bucketName":"elastic-cspm-cloudtrail-test-bucket","Host":"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com","x-amz-copy-source":"elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md","key":"test-copy-object/README-copy.md"},"responseElements":{"x-amz-server-side-encryption":"AES256"},"additionalEventData":{"SignatureVersion":"SigV4","CipherSuite":"TLS_AES_128_GCM_SHA256","bytesTransferredIn":0,"SSEApplied":"Default_SSE_S3","AuthenticationMethod":"AuthHeader","x-amz-id-2":"hFhfe14yINVvz+alr1rC5zPFufFU087OGEbVwf5HpD1BYs5D2llscEUSD7DUGjlSYkOEoay+oVk=","bytesTransferredOut":224},"requestID":"62A9N2AH4P4YKG2B","eventID":"0c06e2ff-5e88-44e6-a081-57871bbe770b","readOnly":false,"resources":[{"type":"AWS::S3::Object","ARN":"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md"},{"accountId":"000000000","type":"AWS::S3::Bucket","ARN":"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket"},{"type":"AWS::S3::Object","ARN":"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md"}],"eventType":"AwsApiCall","recipientAccountId":"000000000","eventCategory":"Data","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com"}}

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-copy-object-json-minimal.log-config.yml

@@ -0,0 +1,13 @@
+dynamic_fields:
+ # This can be removed after ES 8.16.2 is set as the minimum version supported in the manifest.
+ # Once removed, it requires to update pipeline tests to remove the trailing dot where required.
+ # Relates: https://github.com/elastic/elasticsearch/pull/117213
+ "user_agent.version": '^\d+\.\d+(\.|\..*)?$'
+fields:
+ # Simulate @timestamp value from Filebeat.
+ '@timestamp': '2021-11-11T01:02:03.123456789Z'
+ tags:
+ - preserve_original_event
+ - actor_target_mapping
+ _conf:
+ retain: minimal

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-copy-object-json-minimal.log-expected.json

@@ -0,0 +1,159 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2024-10-08T12:24:16.000Z",
+ "actor": {
+ "entity": {
+ "id": [
+ "arn:aws:iam::000000000:user/test@elastic.co"
+ ]
+ }
+ },
+ "aws": {
+ "cloudtrail": {
+ "additional_eventdata": "{SignatureVersion=SigV4, CipherSuite=TLS_AES_128_GCM_SHA256, bytesTransferredIn=0, SSEApplied=Default_SSE_S3, AuthenticationMethod=AuthHeader, x-amz-id-2=hFhfe14yINVvz+alr1rC5zPFufFU087OGEbVwf5HpD1BYs5D2llscEUSD7DUGjlSYkOEoay+oVk=, bytesTransferredOut=224}",
+ "event_category": "Data",
+ "event_type": "AwsApiCall",
+ "event_version": "1.09",
+ "flattened": {
+ "additional_eventdata": {
+ "SSEApplied": "Default_SSE_S3"
+ }
+ },
+ "read_only": false,
+ "recipient_account_id": "000000000",
+ "request_id": "62A9N2AH4P4YKG2B",
+ "request_parameters": "{bucketName=elastic-cspm-cloudtrail-test-bucket, Host=elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com, x-amz-copy-source=elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md, key=test-copy-object/README-copy.md}",
+ "resources": [
+ {
+ "arn": "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md",
+ "type": "AWS::S3::Object"
+ },
+ {
+ "arn": "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md",
+ "type": "AWS::S3::Object"
+ },
+ {
+ "account_id": "000000000",
+ "arn": "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket",
+ "type": "AWS::S3::Bucket"
+ }
+ ],
+ "response_elements": "{x-amz-server-side-encryption=AES256}",
+ "user_identity": {
+ "access_key_id": "ACCESSKEYID",
+ "arn": "arn:aws:iam::000000000:user/test@elastic.co",
+ "type": "IAMUser"
+ }
+ }
+ },
+ "cloud": {
+ "account": {
+ "id": "000000000"
+ },
+ "region": "us-east-1"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "CopyObject",
+ "created": "2021-11-11T01:02:03.123456789Z",
+ "id": "0c06e2ff-5e88-44e6-a081-57871bbe770b",
+ "kind": "event",
+ "original": "{\"eventVersion\":\"1.09\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"ACCESSKEYID\",\"arn\":\"arn:aws:iam::000000000:user/test@elastic.co\",\"accountId\":\"000000000\",\"accessKeyId\":\"ACCESSKEYID\",\"userName\":\"test@elastic.co\"},\"eventTime\":\"2024-10-08T12:24:16Z\",\"eventSource\":\"s3.amazonaws.com\",\"eventName\":\"CopyObject\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"216.160.83.56\",\"userAgent\":\"[aws-cli/2.17.60 md/awscrt#0.21.2 ua/2.0 os/macos#23.6.0 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython cfg/retry-mode#standard md/installer#exe md/prompt#off md/command#s3api.copy-object]\",\"requestParameters\":{\"bucketName\":\"elastic-cspm-cloudtrail-test-bucket\",\"Host\":\"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com\",\"x-amz-copy-source\":\"elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md\",\"key\":\"test-copy-object/README-copy.md\"},\"responseElements\":{\"x-amz-server-side-encryption\":\"AES256\"},\"additionalEventData\":{\"SignatureVersion\":\"SigV4\",\"CipherSuite\":\"TLS_AES_128_GCM_SHA256\",\"bytesTransferredIn\":0,\"SSEApplied\":\"Default_SSE_S3\",\"AuthenticationMethod\":\"AuthHeader\",\"x-amz-id-2\":\"hFhfe14yINVvz+alr1rC5zPFufFU087OGEbVwf5HpD1BYs5D2llscEUSD7DUGjlSYkOEoay+oVk=\",\"bytesTransferredOut\":224},\"requestID\":\"62A9N2AH4P4YKG2B\",\"eventID\":\"0c06e2ff-5e88-44e6-a081-57871bbe770b\",\"readOnly\":false,\"resources\":[{\"type\":\"AWS::S3::Object\",\"ARN\":\"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md\"},{\"accountId\":\"000000000\",\"type\":\"AWS::S3::Bucket\",\"ARN\":\"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket\"},{\"type\":\"AWS::S3::Object\",\"ARN\":\"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"000000000\",\"eventCategory\":\"Data\",\"tlsDetails\":{\"tlsVersion\":\"TLSv1.3\",\"cipherSuite\":\"TLS_AES_128_GCM_SHA256\",\"clientProvidedHostHeader\":\"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com\"}}",
+ "outcome": "success",
+ "provider": "s3.amazonaws.com",
+ "type": [
+ "info"
+ ]
+ },
+ "related": {
+ "entity": [
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket",
+ "test@elastic.co",
+ "elastic-cspm-cloudtrail-test-bucket",
+ "ACCESSKEYID",
+ "arn:aws:iam::000000000:user/test@elastic.co",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md"
+ ],
+ "user": [
+ "ACCESSKEYID",
+ "test@elastic.co"
+ ]
+ },
+ "source": {
+ "address": "216.160.83.56",
+ "as": {
+ "number": 209
+ },
+ "geo": {
+ "city_name": "Milton",
+ "continent_name": "North America",
+ "country_iso_code": "US",
+ "country_name": "United States",
+ "location": {
+ "lat": 47.2513,
+ "lon": -122.3149
+ },
+ "region_iso_code": "US-WA",
+ "region_name": "Washington"
+ },
+ "ip": "216.160.83.56"
+ },
+ "tags": [
+ "preserve_original_event",
+ "actor_target_mapping"
+ ],
+ "target": {
+ "entity": {
+ "id": [
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md"
+ ]
+ }
+ },
+ "tls": {
+ "cipher": "TLS_AES_128_GCM_SHA256",
+ "client": {
+ "server_name": "elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com"
+ },
+ "version": "1.3",
+ "version_protocol": "tls"
+ },
+ "user": {
+ "email": "test@elastic.co",
+ "id": "ACCESSKEYID",
+ "name": "test@elastic.co"
+ },
+ "user_agent": {
+ "device": {
+ "name": "Other"
+ },
+ "name": "aws-cli",
+ "original": "[aws-cli/2.17.60 md/awscrt#0.21.2 ua/2.0 os/macos#23.6.0 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython cfg/retry-mode#standard md/installer#exe md/prompt#off md/command#s3api.copy-object]",
+ "version": "2.17.60"
+ }
+ },
+ {
+ "@timestamp": "2021-11-11T01:02:03.123456789Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "created": "2021-11-11T01:02:03.123456789Z",
+ "kind": "event",
+ "outcome": "success",
+ "type": [
+ "info"
+ ]
+ },
+ "tags": [
+ "preserve_original_event",
+ "actor_target_mapping"
+ ]
+ }
+ ]
+}

packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml

@@ -1719,18 +1719,55 @@ processors:
  # pipeline makes that non-trivial as both the flattened fields and the
  # non-flattened origin fields are referred to throughout the pipeline,
  # making it difficult to disentangle them. This the TODO to be done.
+ - script:
+ if: ctx._conf?.retain == 'minimal' && ctx.aws?.cloudtrail?.flattened != null
+ tag: script_retain_minimal_flattened
+ params:
+ # This list of fields is determined by enumerating all fields containing
+ # the string "cloudtrail.flattened" that are in .attributes.required_fields of the
+ # rules in security_detection_engine/kibana/security_rule.
+ # It must be kept up to date with the rules.
+ required_flattened_fields:
+ - aws.cloudtrail.flattened.additional_eventdata.SSEApplied
+ - aws.cloudtrail.flattened.request_parameters.cidrIp
+ - aws.cloudtrail.flattened.request_parameters.dryRun
+ - aws.cloudtrail.flattened.request_parameters.fromPort
+ - aws.cloudtrail.flattened.request_parameters.includeDeprecated
+ - aws.cloudtrail.flattened.request_parameters.policyArn
+ - aws.cloudtrail.flattened.request_parameters.serialNumber
+ - aws.cloudtrail.flattened.request_parameters.withDecryption
+ - aws.cloudtrail.flattened.request_parameters.x-amz-server-side-encryption-customer-algorithm
+ source: |-
+ def set(Map root, String path, def v) {
+ String[] elems = path.splitOnToken('.');
+ def dst = root;
+ for (int i = 0; i < elems.length-1; i++) {
+ dst = dst.computeIfAbsent(elems[i], _ -> new HashMap());
+ }
+ dst[elems[elems.length-1]] = v;
+ }
+ Map flattened = [:];
+ int prefix = "aws.cloudtrail.flattened.".length();
+ for (String f: params.required_flattened_fields) {
+ def v = field(f).get(null);
+ if (v == null) {
+ continue;
+ }
+ set(flattened, f.substring(prefix), v);
+ }
+ ctx.aws.cloudtrail.flattened = flattened;
  - remove:
  field:
  - aws.cloudtrail.flattened
  ignore_missing: true
- if: ctx._conf?.retain != null && ctx._conf.retain != '' && ctx._conf.retain != 'all' && ctx._conf.retain != 'flattened'
+ if: ctx._conf?.retain != null && ctx._conf.retain != '' && ctx._conf.retain != 'all' && ctx._conf.retain != 'flattened' && ctx._conf.retain != 'minimal'
  - remove:
  field:
  - aws.cloudtrail.response_elements
  - aws.cloudtrail.request_parameters
  - aws.cloudtrail.additional_eventdata
  ignore_missing: true
- if: ctx._conf?.retain != null && ctx._conf.retain != '' && ctx._conf.retain != 'all' && ctx._conf.retain != 'keyword'
+ if: ctx._conf?.retain != null && ctx._conf.retain != '' && ctx._conf.retain != 'all' && ctx._conf.retain != 'keyword' && ctx._conf.retain != 'minimal'
  - remove:
  field:
  - json

packages/aws/data_stream/cloudtrail/manifest.yml

@@ -198,14 +198,17 @@ streams:
  description: >-
  Cloudtrail `response_elements`, `request_parameters` and `additional_eventdata` data can
  be placed in keyword and text fields as JSON, and in flattened fields. Depending on requirements
- This configuration determines which fields will be retained in the final document.
+ This configuration determines which fields will be retained in the final document. The Minimal
+ option retains the minmal set of fields required for the Security Detection Engine rules.
  options:
  - text: Keyword and Flattened
  value: all
  - text: Keyword
  value: keyword
  - text: Flattened
  value: flattened
+ - text: Minimal
+ value: minimal
  - text: Neither
  value: none
  default: all