aws: allow user specification of fields to retain in the cloudtrail data stream

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.10.0"
+ changes:
+ - description: Allow user-specification of fields to retain in the cloudtrail data stream.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/14236
 - version: "3.9.0"
  changes:
  - description: Ingest managed insights from Security Hub.

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-common-config.yml

@@ -9,3 +9,6 @@ fields:
  tags:
  - preserve_original_event
  - actor_target_mapping
+ # _conf.retain may be absent, null or '' with the same effect as 'all'.
+ _conf:
+ retain: all

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-copy-object-json-minimal.log

@@ -0,0 +1,2 @@
+{"eventVersion":"1.09","userIdentity":{"type":"IAMUser","principalId":"ACCESSKEYID","arn":"arn:aws:iam::000000000:user/test@elastic.co","accountId":"000000000","accessKeyId":"ACCESSKEYID","userName":"test@elastic.co"},"eventTime":"2024-10-08T12:24:16Z","eventSource":"s3.amazonaws.com","eventName":"CopyObject","awsRegion":"us-east-1","sourceIPAddress":"216.160.83.56","userAgent":"[aws-cli/2.17.60 md/awscrt#0.21.2 ua/2.0 os/macos#23.6.0 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython cfg/retry-mode#standard md/installer#exe md/prompt#off md/command#s3api.copy-object]","requestParameters":{"bucketName":"elastic-cspm-cloudtrail-test-bucket","Host":"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com","x-amz-copy-source":"elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md","key":"test-copy-object/README-copy.md"},"responseElements":{"x-amz-server-side-encryption":"AES256"},"additionalEventData":{"SignatureVersion":"SigV4","CipherSuite":"TLS_AES_128_GCM_SHA256","bytesTransferredIn":0,"SSEApplied":"Default_SSE_S3","AuthenticationMethod":"AuthHeader","x-amz-id-2":"hFhfe14yINVvz+alr1rC5zPFufFU087OGEbVwf5HpD1BYs5D2llscEUSD7DUGjlSYkOEoay+oVk=","bytesTransferredOut":224},"requestID":"62A9N2AH4P4YKG2B","eventID":"0c06e2ff-5e88-44e6-a081-57871bbe770b","readOnly":false,"resources":[{"type":"AWS::S3::Object","ARN":"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md"},{"accountId":"000000000","type":"AWS::S3::Bucket","ARN":"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket"},{"type":"AWS::S3::Object","ARN":"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md"}],"eventType":"AwsApiCall","recipientAccountId":"000000000","eventCategory":"Data","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com"}}
+

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-copy-object-json-minimal.log-config.yml

@@ -0,0 +1,13 @@
+dynamic_fields:
+ # This can be removed after ES 8.16.2 is set as the minimum version supported in the manifest.
+ # Once removed, it requires to update pipeline tests to remove the trailing dot where required.
+ # Relates: https://github.com/elastic/elasticsearch/pull/117213
+ "user_agent.version": '^\d+\.\d+(\.|\..*)?$'
+fields:
+ # Simulate @timestamp value from Filebeat.
+ '@timestamp': '2021-11-11T01:02:03.123456789Z'
+ tags:
+ - preserve_original_event
+ - actor_target_mapping
+ _conf:
+ retain: minimal

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-copy-object-json-minimal.log-expected.json

@@ -0,0 +1,159 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2024-10-08T12:24:16.000Z",
+ "actor": {
+ "entity": {
+ "id": [
+ "arn:aws:iam::000000000:user/test@elastic.co"
+ ]
+ }
+ },
+ "aws": {
+ "cloudtrail": {
+ "additional_eventdata": "{SignatureVersion=SigV4, CipherSuite=TLS_AES_128_GCM_SHA256, bytesTransferredIn=0, SSEApplied=Default_SSE_S3, AuthenticationMethod=AuthHeader, x-amz-id-2=hFhfe14yINVvz+alr1rC5zPFufFU087OGEbVwf5HpD1BYs5D2llscEUSD7DUGjlSYkOEoay+oVk=, bytesTransferredOut=224}",
+ "event_category": "Data",
+ "event_type": "AwsApiCall",
+ "event_version": "1.09",
+ "flattened": {
+ "additional_eventdata": {
+ "SSEApplied": "Default_SSE_S3"
+ }
+ },
+ "read_only": false,
+ "recipient_account_id": "000000000",
+ "request_id": "62A9N2AH4P4YKG2B",
+ "request_parameters": "{bucketName=elastic-cspm-cloudtrail-test-bucket, Host=elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com, x-amz-copy-source=elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md, key=test-copy-object/README-copy.md}",
+ "resources": [
+ {
+ "arn": "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md",
+ "type": "AWS::S3::Object"
+ },
+ {
+ "arn": "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md",
+ "type": "AWS::S3::Object"
+ },
+ {
+ "account_id": "000000000",
+ "arn": "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket",
+ "type": "AWS::S3::Bucket"
+ }
+ ],
+ "response_elements": "{x-amz-server-side-encryption=AES256}",
+ "user_identity": {
+ "access_key_id": "ACCESSKEYID",
+ "arn": "arn:aws:iam::000000000:user/test@elastic.co",
+ "type": "IAMUser"
+ }
+ }
+ },
+ "cloud": {
+ "account": {
+ "id": "000000000"
+ },
+ "region": "us-east-1"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "CopyObject",
+ "created": "2021-11-11T01:02:03.123456789Z",
+ "id": "0c06e2ff-5e88-44e6-a081-57871bbe770b",
+ "kind": "event",
+ "original": "{\"eventVersion\":\"1.09\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"ACCESSKEYID\",\"arn\":\"arn:aws:iam::000000000:user/test@elastic.co\",\"accountId\":\"000000000\",\"accessKeyId\":\"ACCESSKEYID\",\"userName\":\"test@elastic.co\"},\"eventTime\":\"2024-10-08T12:24:16Z\",\"eventSource\":\"s3.amazonaws.com\",\"eventName\":\"CopyObject\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"216.160.83.56\",\"userAgent\":\"[aws-cli/2.17.60 md/awscrt#0.21.2 ua/2.0 os/macos#23.6.0 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython cfg/retry-mode#standard md/installer#exe md/prompt#off md/command#s3api.copy-object]\",\"requestParameters\":{\"bucketName\":\"elastic-cspm-cloudtrail-test-bucket\",\"Host\":\"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com\",\"x-amz-copy-source\":\"elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md\",\"key\":\"test-copy-object/README-copy.md\"},\"responseElements\":{\"x-amz-server-side-encryption\":\"AES256\"},\"additionalEventData\":{\"SignatureVersion\":\"SigV4\",\"CipherSuite\":\"TLS_AES_128_GCM_SHA256\",\"bytesTransferredIn\":0,\"SSEApplied\":\"Default_SSE_S3\",\"AuthenticationMethod\":\"AuthHeader\",\"x-amz-id-2\":\"hFhfe14yINVvz+alr1rC5zPFufFU087OGEbVwf5HpD1BYs5D2llscEUSD7DUGjlSYkOEoay+oVk=\",\"bytesTransferredOut\":224},\"requestID\":\"62A9N2AH4P4YKG2B\",\"eventID\":\"0c06e2ff-5e88-44e6-a081-57871bbe770b\",\"readOnly\":false,\"resources\":[{\"type\":\"AWS::S3::Object\",\"ARN\":\"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md\"},{\"accountId\":\"000000000\",\"type\":\"AWS::S3::Bucket\",\"ARN\":\"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket\"},{\"type\":\"AWS::S3::Object\",\"ARN\":\"arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"000000000\",\"eventCategory\":\"Data\",\"tlsDetails\":{\"tlsVersion\":\"TLSv1.3\",\"cipherSuite\":\"TLS_AES_128_GCM_SHA256\",\"clientProvidedHostHeader\":\"elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com\"}}",
+ "outcome": "success",
+ "provider": "s3.amazonaws.com",
+ "type": [
+ "info"
+ ]
+ },
+ "related": {
+ "entity": [
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket",
+ "test@elastic.co",
+ "elastic-cspm-cloudtrail-test-bucket",
+ "ACCESSKEYID",
+ "arn:aws:iam::000000000:user/test@elastic.co",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md"
+ ],
+ "user": [
+ "ACCESSKEYID",
+ "test@elastic.co"
+ ]
+ },
+ "source": {
+ "address": "216.160.83.56",
+ "as": {
+ "number": 209
+ },
+ "geo": {
+ "city_name": "Milton",
+ "continent_name": "North America",
+ "country_iso_code": "US",
+ "country_name": "United States",
+ "location": {
+ "lat": 47.2513,
+ "lon": -122.3149
+ },
+ "region_iso_code": "US-WA",
+ "region_name": "Washington"
+ },
+ "ip": "216.160.83.56"
+ },
+ "tags": [
+ "preserve_original_event",
+ "actor_target_mapping"
+ ],
+ "target": {
+ "entity": {
+ "id": [
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README-copy.md",
+ "arn:aws:s3:::elastic-cspm-cloudtrail-test-bucket/test-copy-object/README.md"
+ ]
+ }
+ },
+ "tls": {
+ "cipher": "TLS_AES_128_GCM_SHA256",
+ "client": {
+ "server_name": "elastic-cspm-cloudtrail-test-bucket.s3.us-east-1.amazonaws.com"
+ },
+ "version": "1.3",
+ "version_protocol": "tls"
+ },
+ "user": {
+ "email": "test@elastic.co",
+ "id": "ACCESSKEYID",
+ "name": "test@elastic.co"
+ },
+ "user_agent": {
+ "device": {
+ "name": "Other"
+ },
+ "name": "aws-cli",
+ "original": "[aws-cli/2.17.60 md/awscrt#0.21.2 ua/2.0 os/macos#23.6.0 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython cfg/retry-mode#standard md/installer#exe md/prompt#off md/command#s3api.copy-object]",
+ "version": "2.17.60"
+ }
+ },
+ {
+ "@timestamp": "2021-11-11T01:02:03.123456789Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "created": "2021-11-11T01:02:03.123456789Z",
+ "kind": "event",
+ "outcome": "success",
+ "type": [
+ "info"
+ ]
+ },
+ "tags": [
+ "preserve_original_event",
+ "actor_target_mapping"
+ ]
+ }
+ ]
+}

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json-flattened.log

@@ -0,0 +1 @@
+{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:06:40Z","eventSource":"iam.amazonaws.com","eventName":"UploadSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain","userName":"Alice"},"responseElements":{"sSHPublicKey":{"fingerprint":"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de","status":"Active","uploadDate":"Jan 10, 2020 4:06:40 PM","userName":"Alice","sSHPublicKeyId":"EXAMPLE_KEY_ID","sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain"}},"requestID":"EXAMPLE-44b9-41cd-90f2-EXAMPLE","eventID":"EXAMPLE-9a9d-4da4-9998-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"}

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json-flattened.log-config.yml

@@ -0,0 +1,13 @@
+dynamic_fields:
+ # This can be removed after ES 8.16.2 is set as the minimum version supported in the manifest.
+ # Once removed, it requires to update pipeline tests to remove the trailing dot where required.
+ # Relates: https://github.com/elastic/elasticsearch/pull/117213
+ "user_agent.version": '^\d+\.\d+(\.|\..*)?$'
+fields:
+ # Simulate @timestamp value from Filebeat.
+ '@timestamp': '2021-11-11T01:02:03.123456789Z'
+ tags:
+ - preserve_original_event
+ - actor_target_mapping
+ _conf:
+ retain: flattened

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json-flattened.log-expected.json

@@ -0,0 +1,102 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2020-01-10T16:06:40.000Z",
+ "actor": {
+ "entity": {
+ "id": [
+ "arn:aws:iam::0123456789012:user/Alice"
+ ]
+ }
+ },
+ "aws": {
+ "cloudtrail": {
+ "event_type": "AwsApiCall",
+ "event_version": "1.05",
+ "flattened": {
+ "request_parameters": {
+ "sSHPublicKeyBody": "ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain",
+ "userName": "Alice"
+ },
+ "response_elements": {
+ "sSHPublicKey": {
+ "fingerprint": "de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de",
+ "sSHPublicKeyBody": "ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain",
+ "sSHPublicKeyId": "EXAMPLE_KEY_ID",
+ "status": "Active",
+ "uploadDate": "Jan 10, 2020 4:06:40 PM",
+ "userName": "Alice"
+ }
+ }
+ },
+ "recipient_account_id": "0123456789012",
+ "request_id": "EXAMPLE-44b9-41cd-90f2-EXAMPLE",
+ "user_identity": {
+ "access_key_id": "EXAMPLE_KEY",
+ "arn": "arn:aws:iam::0123456789012:user/Alice",
+ "invoked_by": "signin.amazonaws.com",
+ "session_context": {
+ "creation_date": "2020-01-10T14:38:30.000Z",
+ "mfa_authenticated": "true"
+ },
+ "type": "IAMUser"
+ }
+ }
+ },
+ "cloud": {
+ "account": {
+ "id": "0123456789012"
+ },
+ "region": "us-east-1"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "UploadSSHPublicKey",
+ "created": "2021-11-11T01:02:03.123456789Z",
+ "id": "EXAMPLE-9a9d-4da4-9998-EXAMPLE",
+ "kind": "event",
+ "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:40Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UploadSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\",\"userName\":\"Alice\"},\"responseElements\":{\"sSHPublicKey\":{\"fingerprint\":\"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\",\"status\":\"Active\",\"uploadDate\":\"Jan 10, 2020 4:06:40 PM\",\"userName\":\"Alice\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\"}},\"requestID\":\"EXAMPLE-44b9-41cd-90f2-EXAMPLE\",\"eventID\":\"EXAMPLE-9a9d-4da4-9998-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
+ "outcome": "success",
+ "provider": "iam.amazonaws.com",
+ "type": [
+ "info"
+ ]
+ },
+ "related": {
+ "entity": [
+ "EXAMPLE_KEY",
+ "Alice",
+ "arn:aws:iam::0123456789012:user/Alice"
+ ],
+ "user": [
+ "Alice",
+ "EXAMPLE_ID"
+ ]
+ },
+ "source": {
+ "address": "127.0.0.1",
+ "ip": "127.0.0.1"
+ },
+ "tags": [
+ "preserve_original_event",
+ "actor_target_mapping"
+ ],
+ "user": {
+ "id": "EXAMPLE_ID",
+ "name": "Alice",
+ "target": {
+ "name": "Alice"
+ }
+ },
+ "user_agent": {
+ "device": {
+ "name": "Other"
+ },
+ "name": "Other",
+ "original": "signin.amazonaws.com"
+ }
+ }
+ ]
+}

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json-keyword.log

@@ -0,0 +1 @@
+{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:06:40Z","eventSource":"iam.amazonaws.com","eventName":"UploadSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain","userName":"Alice"},"responseElements":{"sSHPublicKey":{"fingerprint":"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de","status":"Active","uploadDate":"Jan 10, 2020 4:06:40 PM","userName":"Alice","sSHPublicKeyId":"EXAMPLE_KEY_ID","sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain"}},"requestID":"EXAMPLE-44b9-41cd-90f2-EXAMPLE","eventID":"EXAMPLE-9a9d-4da4-9998-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"}

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json-keyword.log-config.yml

@@ -0,0 +1,13 @@
+dynamic_fields:
+ # This can be removed after ES 8.16.2 is set as the minimum version supported in the manifest.
+ # Once removed, it requires to update pipeline tests to remove the trailing dot where required.
+ # Relates: https://github.com/elastic/elasticsearch/pull/117213
+ "user_agent.version": '^\d+\.\d+(\.|\..*)?$'
+fields:
+ # Simulate @timestamp value from Filebeat.
+ '@timestamp': '2021-11-11T01:02:03.123456789Z'
+ tags:
+ - preserve_original_event
+ - actor_target_mapping
+ _conf:
+ retain: keyword