Merge branch 'main' into aws-4.0.0

Author: brijesh-elastic

.buildkite/scripts/common.sh

@@ -746,7 +746,7 @@ is_pr_affected() {
  # Example:
  # https://buildkite.com/elastic/integrations/builds/25606
  # https://github.com/elastic/integrations/pull/13810
- if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml))' > /dev/null; then
+ if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml))' > /dev/null; then
  echo "[${package}] PR is affected: found non-package files"
  return 0
  fi

.github/CODEOWNERS

@@ -197,6 +197,7 @@
 /packages/elastic_agent @elastic/elastic-agent
 /packages/elastic_connectors @elastic/search-extract-and-transform
 /packages/elastic_package_registry @elastic/ecosystem
+/packages/elastic_security @elastic/security-service-integrations
 /packages/elasticsearch @elastic/stack-monitoring
 /packages/endace @elastic/sec-deployment-and-devices @elastic/sec-linux-platform
 /packages/endace/data_stream/flow @elastic/sec-linux-platform

packages/auth0/_dev/build/docs/README.md

@@ -17,22 +17,24 @@ The package collects log events either sent via log stream webhooks, or by API r
 
 The agent running this integration must be able to accept requests from the Internet in order for Auth0 to be able connect. Auth0 requires that the webhook accept requests over HTTPS. So you must either configure the integration with a valid TLS certificate or use a reverse proxy in front of the integration.
 
-For more information, see Auth0's webpage on [integration to Elastic Security](https://marketplace.auth0.com/integrations/elastic-security).
+This integration method is also documented on the Auth0 webpage for the [Elastic Security integration](https://marketplace.auth0.com/integrations/elastic-security).
 
 ### Configure the Auth0 integration
 
 1. Click on **Collect Auth0 log streams events via Webhooks** to enable it.
-2. Enter values for "Listen Address", "Listen Port" and "Webhook path" to form the endpoint URL. Make note of the **Endpoint URL** `https://{AGENT_ADDRESS}:8383/auth0/logs`.
-3. Enter value for "Secret value". This must match the "Authorization Token" value entered when configuring the "Custom Webhook" from Auth0 cloud.
-4. Enter values for "TLS". Auth0 requires that the webhook accept requests over HTTPS. So you must either configure the integration with a valid TLS certificate or use a reverse proxy in front of the integration.
+2. Enter values for **Listen Address**, **Listen Port** and **Webhook Path**.
+3. Enter value for **Authorization Token**. This must match the value entered when configuring the "Custom Webhook" in Auth0 cloud.
+4. In the "Advanced options" section, enter settings for **SSL Configuration**. Auth0 requires that webhook requests use HTTPS. So you must either configure the integration with a valid TLS certificate here, or use a separarately configured reverse proxy in front of the agent.
+
+Using the external address of the agent to which the integration is added, and the configured "Listen Port" and "Webhook Path", make a note of the full endpoint URL. It will have the form `https://{AGENT_ADDRESS}:{LISTEN_PORT}{WEBHOOK_PATH}` (for example, `https://agent01.external.example.com:8383/auth0/logs`).
 
 ### Creating the stream in Auth0
 
 1. From the Auth0 management console, navigate to **Logs > Streams** and click **+ Create Stream**.
 2. Choose **Custom Webhook**.
 3. Name the new **Event Stream** appropriately (e.g. Elastic) and click **Create**.
-4. In **Payload URL**, paste the **Endpoint URL** collected during Step 1 of **Configure the Auth0 integration** section.
-5. In **Authorization Token**, paste the **Authorization Token**. This must match the value entered in Step 2 of **Configure the Auth0 integration** section.
+4. In **Payload URL**, enter the endpoint URL set up in the **Configure the Auth0 integration** section.
+5. In **Authorization Token**, enter the **Authorization Token**. This must match the value entered in Step 3 of the **Configure the Auth0 integration** section.
 6. In **Content Type**, choose **application/json**.
 7. In **Content Format**, choose **JSON Lines**.
 8. Click **Save**.

packages/auth0/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.23.0"
+ changes:
+ - description: Improve webhook setup instructions.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/14434
 - version: "1.22.0"
  changes:
  - description: Standardize user fields processing across integrations.

packages/auth0/data_stream/logs/manifest.yml

@@ -24,15 +24,16 @@ streams:
  default: 8383
  - name: url
  type: text
- title: Webhook path
- description: URL path where the webhook will accept requests.
+ title: Webhook Path
+ description: URL path where the webhook will accept requests. It must include a leading slash.
  multi: false
  required: true
- show_user: false
+ show_user: true
  default: /auth0/logs
  - name: secret_value
+ title: Authorization Token
  type: password
- description: Authorization token
+ description: A token that incoming requests must include in an Authorization header.
  multi: false
  required: false
  show_user: true

packages/auth0/docs/README.md

@@ -17,22 +17,24 @@ The package collects log events either sent via log stream webhooks, or by API r
 
 The agent running this integration must be able to accept requests from the Internet in order for Auth0 to be able connect. Auth0 requires that the webhook accept requests over HTTPS. So you must either configure the integration with a valid TLS certificate or use a reverse proxy in front of the integration.
 
-For more information, see Auth0's webpage on [integration to Elastic Security](https://marketplace.auth0.com/integrations/elastic-security).
+This integration method is also documented on the Auth0 webpage for the [Elastic Security integration](https://marketplace.auth0.com/integrations/elastic-security).
 
 ### Configure the Auth0 integration
 
 1. Click on **Collect Auth0 log streams events via Webhooks** to enable it.
-2. Enter values for "Listen Address", "Listen Port" and "Webhook path" to form the endpoint URL. Make note of the **Endpoint URL** `https://{AGENT_ADDRESS}:8383/auth0/logs`.
-3. Enter value for "Secret value". This must match the "Authorization Token" value entered when configuring the "Custom Webhook" from Auth0 cloud.
-4. Enter values for "TLS". Auth0 requires that the webhook accept requests over HTTPS. So you must either configure the integration with a valid TLS certificate or use a reverse proxy in front of the integration.
+2. Enter values for **Listen Address**, **Listen Port** and **Webhook Path**.
+3. Enter value for **Authorization Token**. This must match the value entered when configuring the "Custom Webhook" in Auth0 cloud.
+4. In the "Advanced options" section, enter settings for **SSL Configuration**. Auth0 requires that webhook requests use HTTPS. So you must either configure the integration with a valid TLS certificate here, or use a separarately configured reverse proxy in front of the agent.
+
+Using the external address of the agent to which the integration is added, and the configured "Listen Port" and "Webhook Path", make a note of the full endpoint URL. It will have the form `https://{AGENT_ADDRESS}:{LISTEN_PORT}{WEBHOOK_PATH}` (for example, `https://agent01.external.example.com:8383/auth0/logs`).
 
 ### Creating the stream in Auth0
 
 1. From the Auth0 management console, navigate to **Logs > Streams** and click **+ Create Stream**.
 2. Choose **Custom Webhook**.
 3. Name the new **Event Stream** appropriately (e.g. Elastic) and click **Create**.
-4. In **Payload URL**, paste the **Endpoint URL** collected during Step 1 of **Configure the Auth0 integration** section.
-5. In **Authorization Token**, paste the **Authorization Token**. This must match the value entered in Step 2 of **Configure the Auth0 integration** section.
+4. In **Payload URL**, enter the endpoint URL set up in the **Configure the Auth0 integration** section.
+5. In **Authorization Token**, enter the **Authorization Token**. This must match the value entered in Step 3 of the **Configure the Auth0 integration** section.
 6. In **Content Type**, choose **application/json**.
 7. In **Content Format**, choose **JSON Lines**.
 8. Click **Save**.

packages/auth0/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: auth0
 title: "Auth0"
-version: "1.22.0"
+version: "1.23.0"
 description: Collect logs from Auth0 with Elastic Agent.
 type: integration
 categories:

packages/aws/changelog.yml

@@ -19,6 +19,14 @@
  - description: Remove metadata fields added by the Agentless policy.
  type: bugfix
  link: https://github.com/elastic/integrations/pull/14306
+- version: "3.11.0"
+ changes:
+ - description: Fix `tlsVersion` parsing when not properly defined in cloudtrail event.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/13345
+ - description: Add empty value removal script to cloudtrail data stream ingest pipeline.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/13345
 - version: "3.10.1"
  changes:
  - description: Fix configuration template typo.
@@ -161,7 +169,7 @@
  link: https://github.com/elastic/integrations/pull/12755
 - version: "2.40.0"
  changes:
- - description: Add support for Kibana `9.0.0`
+ - description: Add support for Kibana `9.0.0`.
  type: enhancement
  link: https://github.com/elastic/integrations/pull/12637
 - version: "2.39.0"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json

@@ -15,7 +15,6 @@
  "error_message": "An unknown error occurred",
  "event_type": "AwsApiCall",
  "event_version": "1.05",
- "flattened": {},
  "recipient_account_id": "0123456789012",
  "request_id": "EXAMPLE-5204-4fed-9c60-9c6EXAMPLE",
  "user_identity": {
@@ -95,7 +94,6 @@
  "cloudtrail": {
  "event_type": "AwsApiCall",
  "event_version": "1.05",
- "flattened": {},
  "recipient_account_id": "0123456789012",
  "request_id": "EXAMPLE-5c16-4eda-9724-EXAMPLE",
  "user_identity": {

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-converse-json.log-expected.json

@@ -59,8 +59,7 @@
  "kind": "event",
  "original": "{\"eventVersion\":\"1.10\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"PRINCIPALID:i-03cd6b2a7eb4bf3ae\",\"arn\":\"arn:aws:sts::00000000000:assumed-role/private-ec2-instance-role/i-03cd6b2a7eb4bf3ae\",\"accountId\":\"00000000000\",\"accessKeyId\":\"ACCESSKEY\",\"sessionContext\":{\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"PRINCIPALID\",\"arn\":\"arn:aws:iam::00000000000:role/private-ec2-instance-role\",\"accountId\":\"00000000000\",\"userName\":\"private-ec2-instance-role\"},\"attributes\":{\"creationDate\":\"2024-10-29T14:29:03Z\",\"mfaAuthenticated\":\"false\"},\"ec2RoleDelivery\":\"2.0\"}},\"eventTime\":\"2024-10-29T15:51:44Z\",\"eventSource\":\"bedrock.amazonaws.com\",\"eventName\":\"Converse\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"216.160.83.56\",\"userAgent\":\"Boto3/1.35.50 md/Botocore#1.35.50 ua/2.0 os/linux#6.8.0-1016-aws md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.50\",\"requestParameters\":{\"modelId\":\"anthropic.claude-3-5-sonnet-20240620-v1:0\"},\"responseElements\":null,\"requestID\":\"aff6f361-1ef0-4460-97af-26528a80b511\",\"eventID\":\"3d67c35a-eef1-4d64-9620-77af8f372ae7\",\"readOnly\":true,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"00000000000\",\"eventCategory\":\"Management\",\"tlsDetails\":{\"tlsVersion\":\"TLSv1.3\",\"cipherSuite\":\"TLS_AES_128_GCM_SHA256\",\"clientProvidedHostHeader\":\"bedrock-runtime.us-east-1.amazonaws.com\"}}",
  "outcome": "success",
- "provider": "bedrock.amazonaws.com",
- "type": []
+ "provider": "bedrock.amazonaws.com"
  },
  "related": {
  "entity": [