Resolve merge conflict

Author: sharadcrest

.buildkite/hooks/pre-command

@@ -31,9 +31,6 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 export REPO_BUILD_TAG
 
-AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
-PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
-
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
 
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
@@ -43,6 +40,8 @@ EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 export ENVIRONMENT="ci"
 export REPO="${REPO_NAME}"
 
+export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
+
 branch_name_label() {
  local branch="$1"
 
@@ -107,32 +106,13 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
  fi
 
  if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
- ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
- export ELASTIC_PACKAGE_AWS_SECRET_KEY
- ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
- export ELASTIC_PACKAGE_AWS_ACCESS_KEY
-
- PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
- export PRIVATE_CI_GCS_CREDENTIALS_SECRET
- export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
-
- # Environment variables required by the service deployer
- export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
- export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
-
  BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
  export BUILDKITE_API_TOKEN
  fi
 fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
  if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
- # Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
- # add the AWS credentials as in the integrations pipeline.
-
- PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
- export PRIVATE_CI_GCS_CREDENTIALS_SECRET
- export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 
  BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
  export BUILDKITE_API_TOKEN

.buildkite/hooks/pre-exit

@@ -7,10 +7,6 @@ set -euo pipefail
 if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
  # FIXME: update condition depending on the pipeline steps triggered
  if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
- unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
- unset ELASTIC_PACKAGE_AWS_SECRET_KEY
- unset AWS_ACCESS_KEY_ID
- unset AWS_SECRET_ACCESS_KEY
 
  # Ensure that kind cluster is deleted
  delete_kind_cluster
@@ -25,10 +21,6 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
  if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
- unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
- unset ELASTIC_PACKAGE_AWS_SECRET_KEY
- unset AWS_ACCESS_KEY_ID
- unset AWS_SECRET_ACCESS_KEY
 
  # Ensure that kind cluster is deleted
  delete_kind_cluster
@@ -44,8 +36,6 @@ fi
 unset_secrets
 cleanup
 
-google_cloud_logout_active_account
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
  cd "${WORKSPACE}"
  git config remote.origin.url "https://github.com/elastic/integrations.git"

.buildkite/pipeline.serverless.yml

@@ -67,6 +67,17 @@ steps:
  agents:
  provider: "gcp"
  image: "${IMAGE_UBUNTU_X86_64}"
+ plugins:
+ # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf
+ # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
+ - elastic/oblt-aws-auth#v0.1.0:
+ duration: 10800 # seconds
+ # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
+ # This plugin authenticates to Google Cloud using the OIDC token.
+ - elastic/oblt-google-auth#v1.3.0:
+ lifetime: 10800 # seconds
+ project-id: "elastic-observability-ci"
+ project-number: "911195782929"
  artifact_paths:
  - "build/test-results/*.xml"
  - "build/elastic-stack-dump/*/logs/*.log"

.buildkite/scripts/common.sh

@@ -10,7 +10,6 @@ platform_type_lowercase="${platform_type,,}"
 
 SCRIPTS_BUILDKITE_PATH="${WORKSPACE}/.buildkite/scripts"
 
-GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/pipelines/"
@@ -255,34 +254,6 @@ with_github_cli() {
  gh version
 }
 
-## Logging and logout from Google Cloud
-google_cloud_auth_safe_logs() {
- local gsUtilLocation
- gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
- local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-
- echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
-
- gcloud auth activate-service-account --key-file "${secretFileLocation}" 2> /dev/null
- export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
-}
-
-google_cloud_logout_active_account() {
- local active_account
- active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null || true)
- if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
- echo "Logging out from GCP for active account"
- gcloud auth revoke "$active_account" > /dev/null 2>&1
- else
- echo "No active GCP accounts found."
- fi
-
- if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
- rm -rf "${GOOGLE_APPLICATION_CREDENTIALS}"
- unset GOOGLE_APPLICATION_CREDENTIALS
- fi
-}
-
 ## Helpers for integrations pipelines
 check_git_diff() {
  cd "${WORKSPACE}"
@@ -983,16 +954,16 @@ upload_safe_logs() {
  local source="$2"
  local target="$3"
 
+ echo "--- Uploading safe logs to GCP bucket ${bucket}"
+
  if ! ls ${source} 2>&1 > /dev/null ; then
  echo "upload_safe_logs: artifacts files not found, nothing will be archived"
  return
  fi
 
- google_cloud_auth_safe_logs
-
- gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+ gcloud storage cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
 
- google_cloud_logout_active_account
+ echo "GCP logout is not required, the BK plugin will do it for us"
 }
 
 clean_safe_logs() {

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -88,6 +88,17 @@ for package in ${PACKAGE_LIST}; do
  FORCE_CHECK_ALL: "${FORCE_CHECK_ALL}"
  SERVERLESS: "false"
  UPLOAD_SAFE_LOGS: ${UPLOAD_SAFE_LOGS}
+ plugins:
+ # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf
+ # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
+ - elastic/oblt-aws-auth#v0.1.0:
+ duration: 10800 # seconds
+ # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
+ # This plugin authenticates to Google Cloud using the OIDC token.
+ - elastic/oblt-google-auth#v1.3.0:
+ lifetime: 10800 # seconds
+ project-id: "elastic-observability-ci"
+ project-number: "911195782929"
  artifact_paths:
  - build/test-results/*.xml
  - build/test-coverage/*.xml

.github/CODEOWNERS

@@ -164,6 +164,7 @@
 /packages/citrix_adc/data_stream/vpn @elastic/obs-infraobs-integrations
 /packages/citrix_waf @elastic/sec-deployment-and-devices
 /packages/claroty_ctd @elastic/security-service-integrations
+/packages/claroty_xdome @elastic/security-service-integrations
 /packages/cloud_defend @elastic/sec-linux-platform
 /packages/cloud_security_posture @elastic/cloud-security-posture
 /packages/cloud_asset_inventory @elastic/cloud-security-posture

packages/awsfirehose/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.0"
+ changes:
+ - description: set event.dataset matching the routed target dataset for ingesting records
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/13910
 - version: "1.6.0"
  changes:
  - description: Add support for AmazonMQ metrics for both managed RabbitMQ and ActiveMQ.

packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-apigateway-log.json-expected.json

@@ -18,6 +18,9 @@
  "ecs": {
  "version": "8.11.0"
  },
+ "event": {
+ "dataset": "aws.apigateway_logs"
+ },
  "event.id": "37670326805251200781477669690942747782212394134076063744",
  "message": "{\"requestId\":\"GQIVriFLIAMEMsA=\",\"ip\":\"1.128.0.0\",\"requestTime\":\"09/Jun/2023:12:54:08 +0000\",\"httpMethod\":\"GET\",\"routeKey\":\"GET /\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"47140\"}"
  },
@@ -39,6 +42,9 @@
  "ecs": {
  "version": "8.11.0"
  },
+ "event": {
+ "dataset": "aws.apigateway_logs"
+ },
  "event.id": "37670326805251200781477669690942747782212394134076063744",
  "message": "{\"requestId\":\"Iq9gjE_aIAMFZTg=\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:20:44 +0000\",\"eventType\":\"CONNECT\",\"routeKey\":\"$connect\",\"status\":\"500\",\"connectionId\":\"Iq8gj1UmIAMCKpA=\",\"apiId\":\"z1ctxygne5\",\"stage\":\"production\",\"domainName\":\"z1ctxygne5.execute-api.us-east-1.amazonaws.com\"}"
  },
@@ -57,6 +63,9 @@
  "data_stream.dataset": "aws.apigateway_logs",
  "data_stream.namespace": "default",
  "data_stream.type": "logs",
+ "event": {
+ "dataset": "aws.apigateway_logs"
+ },
  "ecs": {
  "version": "8.11.0"
  },

packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json-expected.json

@@ -15,6 +15,9 @@
  "ecs": {
  "version": "8.11.0"
  },
+ "event": {
+ "dataset": "aws.cloudfront_logs"
+ },
  "event.id": "37670326805251200781477669690942747782212394134076063744",
  "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -"
  }

packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudtrail-log.json-expected.json

@@ -18,8 +18,11 @@
  "ecs": {
  "version": "8.11.0"
  },
+ "event": {
+ "dataset": "aws.cloudtrail"
+ },
  "event.id": "37670326805251200781477669690942747782212394134076063744",
  "message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AWSService\",\"invokedBy\":\"cloudtrail.amazonaws.com\"},\"eventTime\":\"2023-07-17T21:02:26Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"sa-east-1\",\"sourceIPAddress\":\"cloudtrail.amazonaws.com\",\"userAgent\":\"cloudtrail.amazonaws.com\",\"requestParameters\":{\"roleArn\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\",\"roleSessionName\":\"CLOUDWATCH_LOGS_DELIVERY_SESSION\"},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAZEDJODE3A5LVGLFB\",\"sessionToken\":\"IQoJb3JpZ2luX2VjEGUaCXNhLWVhc3QtMSJHMEUCIHgHmtcrhwDhosJlQVky+C2zsYDKuR99qVlNjGIp8FLWAiEAsJtTDQ3Arq8iXEOHwv0ImEQdGb5tbgc+fLpoK58Enb4q9AII3v//////////ARAEGgw2MjcyODYzNTAxMzQiDN5gNdfO4ZdSqDmmwSrIAicTBYZg+ZXjwiJTN/Bz2YsMWYU6psw5znG3/Gh3EJ1P3RCmB7d79X6XZzFVi2u2xdrnaY/sTKDfp1jdl8OoAsSKYwJiGbzjoQlv59bB6JqPbKfAKUPAmz6JEMWNFgWTtaQL9rNkdPz23u/1msoUSzxCcxR9f3A2dD4yqnVpNJe8ipuhxpBMzQ61vcGL4G5hQEDM/o8sORP2PXbK4O7QAuWOyuryYkHAPwY9RrL0WHfflGBEBQV6XlidGpsRCtIppZVn025n3DQOypDEaL3fKp0gUsMkDH+frFjxop4o4wRYC3CxXe3XRJ5/Te886rQry7RUfXlQtiCfojZO5ohcLB+z6Y/uCK0IHp3zrfl5shKsQIAFt7p0B8W7PK5yHE4W9HHRiktJ9wTtq1YCTaWECpnjW0bISNgumRmDOAJvVHAjSjfkr4yAlJkw4qm8pQY6vwGbBiuf98AfRFrXMy01hVdE3GNTBrIS68zxUJaOjBLgw8l0nEC00L+LPuqaASFWz65Dnq5JAjXaDD9E3iCi4klp4gZFAcj7uGgeBIPkP7Bpr4SvBfnnqCgE2oyFrWke3NnYtqkL5iHLJeGlOTrvI5ND2H4jurQv0KbiqwHt6DmGF3poZOrtf8R3piNcuCCDLU8RvhRVLHy5rKPzsWgNokBc9XXmgltwvB6rIgdZhBJzupzmy/NSoWZcOeH2ooEELw==\",\"expiration\":\"Jul 12, 2023, 10:02:26 PM\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAZEDJODE3NLJAH2FZC:CLOUDWATCH_LOGS_DELIVERY_SESSION\",\"arn\":\"arn:aws:sts::123456:assumed-role/aws-cloudtrail-logs-123456-b888baff_Role/CLOUDWATCH_LOGS_DELIVERY_SESSION\"}},\"requestID\":\"041c9e5f-a031-47d2-a4a0-011bc8d5352c\",\"eventID\":\"3096b662-7aa9-43e6-8bee-541a45686745\",\"readOnly\":true,\"resources\":[{\"accountId\":\"123456\",\"type\":\"AWS::IAM::Role\",\"ARN\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\"}],\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"123456\",\"sharedEventID\":\"a1c94275-884f-4c1f-b8dc-2e1bf4c94d29\",\"eventCategory\":\"Management\"}"
  }
  ]
-}
+}