Merge branch 'main' into nvidia-system-tests

Author: Linu-Elias

.buildkite/hooks/pre-command

@@ -33,7 +33,6 @@ export REPO_BUILD_TAG
 
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
 
-EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 
 # variables required for terraform
@@ -117,10 +116,6 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
  BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
  export BUILDKITE_API_TOKEN
 
- EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
- export EC_API_KEY_SECRET
- EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
- export EC_HOST_SECRET
  EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
  export EC_REGION_SECRET
  fi

.buildkite/hooks/pre-exit

@@ -4,8 +4,14 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
- # FIXME: update condition depending on the pipeline steps triggered
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
+ cd "${WORKSPACE}"
+ git config remote.origin.url "https://github.com/elastic/integrations.git"
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack|integrations-serverless)$ ]]; then
+ # it should match "^test-integration-" steps created in the integrations and integrations-test-stack pipelines (e.g. test-integration-apache or test-integration-aws)
+ # as well as the step ID "test-integrations-serverless-project" from the "integrations-serverless" pipeline
  if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
 
  # Ensure that kind cluster is deleted
@@ -16,27 +22,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]];
  echo "--- Take down the Elastic stack"
  ${ELASTIC_PACKAGE_BIN} stack down -v
  fi
- fi
-fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
- if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
 
- # Ensure that kind cluster is deleted
- delete_kind_cluster
-
- # Ensure elastic stack is stopped
- if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
- echo "--- Take down the Elastic stack"
- EC_API_KEY=${EC_API_KEY_SECRET} EC_HOST=${EC_HOST_SECRET} ${ELASTIC_PACKAGE_BIN} stack down -v
- fi
+ echo "+++ :bookmark: Documentation to access logs"
+ inline_link "https://docs.elastic.dev/ingest-dev-docs/elastic-packages/ecosystem-ci-pipelines#private-logs"
  fi
 fi
 
+echo "--- Cleaning up"
 unset_secrets
 cleanup
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
- cd "${WORKSPACE}"
- git config remote.origin.url "https://github.com/elastic/integrations.git"
-fi

.buildkite/pipeline.serverless.yml

@@ -73,11 +73,16 @@ steps:
  - elastic/oblt-aws-auth#v0.1.0:
  duration: 10800 # seconds
  # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
- # This plugin authenticates to Google Cloud using the OIDC token.
+ # This plugin authenticates to CI Google Cloud using the OIDC token.
  - elastic/oblt-google-auth#v1.3.0:
  lifetime: 10800 # seconds
  project-id: "elastic-observability-ci"
  project-number: "911195782929"
+ lifetime: 10800 # seconds
+ - avaly/gcp-secret-manager#v1.2.0:
+ env:
+ EC_API_KEY: elastic-cloud-observability-team-qa-api-key
+ EC_HOST: elastic-cloud-observability-team-qa-endpoint
  artifact_paths:
  - "build/test-results/*.xml"
  - "build/elastic-stack-dump/*/logs/*.log"

.buildkite/pull-requests.json

@@ -13,7 +13,7 @@
  "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$|^/test benchmark fullreport$",
  "skip_ci_labels": [],
  "skip_target_branches": [],
- "skip_ci_on_only_changed": ["^.github/workflows/", "^.github/dependabot.yml", "^.github/ISSUE_TEMPLATE/", "^docs/"],
+ "skip_ci_on_only_changed": ["^.github/workflows/", "^.github/dependabot.yml$", "^.github/ISSUE_TEMPLATE/", "^docs/", "^catalog-info.yaml$", "^.buildkite/pull-requests.json$"],
  "always_require_ci_on_changed": []
  },
  {

.buildkite/scripts/backport_branch.sh

@@ -96,10 +96,16 @@ createLocalBackportBranch() {
 
 removeOtherPackages() {
  local sourceFolder=$1
+ local currentPackage=""
  for dir in "$sourceFolder"/*; do
  if [[ -d "$dir" ]] && [[ "$(basename "$dir")" != "$PACKAGE_NAME" ]]; then
  echo "Removing directory: $dir"
  rm -rf "$dir"
+
+ currentPackage=$(basename "${dir}")
+ echo "Removing ${currentPackage} from .github/CODEOWNERS"
+ sed -i "/^\/packages\/${currentPackage}\//d" .github/CODEOWNERS
+ sed -i "/^\/packages\/${currentPackage} /d" .github/CODEOWNERS
  fi
  done
 }
@@ -116,70 +122,90 @@ updateBackportBranchContents() {
  local BUILDKITE_FOLDER_PATH=".buildkite"
  local JENKINS_FOLDER_PATH=".ci"
  local files_cached_num=""
+
+ git checkout "$BACKPORT_BRANCH_NAME"
+ echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
+ git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
+ git add $BUILDKITE_FOLDER_PATH
+
  if git ls-tree -d --name-only main:.ci >/dev/null 2>&1; then
- git checkout $BACKPORT_BRANCH_NAME
- echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
- git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
  echo "Copying $JENKINS_FOLDER_PATH from $SOURCE_BRANCH..."
  git checkout $SOURCE_BRANCH -- $JENKINS_FOLDER_PATH
+ git add $JENKINS_FOLDER_PATH
  else
- git checkout $BACKPORT_BRANCH_NAME
- echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
- git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
- echo "Removing $JENKINS_FOLDER_PATH from $BACKPORT_BRANCH_NAME..."
- rm -rf "$JENKINS_FOLDER_PATH"
+ if [ -d "${JENKINS_FOLDER_PATH}" ]; then
+  echo "Removing $JENKINS_FOLDER_PATH from $BACKPORT_BRANCH_NAME..."
+  rm -rf "$JENKINS_FOLDER_PATH"
+  git add "$JENKINS_FOLDER_PATH"
+ fi
  fi
 
  # Update scripts used by mage
  local MAGEFILE_SCRIPTS_FOLDER="dev/citools"
  local TESTSREPORTER_SCRIPTS_FOLDER="dev/testsreporter"
  local COVERAGE_SCRIPTS_FOLDER="dev/coverage"
+ local CODEOWNERS_SCRIPTS_FOLDER="dev/codeowners"
+
  if git ls-tree -d --name-only main:${MAGEFILE_SCRIPTS_FOLDER} > /dev/null 2>&1 ; then
  echo "Copying $MAGEFILE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
  git checkout "$SOURCE_BRANCH" -- "${MAGEFILE_SCRIPTS_FOLDER}"
+ git add ${MAGEFILE_SCRIPTS_FOLDER}
+
  echo "Copying $TESTSREPORTER_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
  git checkout "$SOURCE_BRANCH" -- "${TESTSREPORTER_SCRIPTS_FOLDER}"
+ git add ${TESTSREPORTER_SCRIPTS_FOLDER}
+
  echo "Copying $COVERAGE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
  git checkout "$SOURCE_BRANCH" -- "${COVERAGE_SCRIPTS_FOLDER}"
+ git add ${COVERAGE_SCRIPTS_FOLDER}
+
+ echo "Copying $CODEOWNERS_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
+ git checkout "$SOURCE_BRANCH" -- "${CODEOWNERS_SCRIPTS_FOLDER}"
+ git add ${CODEOWNERS_SCRIPTS_FOLDER}
+
  echo "Copying magefile.go from $SOURCE_BRANCH..."
  git checkout "$SOURCE_BRANCH" -- "magefile.go"
+ git add magefile.go
+
+ # As this script runs in the context of the main branch (mainly go mod tidy), we need to copy
+ # the .go-version file from the main branch to the backport branch. This avoids failures
+ # installing dependencies in the backport Pull Request.
+ echo "Copying .go-version from $SOURCE_BRANCH..."
+ git checkout "$SOURCE_BRANCH" -- ".go-version"
+ git add .go-version
+
  # Run go mod tidy to update just the dependencies related to magefile and dev scripts
  go mod tidy
+
+ git add go.mod go.sum
  fi
 
  if [ "${REMOVE_OTHER_PACKAGES}" == "true" ]; then
  echo "Removing all packages from $PACKAGES_FOLDER_PATH folder"
  removeOtherPackages "${PACKAGES_FOLDER_PATH}"
- ls -la $PACKAGES_FOLDER_PATH
+ ls -la "${PACKAGES_FOLDER_PATH}"
+
+ git add "${PACKAGES_FOLDER_PATH}/"
+ git add .github/CODEOWNERS
  fi
 
+ git status
+
  echo "Setting up git environment..."
  update_git_config
 
- echo "Commiting"
- git add $BUILDKITE_FOLDER_PATH
- if [ -d "${JENKINS_FOLDER_PATH}" ]; then
- git add "${JENKINS_FOLDER_PATH}"
- fi
- if [ -d "${MAGEFILE_SCRIPTS_FOLDER}" ] ; then
- git add ${MAGEFILE_SCRIPTS_FOLDER}
- git add ${TESTSREPORTER_SCRIPTS_FOLDER}
- git add go.mod go.sum
- fi
- git add $PACKAGES_FOLDER_PATH/
- git status
-
  files_cached_num=$(git diff --name-only --cached | wc -l)
  if [ "${files_cached_num}" -gt 0 ]; then
+ echo "Committing changes..."
  git commit -m "Add $BUILDKITE_FOLDER_PATH and $JENKINS_FOLDER_PATH to backport branch: $BACKPORT_BRANCH_NAME from the $SOURCE_BRANCH branch"
  else
  echo "Nothing to commit, skip."
  fi
 
  if [ "$DRY_RUN" == "true" ];then
  echo "DRY_RUN mode, nothing will be pushed."
- # Show just the relevant files diff (go.mod, go.sum, .buildkite, dev and package to be backported)
- git --no-pager diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME go.mod go.sum .buildkite/ dev/ "packages/${PACKAGE_NAME}"
+ # Show just the relevant files diff (go.mod, go.sum, .buildkite, dev, .go-version, .github/CODEOWNERS and package to be backported)
+ git --no-pager diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME .buildkite/ dev/ go.sum go.mod .go-version .github/CODEOWNERS "packages/${PACKAGE_NAME}"
  else
  echo "Pushing..."
  git push origin $BACKPORT_BRANCH_NAME

.buildkite/scripts/common.sh

@@ -540,9 +540,6 @@ prepare_serverless_stack() {
  fi
  create_elastic_package_profile "${profile_name}"
 
- export EC_API_KEY=${EC_API_KEY_SECRET}
- export EC_HOST=${EC_HOST_SECRET}
-
  echo "Boot up the Elastic stack"
  # grep command required to remove password from the output
  if ! ${ELASTIC_PACKAGE_BIN} stack up \
@@ -749,7 +746,7 @@ is_pr_affected() {
  # Example:
  # https://buildkite.com/elastic/integrations/builds/25606
  # https://github.com/elastic/integrations/pull/13810
- if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' > /dev/null; then
+ if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/|catalog-info\.yaml|\.buildkite/pull-requests\.json)' > /dev/null; then
  echo "[${package}] PR is affected: found non-package files"
  return 0
  fi
@@ -1112,3 +1109,19 @@ get_comment_with_pattern() {
 
  echo "${comment_id}"
 }
+
+## Buildkite output
+# https://buildkite.com/docs/pipelines/configure/links-and-images-in-log-output#links
+inline_link() {
+ local url="$1"
+ local text="${2:-""}"
+ local link=""
+
+ link=$(printf "url='%s'" "$url")
+
+ if [[ "${text}" != "" ]]; then
+ link=$(printf "%s;content='%s'" "$link" "$text")
+ fi
+
+ printf '\033]1339;%s\a\n' "$link"
+}

.github/CODEOWNERS

@@ -21,6 +21,7 @@
 /packages/apache_tomcat @elastic/obs-infraobs-integrations
 /packages/apm @elastic/obs-ds-intake-services
 /packages/arista_ngfw @elastic/sec-deployment-and-devices
+/packages/armis @elastic/security-service-integrations
 /packages/atlassian_bitbucket @elastic/security-service-integrations
 /packages/atlassian_confluence @elastic/security-service-integrations
 /packages/atlassian_jira @elastic/security-service-integrations
@@ -38,6 +39,7 @@
 /packages/aws/data_stream/cloudtrail @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/cloudwatch_logs @elastic/obs-ds-hosted-services
 /packages/aws/data_stream/cloudwatch_metrics @elastic/obs-ds-hosted-services
+/packages/aws/data_stream/config @elastic/security-service-integrations
 /packages/aws/data_stream/dynamodb @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/ebs @elastic/obs-ds-hosted-services
 /packages/aws/data_stream/ec2_logs @elastic/obs-ds-hosted-services
@@ -50,6 +52,7 @@
 /packages/aws/data_stream/guardduty @elastic/security-service-integrations
 /packages/aws/data_stream/kinesis @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/lambda @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/lambda_logs @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/natgateway @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/rds @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/redshift @elastic/obs-infraobs-integrations
@@ -126,6 +129,7 @@
 /packages/bbot @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml
 /packages/beat @elastic/stack-monitoring
+/packages/beelzebub @elastic/security-service-integrations
 /packages/beyondinsight_password_safe @elastic/security-service-integrations
 /packages/beyondtrust_pra @elastic/security-service-integrations
 /packages/bitdefender @elastic/security-service-integrations
@@ -188,6 +192,7 @@
 /packages/dga @elastic/ml-ui @elastic/sec-applied-ml
 /packages/digital_guardian @elastic/security-service-integrations
 /packages/docker @elastic/obs-ds-hosted-services
+/packages/docker_otel @elastic/obs-ds-hosted-services
 /packages/elastic_agent @elastic/elastic-agent
 /packages/elastic_connectors @elastic/search-extract-and-transform
 /packages/elastic_package_registry @elastic/ecosystem

.github/ISSUE_TEMPLATE/integration_bug.yml

@@ -131,6 +131,7 @@ body:
  - Defend for Containers [cloud_defend]
  - Digital Guardian [digital_guardian]
  - Docker [docker]
+ - Docker OpenTelemetry Assets [docker_otel]
  - Domain Generation Algorithm Detection [dga]
  - DomainTools Real Time Unified Feeds [ti_domaintools]
  - EclecticIQ [ti_eclecticiq]

.github/ISSUE_TEMPLATE/integration_feature_request.yml

@@ -131,6 +131,7 @@ body:
  - Defend for Containers [cloud_defend]
  - Digital Guardian [digital_guardian]
  - Docker [docker]
+ - Docker OpenTelemetry Assets [docker_otel]
  - Domain Generation Algorithm Detection [dga]
  - DomainTools Real Time Unified Feeds [ti_domaintools]
  - EclecticIQ [ti_eclecticiq]