buildkite: use GCP OIDC (#13856) (#13912)

(cherry picked from commit 47b4432) Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>

Author: mergify[bot]

.buildkite/hooks/pre-command

@@ -29,8 +29,6 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 export REPO_BUILD_TAG
 
-PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
-
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
 
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
@@ -40,6 +38,8 @@ EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 export ENVIRONMENT="ci"
 export REPO="${REPO_NAME}"
 
+export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
+
 branch_name_label() {
  local branch="$1"
 
@@ -104,23 +104,13 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
  fi
 
  if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
- PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
- export PRIVATE_CI_GCS_CREDENTIALS_SECRET
- export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
-
  BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
  export BUILDKITE_API_TOKEN
  fi
 fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
  if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
- # Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
- # add the AWS credentials as in the integrations pipeline.
-
- PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
- export PRIVATE_CI_GCS_CREDENTIALS_SECRET
- export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 
  BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
  export BUILDKITE_API_TOKEN

.buildkite/hooks/pre-exit

@@ -7,10 +7,6 @@ set -euo pipefail
 if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
  # FIXME: update condition depending on the pipeline steps triggered
  if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
- unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
- unset ELASTIC_PACKAGE_AWS_SECRET_KEY
- unset AWS_ACCESS_KEY_ID
- unset AWS_SECRET_ACCESS_KEY
 
  # Ensure that kind cluster is deleted
  delete_kind_cluster
@@ -25,10 +21,6 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
  if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
- unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
- unset ELASTIC_PACKAGE_AWS_SECRET_KEY
- unset AWS_ACCESS_KEY_ID
- unset AWS_SECRET_ACCESS_KEY
 
  # Ensure that kind cluster is deleted
  delete_kind_cluster
@@ -44,8 +36,6 @@ fi
 unset_secrets
 cleanup
 
-google_cloud_logout_active_account
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
  cd "${WORKSPACE}"
  git config remote.origin.url "https://github.com/elastic/integrations.git"

.buildkite/pipeline.serverless.yml

@@ -72,6 +72,12 @@ steps:
  # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
  - elastic/oblt-aws-auth#v0.1.0:
  duration: 10800 # seconds
+ # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
+ # This plugin authenticates to Google Cloud using the OIDC token.
+ - elastic/oblt-google-auth#v1.3.0:
+ lifetime: 10800 # seconds
+ project-id: "elastic-observability-ci"
+ project-number: "911195782929"
  artifact_paths:
  - "build/test-results/*.xml"
  - "build/elastic-stack-dump/*/logs/*.log"

.buildkite/scripts/common.sh

@@ -10,7 +10,6 @@ platform_type_lowercase="${platform_type,,}"
 
 SCRIPTS_BUILDKITE_PATH="${WORKSPACE}/.buildkite/scripts"
 
-GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/pipelines/"
@@ -253,34 +252,6 @@ with_github_cli() {
  gh version
 }
 
-## Logging and logout from Google Cloud
-google_cloud_auth_safe_logs() {
- local gsUtilLocation
- gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
- local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-
- echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
-
- gcloud auth activate-service-account --key-file "${secretFileLocation}" 2> /dev/null
- export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
-}
-
-google_cloud_logout_active_account() {
- local active_account
- active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null || true)
- if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
- echo "Logging out from GCP for active account"
- gcloud auth revoke "$active_account" > /dev/null 2>&1
- else
- echo "No active GCP accounts found."
- fi
-
- if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
- rm -rf "${GOOGLE_APPLICATION_CREDENTIALS}"
- unset GOOGLE_APPLICATION_CREDENTIALS
- fi
-}
-
 ## Helpers for integrations pipelines
 check_git_diff() {
  cd "${WORKSPACE}"
@@ -918,16 +889,16 @@ upload_safe_logs() {
  local source="$2"
  local target="$3"
 
+ echo "--- Uploading safe logs to GCP bucket ${bucket}"
+
  if ! ls ${source} 2>&1 > /dev/null ; then
  echo "upload_safe_logs: artifacts files not found, nothing will be archived"
  return
  fi
 
- google_cloud_auth_safe_logs
-
- gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+ gcloud storage cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
 
- google_cloud_logout_active_account
+ echo "GCP logout is not required, the BK plugin will do it for us"
 }
 
 clean_safe_logs() {

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -74,6 +74,12 @@ for package in ${PACKAGE_LIST}; do
  # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
  - elastic/oblt-aws-auth#v0.1.0:
  duration: 10800 # seconds
+ # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
+ # This plugin authenticates to Google Cloud using the OIDC token.
+ - elastic/oblt-google-auth#v1.3.0:
+ lifetime: 10800 # seconds
+ project-id: "elastic-observability-ci"
+ project-number: "911195782929"
  artifact_paths:
  - build/test-results/*.xml
  - build/test-coverage/*.xml