Merge remote-tracking branch 'upstream/main' into cisco_ios_hostname

Author: mjwolf

.buildkite/hooks/pre-command

@@ -11,6 +11,8 @@ git config --global core.pager 'cat'
 export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export SERVERLESS=${SERVERLESS:-"false"}
 export STACK_VERSION=${STACK_VERSION:-""}
+export ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""}
+export STACK_LOGSDB_ENABLED=${STACK_LOGSDB_ENABLED:-"false"}
 export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
 export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
 

.buildkite/scripts/common.sh

@@ -18,6 +18,8 @@ API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/
 COVERAGE_FORMAT="generic"
 COVERAGE_OPTIONS="--test-coverage --coverage-format=${COVERAGE_FORMAT}"
 
+FATAL_ERROR="Fatal Error"
+
 running_on_buildkite() {
  if [[ "${BUILDKITE:-"false"}" == "true" ]]; then
  return 0
@@ -509,9 +511,15 @@ prepare_stack() {
  fi
 
  if [ "${STACK_LOGSDB_ENABLED:-false}" == "true" ]; then
+ echoerr "- Enable LogsDB"
  args="${args} -U stack.logsdb_enabled=true"
  fi
 
+ if [ "${ELASTIC_SUBSCRIPTION:-""}" != "" ]; then
+ echoerr "- Set Subscription ${ELASTIC_SUBSCRIPTION}"
+ args="${args} -U stack.elastic_subscription=${ELASTIC_SUBSCRIPTION}"
+ fi
+
  if [[ "${STACK_VERSION}" =~ ^7\.17 ]]; then
  # Required starting with STACK_VERSION 7.17.21
  export ELASTIC_AGENT_IMAGE_REF_OVERRIDE="docker.elastic.co/beats/elastic-agent-complete:${STACK_VERSION}-amd64"
@@ -668,6 +676,16 @@ get_to_changeset() {
  echo "${to}"
 }
 
+is_subscription_compatible() {
+ local reason=""
+
+ if ! reason=$(mage -d "${WORKSPACE}" -w . isSubscriptionCompatible) ; then
+ return 1
+ fi
+ echo "${reason}"
+ return 0
+}
+
 is_pr_affected() {
  local package="${1}"
  local from="${2}"
@@ -698,6 +716,15 @@ is_pr_affected() {
  return 1
  fi
  fi
+ local compatible=""
+ if ! compatible=$(is_subscription_compatible); then
+ echo "${FATAL_ERROR}"
+ return 1
+ fi
+ if [[ "${compatible}" == "false" ]]; then
+ echo "[${package}] PR is not affected: subscription not compatible with ${ELASTIC_SUBSCRIPTION}"
+ return 1
+ fi
 
  if [[ "${FORCE_CHECK_ALL}" == "true" ]];then
  echo "[${package}] PR is affected: \"force_check_all\" parameter enabled"

.buildkite/scripts/test_integrations_with_serverless.sh

@@ -43,6 +43,7 @@ if [ ! -d packages ]; then
  exit 1
 fi
 
+echo "--- Install requirements"
 add_bin_path
 
 with_yq
@@ -60,6 +61,7 @@ sleep 120
 echo "Done."
 
 # setting range of changesets to check differences
+echo "--- Get from and to changesets"
 from="$(get_from_changeset)"
 if [[ "${from}" == "" ]]; then
  echo "Missing \"from\" changset".
@@ -76,15 +78,28 @@ any_package_failing=0
 
 pushd packages > /dev/null
 for package in $(list_all_directories); do
+ echo "--- [$package] check if it is required to be tested"
  pushd "${package}" > /dev/null
+ skip_package=false
+ failure=false
  if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then
- echo "${reason}"
+ skip_package=true
+ if [[ "${reason}" == "${FATAL_ERROR}" ]]; then
+ failure=true
+ fi
+ fi
+ popd > /dev/null
+ if [[ "${failure}" == "true" ]]; then
+ echo "Unexpected failure checking ${package}"
+ exit 1
+ fi
+
+ echo "${reason}"
+
+ if [[ "${skip_package}" == "true" ]]; then
  echo "- ${reason}" >> "${SKIPPED_PACKAGES_FILE_PATH}"
- popd > /dev/null
  continue
  fi
- echo "${reason}"
- popd > /dev/null
 
  if ! process_package "${package}" "${FAILED_PACKAGES_FILE_PATH}" ; then
  any_package_failing=1
@@ -94,10 +109,12 @@ popd > /dev/null
 
 if running_on_buildkite ; then
  if [ -f "${SKIPPED_PACKAGES_FILE_PATH}" ]; then
+ echo "--- Create Skip Buildkite annotation"
  create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" "${SKIPPED_PACKAGES_FILE_PATH}" "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}"
  fi
 
  if [ -f "${FAILED_PACKAGES_FILE_PATH}" ]; then
+ echo "--- Create Failed Buildkite annotation"
  create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" "${FAILED_PACKAGES_FILE_PATH}" "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
  fi
 fi

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -4,8 +4,10 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
+echo "--- Install requirements"
 add_bin_path
 with_yq
+with_mage
 
 pushd packages > /dev/null
 PACKAGE_LIST=$(list_all_directories)
@@ -22,8 +24,8 @@ steps:
 EOF
 
 # Get from and to changesets to avoid repeating the same queries for each package
-
 # setting range of changesets to check differences
+echo "--- Get from and to changesets"
 from="$(get_from_changeset)"
 if [[ "${from}" == "" ]]; then
  echo "Missing \"from\" changset".
@@ -51,15 +53,24 @@ packages_to_test=0
 
 for package in ${PACKAGE_LIST}; do
  # check if needed to create an step for this package
+ echo "--- [$package] check if it is required to be tested"
  pushd "packages/${package}" > /dev/null
  skip_package="false"
+ failure="false"
  if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then
  skip_package="true"
+ if [[ "${reason}" == "${FATAL_ERROR}" ]]; then
+ failure=true
+ fi
  fi
- echoerr "${reason}"
  popd > /dev/null
+ if [[ "${failure}" == "true" ]]; then
+ echo "Unexpected failure checking ${package}"
+ exit 1
+ fi
 
- if [[ "$skip_package" == "true" ]] ; then
+ echoerr "${reason}"
+ if [[ "${skip_package}" == "true" ]] ; then
  continue
  fi
 
@@ -87,8 +98,10 @@ EOF
 done
 
 if [ ${packages_to_test} -eq 0 ]; then
+ echo "--- Create Buildkite annotation no packages to be tested"
  buildkite-agent annotate "No packages to be tested" --context "ctx-no-packages" --style "warning"
  exit 0
 fi
 
+echo "--- Upload Buildkite pipeline"
 cat ${PIPELINE_FILE} | buildkite-agent pipeline upload

.go-version

@@ -1 +1 @@
-1.23.4
+1.24.2

dev/citools/packagemanifest.go

@@ -0,0 +1,48 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package citools
+
+import (
+"fmt"
+
+"github.com/elastic/go-ucfg"
+"github.com/elastic/go-ucfg/yaml"
+)
+
+// kibanaConditions defines conditions for Kibana (e.g. required version).
+type kibanaConditions struct {
+Version string `config:"version" json:"version" yaml:"version"`
+}
+
+// elasticConditions defines conditions related to Elastic subscriptions or partnerships.
+type elasticConditions struct {
+Subscription string `config:"subscription" json:"subscription" yaml:"subscription"`
+}
+
+// conditions define requirements for different parts of the Elastic stack.
+type conditions struct {
+Kibana kibanaConditions `config:"kibana" json:"kibana" yaml:"kibana"`
+Elastic elasticConditions `config:"elastic" json:"elastic" yaml:"elastic"`
+}
+
+type packageManifest struct {
+Name string `config:"name" json:"name" yaml:"name"`
+License string `config:"license" json:"license" yaml:"license"`
+Conditions conditions `config:"conditions" json:"conditions" yaml:"conditions"`
+}
+
+func readPackageManifest(path string) (*packageManifest, error) {
+cfg, err := yaml.NewConfigWithFile(path, ucfg.PathSep("."))
+if err != nil {
+return nil, fmt.Errorf("reading file failed (path: %s): %w", path, err)
+}
+
+var manifest packageManifest
+err = cfg.Unpack(&manifest)
+if err != nil {
+return nil, fmt.Errorf("unpacking package manifest failed (path: %s): %w", path, err)
+}
+return &manifest, nil
+}

dev/citools/subscription.go

@@ -0,0 +1,47 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package citools
+
+import (
+"fmt"
+)
+
+func packageSubscription(path string) (string, error) {
+manifest, err := readPackageManifest(path)
+if err != nil {
+return "", err
+}
+
+packageSubscription := manifest.Conditions.Elastic.Subscription
+if packageSubscription == "" {
+packageSubscription = manifest.License
+}
+if packageSubscription == "" {
+packageSubscription = "basic"
+}
+
+return packageSubscription, nil
+}
+
+func IsSubscriptionCompatible(stackSubscription, path string) (bool, error) {
+pkgSubscription, err := packageSubscription(path)
+if err != nil {
+return false, fmt.Errorf("failed to read subscription from manifest: %w", err)
+}
+
+if stackSubscription == "trial" {
+// All subscriptions supported
+return true, nil
+}
+
+if stackSubscription == "basic" {
+if pkgSubscription != "basic" {
+return false, nil
+}
+return true, nil
+}
+
+return false, fmt.Errorf("unknown subscription %s", stackSubscription)
+}