Skip to content

tests: make integration split quantity configurable #17219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 19, 2025
Merged

tests: make integration split quantity configurable #17219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4ac629d
tests: make integration split quantity configurable
yaauie Mar 12, 2025
4f2edde
ci: run integration tests in 3 parts
yaauie Mar 13, 2025
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 35 additions & 12 deletions .buildkite/aarch64_pipeline.yml
View file
Open in desktop
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this correctly generates the parts for logstash-aarch64-pipeline: https://buildkite.com/elastic/logstash-aarch64-pipeline/builds/322#_
Original file line number Diff line number Diff line change
Expand Up @@ -35,48 +35,71 @@ steps:
automatic:
- limit: 3

- label: ":lab_coat: Integration Tests / part 1"
key: "integration-tests-part-1"
- label: ":lab_coat: Integration Tests / part 1-of-3"
key: "integration-tests-part-1-of-3"
command: |
set -euo pipefail

source .buildkite/scripts/common/vm-agent.sh
ci/integration_tests.sh split 0
ci/integration_tests.sh split 0 3
retry:
automatic:
- limit: 3

- label: ":lab_coat: Integration Tests / part 2"
key: "integration-tests-part-2"
- label: ":lab_coat: Integration Tests / part 2-of-3"
key: "integration-tests-part-2-of-3"
command: |
set -euo pipefail

source .buildkite/scripts/common/vm-agent.sh
ci/integration_tests.sh split 1
ci/integration_tests.sh split 1 3
retry:
automatic:
- limit: 3

- label: ":lab_coat: IT Persistent Queues / part 1"
key: "integration-tests-qa-part-1"
- label: ":lab_coat: Integration Tests / part 3-of-3"
key: "integration-tests-part-3-of-3"
command: |
set -euo pipefail

source .buildkite/scripts/common/vm-agent.sh
ci/integration_tests.sh split 2 3
retry:
automatic:
- limit: 3

- label: ":lab_coat: IT Persistent Queues / part 1-of-3"
key: "integration-tests-qa-part-1-of-3"
command: |
set -euo pipefail

source .buildkite/scripts/common/vm-agent.sh
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split 0 3
retry:
automatic:
- limit: 3

- label: ":lab_coat: IT Persistent Queues / part 2-of-3"
key: "integration-tests-qa-part-2-of-3"
command: |
set -euo pipefail

source .buildkite/scripts/common/vm-agent.sh
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split 0
ci/integration_tests.sh split 1 3
retry:
automatic:
- limit: 3

- label: ":lab_coat: IT Persistent Queues / part 2"
key: "integration-tests-qa-part-2"
- label: ":lab_coat: IT Persistent Queues / part 3-of-3"
key: "integration-tests-qa-part-3-of-3"
command: |
set -euo pipefail

source .buildkite/scripts/common/vm-agent.sh
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split 1
ci/integration_tests.sh split 2 3
retry:
automatic:
- limit: 3
Expand Down
61 changes: 49 additions & 12 deletions .buildkite/pull_request_pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ steps:
manual:
allowed: true

- label: ":lab_coat: Integration Tests / part 1"
key: "integration-tests-part-1"
- label: ":lab_coat: Integration Tests / part 1-of-3"
key: "integration-tests-part-1-of-3"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
cpu: "8"
Expand All @@ -97,10 +97,10 @@ steps:
set -euo pipefail

source .buildkite/scripts/common/container-agent.sh
ci/integration_tests.sh split 0
ci/integration_tests.sh split 0 3

- label: ":lab_coat: Integration Tests / part 2"
key: "integration-tests-part-2"
- label: ":lab_coat: Integration Tests / part 2-of-3"
key: "integration-tests-part-2-of-3"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
cpu: "8"
Expand All @@ -115,10 +115,47 @@ steps:
set -euo pipefail

source .buildkite/scripts/common/container-agent.sh
ci/integration_tests.sh split 1
ci/integration_tests.sh split 1 3

- label: ":lab_coat: IT Persistent Queues / part 1"
key: "integration-tests-qa-part-1"
- label: ":lab_coat: Integration Tests / part 3-of-3"
key: "integration-tests-part-3-of-3"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
cpu: "8"
memory: "16Gi"
ephemeralStorage: "100Gi"
# Run as a non-root user
imageUID: "1002"
retry:
automatic:
- limit: 3
command: |
set -euo pipefail

source .buildkite/scripts/common/container-agent.sh
ci/integration_tests.sh split 2 3

- label: ":lab_coat: IT Persistent Queues / part 1-of-3"
key: "integration-tests-qa-part-1-of-3"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
cpu: "8"
memory: "16Gi"
ephemeralStorage: "100Gi"
# Run as non root (logstash) user. UID is hardcoded in image.
imageUID: "1002"
retry:
automatic:
- limit: 3
command: |
set -euo pipefail

source .buildkite/scripts/common/container-agent.sh
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split 0 3

- label: ":lab_coat: IT Persistent Queues / part 2-of-3"
key: "integration-tests-qa-part-2-of-3"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
cpu: "8"
Expand All @@ -134,10 +171,10 @@ steps:

source .buildkite/scripts/common/container-agent.sh
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split 0
ci/integration_tests.sh split 1 3

- label: ":lab_coat: IT Persistent Queues / part 2"
key: "integration-tests-qa-part-2"
- label: ":lab_coat: IT Persistent Queues / part 3-of-3"
key: "integration-tests-qa-part-3-of-3"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
cpu: "8"
Expand All @@ -153,7 +190,7 @@ steps:

source .buildkite/scripts/common/container-agent.sh
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split 1
ci/integration_tests.sh split 2 3

- label: ":lab_coat: x-pack unit tests"
key: "x-pack-unit-tests"
Expand Down
50 changes: 21 additions & 29 deletions .buildkite/scripts/jdk-matrix-tests/generate-steps.py
View file
Open in desktop
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this correctly generates the parts for logstash-exhaustive-tests-pipeline: https://buildkite.com/elastic/logstash-exhaustive-tests-pipeline/builds/1516
Original file line number Diff line number Diff line change
Expand Up @@ -177,17 +177,15 @@ def __init__(self, os: str, jdk: str, group_key: str, agent: typing.Union[GCPAge
super().__init__(os=os, jdk=jdk, group_key=group_key, agent=agent)

def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]:
return [
self.init_annotation,
self.java_unit_test,
self.ruby_unit_test,
self.integration_tests_part_1,
self.integration_tests_part_2,
self.pq_integration_tests_part_1,
self.pq_integration_tests_part_2,
self.x_pack_unit_tests,
self.x_pack_integration,
]
jobs=list()
jobs.append(self.init_annotation)
jobs.append(self.java_unit_test)
jobs.append(self.ruby_unit_test)
jobs.extend(self.integration_test_parts(3))
jobs.extend(self.pq_integration_test_parts(3))
jobs.append(self.x_pack_unit_tests)
jobs.append(self.x_pack_integration)
return jobs

def prepare_shell(self) -> str:
jdk_dir = f"/opt/buildkite-agent/.java/{self.jdk}"
Expand Down Expand Up @@ -259,17 +257,14 @@ def ruby_unit_test(self) -> JobRetValues:
retry=copy.deepcopy(ENABLED_RETRIES),
)

def integration_tests_part_1(self) -> JobRetValues:
return self.integration_tests(part=1)
def integration_test_parts(self, parts) -> list[JobRetValues]:
return list(map(lambda idx: integration_tests(self, idx+1, parts), range(parts))

def integration_tests_part_2(self) -> JobRetValues:
return self.integration_tests(part=2)

def integration_tests(self, part: int) -> JobRetValues:
step_name_human = f"Integration Tests - {part}"
step_key = f"{self.group_key}-integration-tests-{part}"
def integration_tests(self, part: int, parts: int) -> JobRetValues:
step_name_human = f"Integration Tests - {part}/{parts}"
step_key = f"{self.group_key}-integration-tests-{part}-of-{parts}"
test_command = f"""
ci/integration_tests.sh split {part-1}
ci/integration_tests.sh split {part-1} {parts}
"""

return JobRetValues(
Expand All @@ -281,18 +276,15 @@ def integration_tests(self, part: int) -> JobRetValues:
retry=copy.deepcopy(ENABLED_RETRIES),
)

def pq_integration_tests_part_1(self) -> JobRetValues:
return self.pq_integration_tests(part=1)

def pq_integration_tests_part_2(self) -> JobRetValues:
return self.pq_integration_tests(part=2)
def pq_integration_test_parts(self, parts) -> list[JobRetValues]:
return list(map(lambda idx: pq_integration_tests(self, idx+1, parts), range(parts))

def pq_integration_tests(self, part: int) -> JobRetValues:
step_name_human = f"IT Persistent Queues - {part}"
step_key = f"{self.group_key}-it-persistent-queues-{part}"
def pq_integration_tests(self, part: int, parts: int) -> JobRetValues:
step_name_human = f"IT Persistent Queues - {part}/{parts}"
step_key = f"{self.group_key}-it-persistent-queues-{part}-of-{parts}"
test_command = f"""
export FEATURE_FLAG=persistent_queues
ci/integration_tests.sh split {part-1}
ci/integration_tests.sh split {part-1} {parts}
"""

return JobRetValues(
Expand Down
27 changes: 0 additions & 27 deletions ci/get-test-half.sh
View file
Open in desktop

This file was deleted.

20 changes: 9 additions & 11 deletions ci/integration_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,6 @@ export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g -Dorg.gradle.daemon=false
export SPEC_OPTS="--order rand --format documentation"
export CI=true

# Source shared function for splitting integration tests
source "$(dirname "${BASH_SOURCE[0]}")/get-test-half.sh"

if [ -n "$BUILD_JAVA_HOME" ]; then
GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
fi
Expand All @@ -22,14 +19,15 @@ if [[ $1 = "setup" ]]; then
exit 0

elif [[ $1 == "split" ]]; then
if [[ $2 =~ ^[01]$ ]]; then
specs=$(get_test_half "$2")
echo "Running half $2 of integration specs: $specs"
./gradlew runIntegrationTests -PrubyIntegrationSpecs="$specs" --console=plain
else
echo "Error, must specify 0 or 1 after the split. For example ci/integration_tests.sh split 0"
exit 1
fi
# Source shared function for splitting integration tests
source "$(dirname "${BASH_SOURCE[0]}")/partition-files.lib.sh"

index="${2:?index}"
count="${3:-2}"
specs=($(cd qa/integration; partition_files "${index}" "${count}" < <(find specs -name '*_spec.rb') ))

echo "Running integration tests partition[${index}] of ${count}: ${specs[*]}"
./gradlew runIntegrationTests -PrubyIntegrationSpecs="${specs[*]}" --console=plain

elif [[ ! -z $@ ]]; then
echo "Running integration tests 'rspec $@'"
Expand Down
Loading