Skip to content

Commit 214b158

Browse files
davinchiadavinchia
authored
Add labels to EC2 build machines. Refactor actions to reduce duplicate start ec2 runner actions. (#7119)
As the title. I also took the chance to compose the actions, instead of injecting it everywhere.
1 parent 1441bf9 commit 214b158

File tree

5 files changed

+96
-84
lines changed

5 files changed

+96
-84
lines changed

‎.github/actions/start-aws-runner/action.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: 'Runner Start (AWS)'
2+
description: 'Starting Runner on AWS Cloud'
3+
inputs:
4+
aws-access-key-id:
5+
required: true
6+
aws-secret-access-key:
7+
required: true
8+
github-token:
9+
required: true
10+
ec2-image-id:
11+
default: 'ami-04bd6e81239f4f3fb'
12+
required: true
13+
ec2-instance-type:
14+
default: 'c5.2xlarge'
15+
required: true
16+
subnet-id:
17+
default: 'subnet-0469a9e68a379c1d3'
18+
required: true
19+
security-group-id:
20+
default: 'sg-0793f3c9413f21970'
21+
required: true
22+
label:
23+
required: false
24+
ec2-instance-id:
25+
required: false
26+
outputs:
27+
label:
28+
value: ${{ steps.start-ec2-runner.outputs.label }}
29+
ec2-instance-id:
30+
value: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
31+
32+
runs:
33+
using: "composite"
34+
steps:
35+
- name: Configure AWS credentials
36+
uses: aws-actions/configure-aws-credentials@v1
37+
with:
38+
aws-access-key-id: ${{ inputs.aws-access-key-id }}
39+
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
40+
aws-region: us-east-2
41+
- name: Start EC2 runner
42+
id: start-ec2-runner
43+
uses: machulav/ec2-github-runner@v2.2.1
44+
with:
45+
mode: start
46+
github-token: ${{ inputs.github-token }}
47+
ec2-image-id: ${{ inputs.ec2-image-id }}
48+
ec2-instance-type: ${{ inputs.ec2-instance-type }}
49+
subnet-id: ${{ inputs.subnet-id }}
50+
security-group-id: ${{ inputs.security-group-id }}
51+
aws-resource-tags: >
52+
[
53+
{"Key": "BuildType", "Value": "oss"},
54+
{"Key": "Repository", "Value": "${{ github.repository }}"},
55+
{"Key": "Branch", "Value": "${{ github.ref }}"}
56+
]

‎.github/workflows/gke-kube-test-command.yml‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,15 @@ jobs:
1717
label: ${{ steps.start-ec2-runner.outputs.label }}
1818
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
1919
steps:
20-
- name: Configure AWS credentials
21-
uses: aws-actions/configure-aws-credentials@v1
20+
- name: Checkout Airbyte
21+
uses: actions/checkout@v2
22+
- name: Start AWS Runner
23+
id: start-ec2-runner
24+
uses: ./.github/actions/start-aws-runner
2225
with:
2326
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
2427
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
25-
aws-region: us-east-2
26-
- name: Start EC2 runner
27-
id: start-ec2-runner
28-
uses: machulav/ec2-github-runner@v2.3.0
29-
with:
30-
mode: start
3128
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
32-
ec2-image-id: ami-04bd6e81239f4f3fb
33-
ec2-instance-type: c5.2xlarge
34-
subnet-id: subnet-0469a9e68a379c1d3
35-
security-group-id: sg-0793f3c9413f21970
3629
gke-kube-acceptance-test:
3730
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
3831
needs: start-gke-kube-acceptance-test-runner # required to start the main job when the runner is ready
@@ -47,6 +40,7 @@ jobs:
4740
comment-id: ${{ github.event.inputs.comment-id }}
4841
body: |
4942
> :clock2: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
43+
5044
- name: Checkout Airbyte
5145
uses: actions/checkout@v2
5246

‎.github/workflows/gradle.yml‎

Lines changed: 20 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,15 @@ jobs:
1616
label: ${{ steps.start-ec2-runner.outputs.label }}
1717
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
1818
steps:
19-
- name: Configure AWS credentials
20-
uses: aws-actions/configure-aws-credentials@v1
19+
- name: Checkout Airbyte
20+
uses: actions/checkout@v2
21+
- name: Start AWS Runner
22+
id: start-ec2-runner
23+
uses: ./.github/actions/start-aws-runner
2124
with:
2225
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
2326
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
24-
aws-region: us-east-2
25-
- name: Start EC2 Runner
26-
id: start-ec2-runner
27-
uses: machulav/ec2-github-runner@v2.3.0
28-
with:
29-
mode: start
3027
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
31-
ec2-image-id: ami-04bd6e81239f4f3fb
32-
ec2-instance-type: c5.2xlarge
33-
subnet-id: subnet-0469a9e68a379c1d3
34-
security-group-id: sg-0793f3c9413f21970
3528

3629
ensure-images-exist:
3730
name: "Ensure all required Docker images exist on Dockerhub"
@@ -171,22 +164,15 @@ jobs:
171164
label: ${{ steps.start-ec2-runner.outputs.label }}
172165
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
173166
steps:
174-
- name: Configure AWS credentials
175-
uses: aws-actions/configure-aws-credentials@v1
167+
- name: Checkout Airbyte
168+
uses: actions/checkout@v2
169+
- name: Start AWS Runner
170+
id: start-ec2-runner
171+
uses: ./.github/actions/start-aws-runner
176172
with:
177173
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
178174
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
179-
aws-region: us-east-2
180-
- name: Start EC2 Runner
181-
id: start-ec2-runner
182-
uses: machulav/ec2-github-runner@v2.3.0
183-
with:
184-
mode: start
185175
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
186-
ec2-image-id: ami-04bd6e81239f4f3fb
187-
ec2-instance-type: c5.2xlarge
188-
subnet-id: subnet-0469a9e68a379c1d3
189-
security-group-id: sg-0793f3c9413f21970
190176
platform-build:
191177
# In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
192178
needs: start-platform-build-runner # required to start the main job when the runner is ready
@@ -330,22 +316,15 @@ jobs:
330316
label: ${{ steps.start-ec2-runner.outputs.label }}
331317
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
332318
steps:
333-
- name: Configure AWS credentials
334-
uses: aws-actions/configure-aws-credentials@v1
319+
- name: Checkout Airbyte
320+
uses: actions/checkout@v2
321+
- name: Start AWS Runner
322+
id: start-ec2-runner
323+
uses: ./.github/actions/start-aws-runner
335324
with:
336325
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
337326
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
338-
aws-region: us-east-2
339-
- name: Start EC2 Runner
340-
id: start-ec2-runner
341-
uses: machulav/ec2-github-runner@v2.3.0
342-
with:
343-
mode: start
344327
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
345-
ec2-image-id: ami-04bd6e81239f4f3fb
346-
ec2-instance-type: c5.2xlarge
347-
subnet-id: subnet-0469a9e68a379c1d3
348-
security-group-id: sg-0793f3c9413f21970
349328
frontend-test:
350329
needs: start-frontend-test-runner # required to start the main job when the runner is ready
351330
runs-on: ${{ needs.start-frontend-test-runner.outputs.label }} # run the job on the newly created runner
@@ -422,22 +401,15 @@ jobs:
422401
label: ${{ steps.start-ec2-runner.outputs.label }}
423402
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
424403
steps:
425-
- name: Configure AWS credentials
426-
uses: aws-actions/configure-aws-credentials@v1
404+
- name: Checkout Airbyte
405+
uses: actions/checkout@v2
406+
- name: Start AWS Runner
407+
id: start-ec2-runner
408+
uses: ./.github/actions/start-aws-runner
427409
with:
428410
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
429411
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
430-
aws-region: us-east-2
431-
- name: Start EC2 runner
432-
id: start-ec2-runner
433-
uses: machulav/ec2-github-runner@v2.3.0
434-
with:
435-
mode: start
436412
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
437-
ec2-image-id: ami-0d4083c04fde515c4
438-
ec2-instance-type: c5.2xlarge
439-
subnet-id: subnet-0469a9e68a379c1d3
440-
security-group-id: sg-0793f3c9413f21970
441413
kube-acceptance-test:
442414
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
443415
needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready

‎.github/workflows/publish-command.yml‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,22 +23,17 @@ jobs:
2323
label: ${{ steps.start-ec2-runner.outputs.label }}
2424
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
2525
steps:
26-
- name: Configure AWS credentials
27-
uses: aws-actions/configure-aws-credentials@v1
26+
- name: Checkout Airbyte
27+
uses: actions/checkout@v2
2828
with:
29-
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
30-
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
31-
aws-region: us-east-2
32-
- name: Start EC2 Runner
29+
repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
30+
- name: Start AWS Runner
3331
id: start-ec2-runner
34-
uses: machulav/ec2-github-runner@v2
32+
uses: ./.github/actions/start-aws-runner
3533
with:
36-
mode: start
34+
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
35+
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
3736
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
38-
ec2-image-id: ami-0d648081937c75a73
39-
ec2-instance-type: c5.2xlarge
40-
subnet-id: subnet-0469a9e68a379c1d3
41-
security-group-id: sg-0793f3c9413f21970
4237
publish-image:
4338
needs: start-publish-image-runner
4439
runs-on: ${{ needs.start-publish-image-runner.outputs.label }}

‎.github/workflows/test-command.yml‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,22 +22,17 @@ jobs:
2222
label: ${{ steps.start-ec2-runner.outputs.label }}
2323
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
2424
steps:
25-
- name: Configure AWS credentials
26-
uses: aws-actions/configure-aws-credentials@v1
25+
- name: Checkout Airbyte
26+
uses: actions/checkout@v2
2727
with:
28-
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
29-
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
30-
aws-region: us-east-2
31-
- name: Start EC2 Runner
28+
repository: ${{ github.event.inputs.repo }}
29+
- name: Start AWS Runner
3230
id: start-ec2-runner
33-
uses: machulav/ec2-github-runner@v2
31+
uses: ./.github/actions/start-aws-runner
3432
with:
35-
mode: start
33+
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
34+
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
3635
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
37-
ec2-image-id: ami-0d648081937c75a73
38-
ec2-instance-type: c5.2xlarge
39-
subnet-id: subnet-0469a9e68a379c1d3
40-
security-group-id: sg-0793f3c9413f21970
4136
integration-test:
4237
timeout-minutes: 240
4338
needs: start-test-runner

0 commit comments

Comments
 (0)