Skip to content

Commit abb3140

Browse files
liudmylaruliudmylaru
authored
Re-arrange unitary tests workflows (#374)
1 parent b835edb commit abb3140

File tree

6 files changed

+153
-83
lines changed

6 files changed

+153
-83
lines changed

‎.github/workflows/create-more-space/action.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# We are running out space on the disk, solution described here:
2+
# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
3+
4+
name: "Create more disk space"
5+
description: "Removing some folders to create more disk space"
6+
runs:
7+
using: composite
8+
steps:
9+
- shell: bash
10+
run: |
11+
sudo rm -rf /usr/share/dotnet
12+
sudo rm -rf /opt/ghc
13+
sudo rm -rf "/usr/local/share/boost"
14+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"

‎.github/workflows/run-unittests-default_setup.yml‎

Lines changed: 8 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,13 @@
1-
name: tests/unitary/default_setup/**
1+
name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
22

33
on:
44
workflow_dispatch:
55
pull_request:
6-
branches:
7-
- main
8-
- "release/**"
9-
- develop
106
paths:
117
- "ads/**"
128
- pyproject.toml
139
- "**requirements.txt"
14-
- .github/workflows/run-unittests.yml
15-
- .github/workflows/run-unittests-default_setup.yml
10+
- ".github/workflows/run-unittests*.yml"
1611

1712
# Cancel in progress workflows on pull_requests.
1813
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -41,37 +36,16 @@ jobs:
4136
steps:
4237
- uses: actions/checkout@v3
4338

44-
# Caching python libraries installed with pip
45-
# https://github.com/actions/cache/blob/main/examples.md#python---pip
46-
- uses: actions/cache@v3
47-
with:
48-
path: ~/.cache/pip
49-
key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
50-
restore-keys: |
51-
${{ runner.os }}-pip-
5239
- uses: actions/setup-python@v4
5340
with:
5441
python-version: ${{ matrix.python-version }}
42+
cache: "pip"
43+
cache-dependency-path: |
44+
pyproject.toml
45+
"**requirements.txt"
5546
56-
- name: "Test config setup"
57-
shell: bash
58-
env:
59-
HOME_RUNNER_DIR: /home/runner
60-
run: |
61-
set -x # print commands that are executed
62-
mkdir -p "$HOME_RUNNER_DIR"/.oci
63-
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
64-
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
65-
[DEFAULT]
66-
user=ocid1.user.oc1..xxx
67-
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
68-
tenancy=ocid1.tenancy.oc1..xxx
69-
region=test_region
70-
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
71-
EOT
72-
ls -lha "$HOME_RUNNER_DIR"/.oci
73-
echo "Test config file:"
74-
cat $HOME_RUNNER_DIR/.oci/config
47+
- uses: ./.github/workflows/set-dummy-conf
48+
name: "Test config setup"
7549

7650
- name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
7751
timeout-minutes: 15

‎.github/workflows/run-unittests.yml‎ renamed to ‎.github/workflows/run-unittests-py38-cov-report.yml‎

Lines changed: 20 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,13 @@
1-
name: tests/unitary/**
1+
name: "[Py3.8][COV REPORT] tests/unitary/**"
22

33
on:
44
workflow_dispatch:
55
pull_request:
6-
branches:
7-
- main
8-
- "release/**"
9-
- develop
106
paths:
117
- "ads/**"
128
- pyproject.toml
139
- "**requirements.txt"
14-
- .github/workflows/run-unittests.yml
15-
- .github/workflows/run-unittests-default_setup.yml
10+
- ".github/workflows/run-unittests*.yml"
1611

1712
# Cancel in progress workflows on pull_requests.
1813
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -30,58 +25,37 @@ env:
3025

3126
jobs:
3227
test:
33-
name: python ${{ matrix.python-version }}, ${{ matrix.name }}
28+
name: python 3.8, ${{ matrix.name }}
3429
runs-on: ubuntu-latest
3530
timeout-minutes: 90
3631

3732
strategy:
3833
fail-fast: false
3934
matrix:
40-
python-version: ["3.8", "3.9", "3.10"]
4135
test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
4236
include:
4337
- test-path: "tests/unitary/with_extras tests/unitary/default_setup"
4438
ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
4539
name: "unitary"
4640
- test-path: "tests/unitary/with_extras/model"
4741
name: "model"
48-
- python-version: "3.8"
49-
cov-reports: --cov=ads --cov-report=xml --cov-report=html
5042

5143
steps:
5244
- uses: actions/checkout@v3
5345

54-
# Caching python libraries installed with pip
55-
# https://github.com/actions/cache/blob/main/examples.md#python---pip
56-
- uses: actions/cache@v3
57-
with:
58-
path: ~/.cache/pip
59-
key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
60-
restore-keys: |
61-
${{ runner.os }}-pip-
46+
- uses: ./.github/workflows/create-more-space
47+
name: "Create more disk space"
48+
6249
- uses: actions/setup-python@v4
6350
with:
64-
python-version: ${{ matrix.python-version }}
51+
python-version: "3.8"
52+
cache: "pip"
53+
cache-dependency-path: |
54+
pyproject.toml
55+
"**requirements.txt"
6556
66-
- name: "Test config setup"
67-
shell: bash
68-
env:
69-
HOME_RUNNER_DIR: /home/runner
70-
run: |
71-
set -x # print commands that are executed
72-
mkdir -p "$HOME_RUNNER_DIR"/.oci
73-
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
74-
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
75-
[DEFAULT]
76-
user=ocid1.user.oc1..xxx
77-
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
78-
tenancy=ocid1.tenancy.oc1..xxx
79-
region=test_region
80-
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
81-
EOT
82-
ls -lha "$HOME_RUNNER_DIR"/.oci
83-
echo "Test config file:"
84-
cat $HOME_RUNNER_DIR/.oci/config
57+
- uses: ./.github/workflows/set-dummy-conf
58+
name: "Test config setup"
8559

8660
- name: "Test env setup"
8761
timeout-minutes: 20
@@ -104,22 +78,19 @@ jobs:
10478
set -x # print commands that are executed
10579
10680
# Setup project and tests folder for cov reports to not be overwritten by another parallel step
107-
if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
108-
mkdir -p cov-${{ matrix.name }}
109-
cd cov-${{ matrix.name }}
110-
ln -s ../tests tests
111-
ln -s ../ads ads
112-
ln -s ../.coveragerc .coveragerc
113-
fi
81+
mkdir -p cov-${{ matrix.name }}
82+
cd cov-${{ matrix.name }}
83+
ln -s ../tests tests
84+
ln -s ../ads ads
85+
ln -s ../.coveragerc .coveragerc
11486
11587
# Run tests
11688
python -m pytest -v -p no:warnings --durations=5 \
117-
-n auto --dist loadfile ${{ matrix.cov-reports }} \
89+
-n auto --dist loadfile --cov=ads --cov-report=xml --cov-report=html \
11890
${{ matrix.test-path }} ${{ matrix.ignore-path }}
11991
12092
- name: "Save coverage files"
12193
uses: actions/upload-artifact@v3
122-
if: ${{ matrix.cov-reports }}
12394
with:
12495
name: cov-reports-${{ matrix.name }}
12596
path: |
@@ -132,7 +103,7 @@ jobs:
132103
runs-on: ubuntu-latest
133104
continue-on-error: true
134105
needs: test
135-
if: ${{ success() }} && ${{ github.event.issue.pull_request }}
106+
if: ${{ success() }} && && github.event_name == 'pull_request'
136107
env:
137108
COMPARE_BRANCH: main
138109

‎.github/workflows/run-unittests-py39-py310.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: "[Py3.9][Py3.10] - tests/unitary/**"
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- "ads/**"
8+
- pyproject.toml
9+
- "**requirements.txt"
10+
- ".github/workflows/run-unittests*.yml"
11+
12+
# Cancel in progress workflows on pull_requests.
13+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
16+
cancel-in-progress: true
17+
18+
permissions:
19+
contents: read
20+
pull-requests: write
21+
22+
# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
23+
env:
24+
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
25+
26+
jobs:
27+
test:
28+
name: python ${{ matrix.python-version }}, ${{ matrix.name }}
29+
runs-on: ubuntu-latest
30+
timeout-minutes: 90
31+
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
python-version: ["3.9", "3.10"]
36+
test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
37+
include:
38+
- test-path: "tests/unitary/with_extras tests/unitary/default_setup"
39+
ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
40+
name: "unitary"
41+
- test-path: "tests/unitary/with_extras/model"
42+
name: "model"
43+
44+
steps:
45+
- uses: actions/checkout@v3
46+
47+
- uses: ./.github/workflows/create-more-space
48+
name: "Create more disk space"
49+
50+
- uses: actions/setup-python@v4
51+
with:
52+
python-version: ${{ matrix.python-version }}
53+
cache: "pip"
54+
cache-dependency-path: |
55+
pyproject.toml
56+
"**requirements.txt"
57+
58+
- uses: ./.github/workflows/set-dummy-conf
59+
name: "Test config setup"
60+
61+
- name: "Test env setup"
62+
timeout-minutes: 20
63+
shell: bash
64+
run: |
65+
set -x # print commands that are executed
66+
67+
sudo apt-get install libkrb5-dev graphviz
68+
$CONDA/bin/conda init
69+
source /home/runner/.bashrc
70+
71+
pip install -r dev-requirements.txt
72+
73+
- name: "Run unitary tests folder with maximum ADS dependencies"
74+
timeout-minutes: 60
75+
shell: bash
76+
env:
77+
CONDA_PREFIX: /usr/share/miniconda
78+
run: |
79+
set -x # print commands that are executed
80+
81+
# Run tests
82+
python -m pytest -v -p no:warnings --durations=5 \
83+
-n auto --dist loadfile \
84+
${{ matrix.test-path }} ${{ matrix.ignore-path }}

‎.github/workflows/set-dummy-conf/action.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# This composite action is to avoid duplicating code across test actions.
2+
# Ref: https://docs.github.com/en/actions/creating-actions/creating-a-composite-action
3+
4+
name: "Create dummy oci config"
5+
description: "Creates dummy oci config folder in ~/.oci"
6+
runs:
7+
using: composite
8+
steps:
9+
- shell: bash
10+
env:
11+
HOME_RUNNER_DIR: /home/runner
12+
run: |
13+
set -x # print commands that are executed
14+
mkdir -p "$HOME_RUNNER_DIR"/.oci
15+
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
16+
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
17+
[DEFAULT]
18+
user=ocid1.user.oc1..xxx
19+
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
20+
tenancy=ocid1.tenancy.oc1..xxx
21+
region=test_region
22+
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
23+
EOT
24+
ls -lha "$HOME_RUNNER_DIR"/.oci
25+
echo "Test config file:"
26+
cat $HOME_RUNNER_DIR/.oci/config

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ repos:
3838
rev: v8.17.0
3939
hooks:
4040
- id: gitleaks
41+
exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
4142
# Oracle copyright checker
4243
- repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
4344
rev: cbe0136f7aaffe463b31ddf3f34b0e16b4b124ff

0 commit comments

Comments
 (0)