ci(performance): Add performance tests to CI (espressif#9560)

* ci(performance): Add performance tests to CI * ci(req): Fix requirements * ci(pre-commit): Apply automatic fixes * ci(pre-commit): Increase maximum allowed complexity for python --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jan Procházka <90197375+P-R-O-C-H-Y@users.noreply.github.com>

Author: 3 people

.flake8

@@ -1,12 +1,10 @@
 # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-python/.flake8
 # See: https://flake8.pycqa.org/en/latest/user/configuration.html
-# The code style defined in this file is the official standardized style to be used in all Arduino tooling projects and
-# should not be modified.
 
 [flake8]
 doctests = True
 # W503 and W504 are mutually exclusive. PEP 8 recommends line break before.
 ignore = W503,E203
-max-complexity = 10
+max-complexity = 20
 max-line-length = 120
 select = E,W,F,C,N

.github/scripts/sketch_utils.sh

@@ -121,7 +121,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
  fi
 
  if [ -z "$fqbn" ]; then
- echo "No FQBN passed or unvalid chip: $target"
+ echo "No FQBN passed or invalid chip: $target"
  exit 1
  fi
 
@@ -139,7 +139,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
  echo "Skipping $sketchname for target $target"
  exit 0
  fi
- 
+
  ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
  if [ -n "$ARDUINO_BUILD_DIR" ]; then
  build_dir="$ARDUINO_BUILD_DIR"
@@ -177,7 +177,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
  --build-path "$build_dir" \
  $xtra_opts "${sketchdir}" \
  > $output_file
- 
+
  exit_status=$?
  if [ $exit_status -ne 0 ]; then
  echo ""ERROR: Compilation failed with error code $exit_status""
@@ -198,11 +198,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
  # Extract the desired substring using sed
  lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
  #append json file where key is fqbn, sketch name, sizes -> extracted values
- echo "{\"name\": \"$lib_sketch_name\", 
+ echo "{\"name\": \"$lib_sketch_name\",
  \"sizes\": [{
- \"flash_bytes\": $flash_bytes, 
- \"flash_percentage\": $flash_percentage, 
- \"ram_bytes\": $ram_bytes, 
+ \"flash_bytes\": $flash_bytes,
+ \"flash_percentage\": $flash_percentage,
+ \"ram_bytes\": $ram_bytes,
  \"ram_percentage\": $ram_percentage
  }]
  }," >> "$sizes_file"
@@ -365,6 +365,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
  start_index=$(( $chunk_index * $chunk_size ))
  if [ "$sketchcount" -le "$start_index" ]; then
  echo "Skipping job"
+ touch ~/.build_skipped
  return 0
  fi
 
@@ -386,7 +387,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
  if [ $log_compilation ]; then
  #echo board,target and start of sketches to sizes_file json
  echo "{ \"board\": \"$fqbn\",
- \"target\": \"$target\", 
+ \"target\": \"$target\",
  \"sketches\": [" >> "$sizes_file"
  fi
 

.github/scripts/tests_build.sh

@@ -2,19 +2,20 @@
 
 USAGE="
 USAGE:
- ${0} -c <chunk_build_opts>
- Example: ${0} -c -t esp32 -i 0 -m 15
+ ${0} -c -type <test_type> <chunk_build_opts>
+ Example: ${0} -c -type validation -t esp32 -i 0 -m 15
  ${0} -s sketch_name <build_opts>
  Example: ${0} -s hello_world -t esp32
  ${0} -clean
  Remove build and test generated files
 "
 
 function clean(){
- rm -rf tests/*/build*/
  rm -rf tests/.pytest_cache
- rm -rf tests/*/__pycache__/
- rm -rf tests/*/*.xml
+ find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
+ find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
+ find tests/ -name '*.xml' -exec rm -rf "{}" \+
+ find tests/ -name 'result_*.json' -exec rm -rf "{}" \+
 }
 
 SCRIPTS_DIR="./.github/scripts"
@@ -35,6 +36,10 @@ while [ ! -z "$1" ]; do
  echo "$USAGE"
  exit 0
  ;;
+ -type )
+ shift
+ test_type=$1
+ ;;
  -clean )
  clean
  exit 0
@@ -52,12 +57,25 @@ source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
 args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
 
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+ if [ -n "$sketch" ]; then
+ tmp_sketch_path=$(find tests -name $sketch.ino)
+ test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+ echo "Sketch $sketch test type: $test_type"
+ test_folder="$PWD/tests/$test_type"
+ else
+ test_folder="$PWD/tests"
+ fi
+else
+ test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_build -eq 1 ]; then
  BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
- args+=" -p $PWD/tests"
+ args+=" -p $test_folder"
 else
  BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
- args+=" -s $PWD/tests/$sketch"
+ args+=" -s $test_folder/$sketch"
 fi
 
 ${BUILD_CMD} ${args} $*

.github/scripts/tests_run.sh

@@ -15,9 +15,9 @@ function run_test() {
  fi
 
  if [ $len -eq 1 ]; then
- # build_dir="tests/$sketchname/build"
+ # build_dir="$sketchdir/build"
  build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
- report_file="tests/$sketchname/$sketchname.xml"
+ report_file="$sketchdir/$sketchname.xml"
  fi
 
  for i in `seq 0 $(($len - 1))`
@@ -28,9 +28,9 @@ function run_test() {
  fi
 
  if [ $len -ne 1 ]; then
- # build_dir="tests/$sketchname/build$i"
+ # build_dir="$sketchdir/build$i"
  build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
- report_file="tests/$sketchname/$sketchname$i.xml"
+ report_file="$sketchdir/$sketchname$i.xml"
  fi
 
  pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
@@ -79,6 +79,10 @@ while [ ! -z "$1" ]; do
  echo "$USAGE"
  exit 0
  ;;
+ -type )
+ shift
+ test_type=$1
+ ;;
  * )
  break
  ;;
@@ -88,21 +92,39 @@ done
 
 source ${SCRIPTS_DIR}/install-arduino-ide.sh
 
+# If sketch is provided and test type is not, test type is inferred from the sketch path
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+ if [ -n "$sketch" ]; then
+ tmp_sketch_path=$(find tests -name $sketch.ino)
+ test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+ echo "Sketch $sketch test type: $test_type"
+ test_folder="$PWD/tests/$test_type"
+ else
+ test_folder="$PWD/tests"
+ fi
+else
+ test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_run -eq 0 ]; then
- run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase
+ if [ -z $sketch ]; then
+ echo "ERROR: Sketch name is required for single test run"
+ exit 1
+ fi
+ run_test $target $test_folder/$sketch/$sketch.ino $options $erase
 else
  if [ "$chunk_max" -le 0 ]; then
  echo "ERROR: Chunks count must be positive number"
- return 1
+ exit 1
  fi
 
  if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
  echo "ERROR: Chunk index must be less than chunks count"
- return 1
+ exit 1
  fi
 
  set +e
- ${COUNT_SKETCHES} $PWD/tests $target
+ ${COUNT_SKETCHES} $test_folder $target
  sketchcount=$?
  set -e
  sketches=$(cat sketches.txt)
@@ -123,7 +145,8 @@ else
  start_index=$(( $chunk_index * $chunk_size ))
  if [ "$sketchcount" -le "$start_index" ]; then
  echo "Skipping job"
- return 0
+ touch ~/.test_skipped
+ exit 0
  fi
 
  end_index=$(( $(( $chunk_index + 1 )) * $chunk_size ))

.github/workflows/hil.yml

@@ -18,11 +18,14 @@ jobs:
  gen_chunks:
  if: |
  contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+ contains(github.event.pull_request.labels.*.name, 'perf_test') ||
  (github.event_name == 'schedule' && github.repository == 'espressif/arduino-esp32')
  name: Generate Chunks matrix
  runs-on: ubuntu-latest
  outputs:
  chunks: ${{ steps.gen-chunks.outputs.chunks }}
+ test_folder: ${{ steps.gen-chunks.outputs.test_folder }}
+ test_type: ${{ steps.gen-chunks.outputs.test_type }}
  steps:
  - name: Checkout Repository
  uses: actions/checkout@v4
@@ -31,15 +34,29 @@ jobs:
  id: gen-chunks
  run: |
  set +e
- .github/scripts/sketch_utils.sh count tests
+ if [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "true" ] && \
+ [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "false" ]; then
+ test_folder="tests/validation"
+ test_type="validation"
+ elif [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "false" ] && \
+ [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "true" ]; then
+ test_folder="tests/performance"
+ test_type="performance"
+ else
+ test_folder="tests"
+ test_type="all"
+ fi
+ .github/scripts/sketch_utils.sh count $test_folder
  sketches=$?
  if [[ $sketches -ge ${{env.MAX_CHUNKS}} ]]; then
  $sketches=${{env.MAX_CHUNKS}}
  fi
  set -e
  rm sketches.txt
  CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`)
- echo "chunks=${CHUNKS}" >>$GITHUB_OUTPUT
+ echo "chunks=${CHUNKS}" >> $GITHUB_OUTPUT
+ echo "test_folder=${test_folder}" >> $GITHUB_OUTPUT
+ echo "test_type=${test_type}" >> $GITHUB_OUTPUT
 
  Build:
  needs: gen_chunks
@@ -52,17 +69,21 @@ jobs:
  steps:
  - name: Checkout Repository
  uses: actions/checkout@v4
+
  - name: Build sketches
  run: |
- bash .github/scripts/tests_build.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
+ bash .github/scripts/tests_build.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
+
  - name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
  uses: actions/upload-artifact@v4
  with:
  name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
- path: |
- ~/.arduino/tests/*/build*.tmp/*.bin
- ~/.arduino/tests/*/build*.tmp/*.json
  if-no-files-found: error
+ path: |
+ ~/.build_skipped
+ ~/.arduino/tests/**/build*.tmp/*.bin
+ ~/.arduino/tests/**/build*.tmp/*.json
+
  Test:
  needs: [gen_chunks, Build]
  name: ${{matrix.chip}}-Test#${{matrix.chunks}}
@@ -77,36 +98,49 @@ jobs:
  options: --privileged
 
  steps:
-  - name: Checkout repository
-  uses: actions/checkout@v4
+ - name: Checkout repository
+ uses: actions/checkout@v4
 
-  - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
-  uses: actions/download-artifact@v4
-  with:
-  name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
-  path: ~/.arduino/tests/
+ - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
+ path: ~/
 
-  - name: Install dependencies
-  run: |
-  pip install -U pip
-  pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
-  apt update && apt install -y -qq jq
+ - name: Install dependencies
+ run: |
+ pip install -U pip
+ pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
+ apt update && apt install -y -qq jq
 
-  - name: Run Tests
-  run: |
-  bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
+ - name: Run Tests
+ run: |
+ bash .github/scripts/tests_run.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
 
- - name: Upload test result artifacts
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: test_results-${{matrix.chip}}-${{matrix.chunks}}
- path: tests/*/*.xml
+ - name: Check if tests were skipped
+ id: check-test-skipped
+ run: |
+ if [ -f ~/.test_skipped ]; then
+ echo "skipped=true" >> $GITHUB_OUTPUT
+ else
+ echo "skipped=false" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Upload test result artifacts
+ uses: actions/upload-artifact@v4
+ if: ${{ always() && steps.check-test-skipped.outputs.skipped == 'false' }}
+ with:
+ name: test_results-${{matrix.chip}}-${{matrix.chunks}}
+ if-no-files-found: error
+ path: |
+ tests/**/*.xml
+ tests/**/result_*.json
 
  event_file:
  name: "Event File"
  if: |
  contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+ contains(github.event.pull_request.labels.*.name, 'perf_test') ||
  github.event_name == 'schedule'
  needs: Test
  runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -1,4 +1,8 @@
-exclude: ".github/.*"
+exclude: |
+ (?x)(
+ ^\.github\/|
+ ^tests\/performance\/coremark\/.*\.[ch]$
+ )
 
 default_language_version:
  # force all unspecified python hooks to run python3

tests/.gitignore

@@ -1,3 +1,4 @@
 build*/
 __pycache__/
 *.xml
+result_*.json