ci(tools): Fix tools workflows (espressif#9846)

* ci(tools): Remove ARM64 runner and use get.exe * ci(tools): Optimize get.py and verify extraction * change(tools): Push generated binaries to PR --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: lucasssvaz

.github/scripts/install-arduino-core-esp32.sh

@@ -28,7 +28,11 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
  #git submodule update --init --recursive > /dev/null 2>&1
 
  echo "Installing Platform Tools ..."
- cd tools && python get.py
+ if [ "$OS_IS_WINDOWS" == "1" ]; then
+ cd tools && ./get.exe
+ else
+ cd tools && python get.py
+ fi
  cd $script_init_path
 
  echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'"

.github/workflows/build_py_tools.yml

@@ -3,10 +3,11 @@ name: Build Python Tools
 on:
  pull_request:
  paths:
- - 'tools/get.py'
- - 'tools/espota.py'
- - 'tools/gen_esp32part.py'
- - 'tools/gen_insights_package.py'
+ - '.github/workflows/build_py_tools.yml'
+ - 'tools/get.py'
+ - 'tools/espota.py'
+ - 'tools/gen_esp32part.py'
+ - 'tools/gen_insights_package.py'
 
 jobs:
  find-changed-tools:
@@ -21,6 +22,13 @@ jobs:
  with:
  fetch-depth: 2
  ref: ${{ github.event.pull_request.head.ref }}
+
+ - name: Check if checkout failed
+ if: failure()
+ run: |
+ echo "Checkout failed."
+ echo "Make sure you are using a branch inside the repository and not a fork."
+
  - name: Verify Python Tools Changed
  uses: tj-actions/changed-files@v41
  id: verify-changed-files
@@ -47,7 +55,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- os: [windows-latest, macos-latest, ubuntu-20.04, ARM, ARM64]
+ os: [windows-latest, macos-latest, ubuntu-20.04, ARM]
  include:
  - os: windows-latest
  TARGET: win64
@@ -63,10 +71,6 @@ jobs:
  CONTAINER: python:3.8-bullseye
  TARGET: arm
  SEPARATOR: ':'
- - os: ARM64
- CONTAINER: python:3.8-bullseye
- TARGET: arm64
- SEPARATOR: ':'
  container: ${{ matrix.CONTAINER }} # use python container on ARM
  env:
  DISTPATH: pytools-${{ matrix.TARGET }}
@@ -93,7 +97,7 @@ jobs:
  ref: ${{ github.event.pull_request.head.ref }}
  - name: Set up Python 3.8
  # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
- if: matrix.os != 'ARM' && matrix.os != 'ARM64'
+ if: matrix.os != 'ARM'
  uses: actions/setup-python@master
  with:
  python-version: 3.8
@@ -108,7 +112,7 @@ jobs:
  pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=.github/pytools/espressif.ico tools/$tool.py
  done
  - name: Sign binaries
- if: matrix.os == 'windows-latest' && env.CERTIFICATE != '' && env.CERTIFICATE_PASSWORD != ''
+ if: matrix.os == 'windows-latest'
  env:
  CERTIFICATE: ${{ secrets.CERTIFICATE }}
  CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}

tools/get.exe

@@ -101,52 +101,46 @@ def verify_files(filename, destination, rename_to):
  t1 = time.time()
  if filename.endswith(".zip"):
  try:
- with zipfile.ZipFile(filename, "r") as archive:
- first_dir = archive.namelist()[0].split("/")[0]
- total_files = len(archive.namelist())
- for i, zipped_file in enumerate(archive.namelist(), 1):
- local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
- if not os.path.exists(local_path):
- print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
- print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
- return False
- print_verification_progress(total_files, i, t1)
+ archive = zipfile.ZipFile(filename, "r")
+ file_list = archive.namelist()
  except zipfile.BadZipFile:
- print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+ if verbose:
+ print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
  return False
  elif filename.endswith(".tar.gz"):
  try:
- with tarfile.open(filename, "r:gz") as archive:
- first_dir = archive.getnames()[0].split("/")[0]
- total_files = len(archive.getnames())
- for i, zipped_file in enumerate(archive.getnames(), 1):
- local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
- if not os.path.exists(local_path):
- print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
- print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
- return False
- print_verification_progress(total_files, i, t1)
+ archive = tarfile.open(filename, "r:gz")
+ file_list = archive.getnames()
  except tarfile.ReadError:
- print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+ if verbose:
+ print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
  return False
  elif filename.endswith(".tar.xz"):
  try:
- with tarfile.open(filename, "r:xz") as archive:
- first_dir = archive.getnames()[0].split("/")[0]
- total_files = len(archive.getnames())
- for i, zipped_file in enumerate(archive.getnames(), 1):
- local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
- if not os.path.exists(local_path):
- print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
- print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
- return False
- print_verification_progress(total_files, i, t1)
+ archive = tarfile.open(filename, "r:xz")
+ file_list = archive.getnames()
  except tarfile.ReadError:
- print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+ if verbose:
+ print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
  return False
  else:
  raise NotImplementedError("Unsupported archive type")
 
+ try:
+ first_dir = file_list[0].split("/")[0]
+ total_files = len(file_list)
+ for i, zipped_file in enumerate(file_list, 1):
+ local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
+ if not os.path.exists(local_path):
+ if verbose:
+ print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
+ print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+ return False
+ print_verification_progress(total_files, i, t1)
+ except Exception as e:
+ print(f"\nError: {e}")
+ return False
+
  if verbose:
  print(f"\nVerification passed; completed in {format_time(time.time() - t1)}")
 
@@ -231,7 +225,12 @@ def unpack(filename, destination, force_extract): # noqa: C901
  shutil.rmtree(rename_to)
  shutil.move(dirname, rename_to)
 
- return True
+ if verify_files(filename, destination, rename_to):
+ print(" Files extracted successfully.")
+ return True
+ else:
+ print(" Failed to extract files.")
+ return False
 
 
 def download_file_with_progress(url, filename, start_time):
@@ -291,6 +290,7 @@ def get_tool(tool, force_download, force_extract):
  local_path = dist_dir + archive_name
  url = tool["url"]
  start_time = time.time()
+ print("")
  if not os.path.isfile(local_path) or force_download:
  if verbose:
  print("Downloading '" + archive_name + "' to '" + local_path + "'")
@@ -421,6 +421,9 @@ def identify_platform():
  current_dir + "/../package/package_esp32_index.template.json", identified_platform
  )
  mkdir_p(dist_dir)
+
+ print("\nDownloading and extracting tools...")
+
  for tool in tools_to_download:
  if is_test:
  print("Would install: {0}".format(tool["archiveFileName"]))
@@ -432,4 +435,4 @@ def identify_platform():
  print(f"Tool {tool['archiveFileName']} was corrupted, but re-downloading did not help!\n")
  sys.exit(1)
 
- print("Platform Tools Installed")
+ print("\nPlatform Tools Installed")