Replace the FTL test script with FTL GHA (#1060)

Author: sunmou99

.github/workflows/integration_tests.yml

@@ -988,9 +988,10 @@ jobs:
  timeout_minutes: 5
  max_attempts: 3
  command: scripts/gha/install_test_workflow_prereqs.sh -p Android -t true
- - id: get-device-type
+ - id: device-info
  run: |
- echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.android_device }} )"
+ echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_device_type)"
+ echo "::set-output name=device::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_ftl_device)"
  - name: Set up Node (16)
  uses: actions/setup-node@v2
  with:
@@ -1001,15 +1002,15 @@ jobs:
  distribution: 'temurin'
  java-version: '17'
  - name: Setup Firestore Emulator
- if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+ if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
  uses: nick-invision/retry@v2
  with:
  shell: bash
  timeout_minutes: 5
  max_attempts: 3
  command: npm install -g firebase-tools 
  - name: Start Firestore Emulator
- if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+ if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
  run: |
  firebase emulators:start --only firestore --project demo-example &
  - name: Setup java 8 for test_simulator.py
@@ -1018,27 +1019,31 @@ jobs:
  distribution: 'temurin'
  java-version: '8'
  - name: Run Android integration tests on Emulator locally
- timeout-minutes: 120
- if: steps.get-device-type.outputs.device_type == 'virtual'
+ timeout-minutes: 150
+ if: steps.device-info.outputs.device_type == 'virtual'
  run: |
  python scripts/gha/test_simulator.py --testapp_dir testapps \
  --test_type "${{ matrix.test_type }}" \
  --android_device "${{ matrix.android_device }}" \
  --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}" \
  --ci
- - name: Install Cloud SDK
- if: steps.get-device-type.outputs.device_type == 'real'
- uses: google-github-actions/setup-gcloud@v0
- - name: Run Android integration tests on Real Device via FTL
- timeout-minutes: 60
- if: steps.get-device-type.outputs.device_type == 'real'
+ - id: ftl_test
+ if: steps.device-info.outputs.device_type == 'real'
+ uses: FirebaseExtended/github-actions/firebase-test-lab@v1.1
+ timeout-minutes: 90
+ with:
+ credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
+ testapp_dir: testapps
+ test_type: "game-loop"
+ test_devices: ${{ steps.device-info.outputs.device }}
+ max_attempts: 3
+ validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py
+ - name: Read FTL Test Result
+ if: steps.device-info.outputs.device_type == 'real'
+ shell: bash
  run: |
- python scripts/gha/test_lab.py --testapp_dir testapps \
- --android_device "${{ matrix.android_device }}" \
- --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}" \
- --code_platform cpp \
- --key_file_encrypted scripts/gha-encrypted/gcs_key_file.json.gpg \
- --key_file_passphrase "${{ secrets.TEST_SECRET }}"
+ python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
+ --output_path testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.log
  - name: Prepare results summary artifact
  if: ${{ !cancelled() }}
  shell: bash
@@ -1055,14 +1060,14 @@ jobs:
  path: testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}*
  retention-days: ${{ env.artifactRetentionDays }}
  - name: Upload Android test video artifact
- if: ${{ steps.get-device-type.outputs.device_type == 'virtual' && !cancelled() }}
+ if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
  uses: actions/upload-artifact@v3
  with:
  name: mobile-simulator-test-video-artifact
  path: testapps/video-*-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.mp4
  retention-days: ${{ env.artifactRetentionDays }}
  - name: Upload Android test logcat artifact
- if: ${{ steps.get-device-type.outputs.device_type == 'virtual' && !cancelled() }}
+ if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
  uses: actions/upload-artifact@v3
  with:
  name: mobile-simulator-test-logcat-artifact
@@ -1130,9 +1135,10 @@ jobs:
  timeout_minutes: 3
  max_attempts: 3
  command: scripts/gha/install_test_workflow_prereqs.sh -p iOS -t true
- - id: get-device-type
+ - id: device-info
  run: |
- echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.ios_device }} )"
+ echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_device_type)"
+ echo "::set-output name=device::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_ftl_device)"
  - name: Set up Node (16)
  uses: actions/setup-node@v2
  with:
@@ -1143,40 +1149,43 @@ jobs:
  distribution: 'temurin'
  java-version: '17'
  - name: Setup Firestore Emulator
- if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+ if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
  uses: nick-invision/retry@v2
  with:
  shell: bash
  timeout_minutes: 5
  max_attempts: 3
  command: npm install -g firebase-tools 
  - name: Start Firestore Emulator
- if: steps.get-device-type.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
+ if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
  run: |
  firebase emulators:start --only firestore --project demo-example &
  - name: Run iOS integration tests on Simulator locally
- timeout-minutes: 120
- if: steps.get-device-type.outputs.device_type == 'virtual'
+ timeout-minutes: 150
+ if: steps.device-info.outputs.device_type == 'virtual'
  run: |
  python scripts/gha/test_simulator.py --testapp_dir testapps \
  --test_type "${{ matrix.test_type }}" \
  --ios_device "${{ matrix.ios_device }}" \
  --logfile_name "ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}" \
  --ci
- - name: Install Cloud SDK
- if: steps.get-device-type.outputs.device_type == 'real'
- uses: google-github-actions/setup-gcloud@v0
- - name: Run iOS integration tests on Real Device via FTL
- # max 3 retry and 10m timeout for each testapp, plus other steps
- timeout-minutes: 60
- if: steps.get-device-type.outputs.device_type == 'real'
+ - id: ftl_test
+ if: steps.device-info.outputs.device_type == 'real'
+ uses: FirebaseExtended/github-actions/firebase-test-lab@v1.1
+ timeout-minutes: 90
+ with:
+ credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
+ testapp_dir: testapps
+ test_type: "game-loop"
+ test_devices: ${{ steps.device-info.outputs.device }}
+ max_attempts: 3
+ validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py
+ - name: Read FTL Test Result
+ if: steps.device-info.outputs.device_type == 'real'
+ shell: bash
  run: |
- python scripts/gha/test_lab.py --testapp_dir testapps \
- --ios_device "${{ matrix.ios_device }}" \
- --logfile_name "ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}" \
- --code_platform cpp \
- --key_file_encrypted scripts/gha-encrypted/gcs_key_file.json.gpg \
- --key_file_passphrase "${{ secrets.TEST_SECRET }}"
+ python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
+ --output_path testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.log
  - name: Prepare results summary artifact
  if: ${{ !cancelled() }}
  shell: bash
@@ -1193,7 +1202,7 @@ jobs:
  path: testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}*
  retention-days: ${{ env.artifactRetentionDays }}
  - name: Upload iOS test video artifact
- if: ${{ steps.get-device-type.outputs.device_type == 'virtual' && !cancelled() }}
+ if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
  uses: actions/upload-artifact@v3
  with:
  name: mobile-simulator-test-video-artifact

scripts/gha/integration_testing/ftl_gha_validator.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# Copyright 2022 Google
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import subprocess
+import shutil
+import re
+
+GSUTIL = shutil.which("gsutil")
+
+def validate(test_summary):
+ if not test_summary:
+ logging.error("Nothing to be validate! Please provide test_summary")
+ return False
+
+ app_path = test_summary.get("testapp_path")
+ return_code = test_summary.get("return_code")
+ logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code))
+ if return_code == 0:
+ gcs_dir = test_summary.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://")
+ logging.info("gcs_dir: %s" % gcs_dir)
+ logs = _get_testapp_log_text_from_gcs(gcs_dir)
+ logging.info("Test result: %s", logs)
+ return _validate_logs(logs)
+ else:
+ logging.error("Test failed: %s", app_path)
+ return False
+
+
+def _get_testapp_log_text_from_gcs(gcs_path):
+ """Gets the testapp log text generated by game loops."""
+ try:
+ gcs_contents = _gcs_list_dir(gcs_path)
+ except subprocess.CalledProcessError as e:
+ logging.error("Unexpected error searching GCS logs:\n%s", e)
+ return None
+ # The path to the testapp log depends on the platform, device, and scenario
+ # being tested. Search for a json file with 'results' in the name to avoid
+ # hard-coding too many assumptions about the path. The testapp log should be
+ # the only json, but 'results' adds some redundancy in case this changes.
+ matching_gcs_logs = [
+ line for line in gcs_contents
+ if line.endswith(".json") and "results" in line.lower()
+ ]
+ if not matching_gcs_logs:
+ logging.error("Failed to find results log on GCS.")
+ return None
+ # This assumes testapps only have one scenario. Could change in the future.
+ if len(matching_gcs_logs) > 1:
+ logging.warning("Multiple scenario logs found.")
+ gcs_log = matching_gcs_logs[0]
+ try:
+ logging.info("Found results log: %s", gcs_log)
+ log_text = _gcs_read_file(gcs_log)
+ if not log_text:
+ logging.warning("Testapp log is empty. App may have crashed.")
+ return log_text
+ except subprocess.CalledProcessError as e:
+ logging.error("Unexpected error reading GCS log:\n%s", e)
+ return None
+
+
+def _gcs_list_dir(gcs_path):
+ """Recursively returns a list of contents for a directory on GCS."""
+ args = [GSUTIL, "ls", "-r", gcs_path]
+ logging.info("Listing GCS contents: %s", " ".join(args))
+ result = subprocess.run(args=args, capture_output=True, text=True, check=True)
+ return result.stdout.splitlines()
+
+
+def _gcs_read_file(gcs_path):
+ """Extracts the contents of a file on GCS."""
+ args = [GSUTIL, "cat", gcs_path]
+ logging.info("Reading GCS file: %s", " ".join(args))
+ result = subprocess.run(args=args, capture_output=True, text=True, check=True)
+ return result.stdout
+
+
+def _validate_logs(log_text):
+ if not log_text:
+ return False
+ 
+ # The gtest runner dumps a useful summary of tests after the tear down.
+ end_marker = "Global test environment tear-down"
+ complete = end_marker in log_text
+
+ if complete:
+ # rpartition splits a string into three components around the final
+ # occurrence of the end marker, returning a triplet (before, marker, after)
+ result_summary = log_text.rpartition(end_marker)[2].lstrip()
+ failures = re.search(r"\[ FAILED \] (?P<count>[0-9]+) test", result_summary)
+ logging.info("_validate_logs failures: %s", failures)
+ if failures:
+ return False
+ else:
+ return True
+ else:
+ return False

scripts/gha/print_matrix_configuration.py

@@ -161,17 +161,20 @@
  "tvos": ["os", "xcode_version", "tvos_device"]
 }
 
+# Check currently supported models and versions with the following commands:
+# gcloud firebase test android models list
+# gcloud firebase test ios models list
 TEST_DEVICES = {
- "android_min": {"type": "real", "model":"Nexus10", "version":"19"},
- "android_target": {"type": "real", "model":"gts4lltevzw", "version":"28"},
- "android_latest": {"type": "real", "model":"redfin", "version":"30"},
+ "android_min": {"type": "real", "device": "model=Nexus10,version=19"},
+ "android_target": {"type": "real", "device": "model=gts4lltevzw,version=28"},
+ "android_latest": {"type": "real", "device": "model=oriole,version=33"},
  "emulator_min": {"type": "virtual", "image":"system-images;android-18;google_apis;x86"},
  "emulator_target": {"type": "virtual", "image":"system-images;android-30;google_apis;x86_64"},
  "emulator_latest": {"type": "virtual", "image":"system-images;android-32;google_apis;x86_64"},
  "emulator_32bit": {"type": "virtual", "image":"system-images;android-30;google_apis;x86"},
- "ios_min": {"type": "real", "model":"iphone8", "version":"12.4"},
- "ios_target": {"type": "real", "model":"iphone8", "version":"13.6"},
- "ios_latest": {"type": "real", "model":"iphone11pro", "version":"14.7"},
+ "ios_min": {"type": "real", "device": "model=iphonexr,version=13.2"},
+ "ios_target": {"type": "real", "device": "model=iphone8,version=13.6"},
+ "ios_latest": {"type": "real", "device": "model=iphone11pro,version=14.7"},
  "simulator_min": {"type": "virtual", "name":"iPhone 8", "version":"13.7"},
  "simulator_target": {"type": "virtual", "name":"iPhone 8", "version":"14.5"},
  "simulator_latest": {"type": "virtual", "name":"iPhone 11", "version":"15.2"},
@@ -356,9 +359,12 @@ def main():
  print_value(args.override)
  return
 
- if args.device:
+ if args.get_device_type:
  print(TEST_DEVICES.get(args.parm_key).get("type"))
  return 
+ if args.get_ftl_device:
+ print(TEST_DEVICES.get(args.parm_key).get("device"))
+ return 
 
  if args.expanded:
  test_matrix = EXPANDED_KEY
@@ -383,7 +389,8 @@ def parse_cmdline_args():
  parser.add_argument('-k', '--parm_key', required=True, help='Print the value of specified key from matrix or config maps.')
  parser.add_argument('-a', '--auto_diff', metavar='BRANCH', help='Compare with specified base branch to automatically set matrix options')
  parser.add_argument('-o', '--override', help='Override existing value with provided value')
- parser.add_argument('-d', '--device', action='store_true', help='Get the device type, used with -k $device')
+ parser.add_argument('-get_device_type', action='store_true', help='Get the device type, used with -k $device')
+ parser.add_argument('-get_ftl_device', action='store_true', help='Get the ftl test device, used with -k $device')
  parser.add_argument('-t', '--device_type', default=['real', 'virtual'], help='Test on which type of mobile devices')
  parser.add_argument('--apis', default=PARAMETERS["integration_tests"]["config"]["apis"], 
  help='Exclude platform based on apis. Certain platform does not support all apis. e.g. tvOS does not support messaging')