Create mobile test martix

.github/workflows/integration_tests.yml

@@ -24,18 +24,10 @@ on:
  description: 'CSV of desktop SSL variants to use'
  default: 'openssl,boringssl'
  required: true
- android_device:
- description: 'Android device model'
- default: 'flame'
- android_api:
- description: 'Android API level'
- default: '29'
- ios_device:
- description: 'iOS device model'
- default: 'iphone8'
- ios_version:
- description: 'iOS device version'
- default: '12.4'
+ mobile_test_on:
+ description: 'Mobile integration_test apps test on FTL or iOS Simulator/Android Emulator'
+ default: 'real,virtual'
+ required: true 
  use_expanded_matrix:
  description: 'Use an expanded matrix? Note: above config will be ignored.'
  default: '0'
@@ -50,7 +42,6 @@ env:
  statusLabelFailed: "tests: failed"
  statusLabelSucceeded: "tests: succeeded"
  statusCommentIdentifier: "integration-test-status-comment"
- mobileTestOn: "device"
 
 jobs:
  check_trigger:
@@ -161,10 +152,9 @@ jobs:
  matrix_os: ${{ steps.export-result.outputs.matrix_os }}
  matrix_ssl: ${{ steps.export-result.outputs.matrix_ssl }}
  apis: ${{ steps.export-result.outputs.apis }}
+ mobile_test_on: ${{ steps.export-result.outputs.mobile_test_on }}
  android_device: ${{ steps.export-result.outputs.android_device }}
- android_api: ${{ steps.export-result.outputs.android_api }}
  ios_device: ${{ steps.export-result.outputs.ios_device }}
- ios_version: ${{ steps.export-result.outputs.ios_version }}
  steps:
  - uses: actions/checkout@v2
  with:
@@ -194,10 +184,9 @@ jobs:
  else
  echo "::set-output name=matrix_ssl::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k ssl_lib -o "${{github.event.inputs.desktop_ssl_variants}}" )"
  fi
- echo "::set-output name=android_device::$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k android_device -o "${{github.event.inputs.android_device}}" )"
- echo "::set-output name=android_api::$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k android_api -o "${{github.event.inputs.android_api}}" )"
- echo "::set-output name=ios_device::$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k ios_device -o "${{github.event.inputs.ios_device}}" )"
- echo "::set-output name=ios_version::$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k ios_version -o "${{github.event.inputs.ios_version}}" )"
+ echo "::set-output name=mobile_test_on::$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k mobile_test_on -o "${{github.event.inputs.mobile_test_on}}" )"
+ echo "::set-output name=android_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k android_device -d "${{github.event.inputs.mobile_test_on}}" )"
+ echo "::set-output name=ios_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k ios_device -d "${{github.event.inputs.mobile_test_on}}" )"
 
  tests:
  name: ${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}
@@ -345,16 +334,18 @@ jobs:
  additional_flags+=(--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON)
  fi
  fi
- python scripts/gha/build_testapps.py --t ${{ needs.prepare_matrix.outputs.apis }} --p ${{ matrix.target_platform }} --output_directory "${{ github.workspace }}" --ios_sdk "${{ env.mobileTestOn }}" --noadd_timestamp ${additional_flags[*]} --short_output_paths
+ python scripts/gha/build_testapps.py --t ${{ needs.prepare_matrix.outputs.apis }} --p ${{ matrix.target_platform }} --output_directory "${{ github.workspace }}" --ios_sdk "${{ needs.prepare_matrix.outputs.mobile_test_on }}" --noadd_timestamp ${additional_flags[*]} --short_output_paths --artifact_name "testapps-${{ matrix.os }}-${{ matrix.target_platform }}"
 
+ - name: Upload mobile integration tests artifact
+ uses: actions/upload-artifact@v2.2.2
+ if: matrix.target_platform != 'Desktop' && ${{ !cancelled() }}
+ with:
+ name: testapps-${{ matrix.os }}-${{ matrix.target_platform }}.zip
+ path: testapps-${{ matrix.os }}-${{ matrix.target_platform }}.zip
  - name: Run desktop integration tests
  if: matrix.target_platform == 'Desktop' && !cancelled()
  run: |
  python scripts/gha/desktop_tester.py --testapp_dir ta
- - name: Run iOS integration tests on iOS Simulator / Android Emulator locally
- if: contains(env.mobileTestOn, 'simulator') && startsWith(matrix.os, 'macos') && matrix.target_platform != 'Desktop' && !cancelled()
- run: |
- python scripts/gha/test_simulator.py --testapp_dir ta --ci
  # Workaround for https://github.com/GoogleCloudPlatform/github-actions/issues/100
  # Must be run after the Python setup action
  - name: Set CLOUDSDK_PYTHON (Windows)
@@ -368,10 +359,6 @@ jobs:
  if: matrix.target_platform == 'Desktop' && !cancelled()
  run: |
  python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json
- - name: Run mobile integration tests on Real Device via FTL
- if: contains(env.mobileTestOn, 'device') && matrix.target_platform != 'Desktop' && !cancelled()
- run: |
- python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
  - name: Prepare results summary artifact
  if: ${{ !cancelled() }}
  run: |
@@ -437,9 +424,99 @@ jobs:
  exit 1
  fi
 
+ test_ios:
+ name: macos-latest-iOS-${{ matrix.ios_device }}
+ needs: [prepare_matrix, tests]
+ runs-on: macos-latest
+ if: contains(needs.prepare_matrix.outputs.matrix_platform, 'iOS') && needs.prepare_matrix.outputs.apis != '' && !cancelled() && !failure()
+ strategy:
+ fail-fast: false
+ matrix:
+ ios_device: ${{ fromJson(needs.prepare_matrix.outputs.ios_device) }}
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup python
+ uses: actions/setup-python@v2
+ with:
+ python-version: '3.7'
+ - name: Install python deps
+ run: |
+ pip install -r scripts/gha/requirements.txt
+ - name: download artifact
+ uses: actions/download-artifact@v2.0.8
+ with:
+ path: testapps
+ name: testapps-macos-latest-iOS.zip
+ - name: unzip artifact
+ run: |
+ unzip testapps/testapps-macos-latest-iOS.zip
+ - id: get-device
+ run: |
+ device=$(echo "${{ matrix.ios_device }}" | cut -d ":" -f2)
+ echo "::set-output name=device::$device"
+ - name: Run iOS integration tests on Simulator locally
+ if: startsWith(matrix.ios_device, 'virtual')
+ run: |
+ python scripts/gha/test_simulator.py --testapp_dir testapps-macos-latest-iOS --ios_device "${{steps.get-device.outputs.device}}" --ci --artifact
+ - name: Install Cloud SDK
+ if: startsWith(matrix.ios_device, 'real')
+ uses: google-github-actions/setup-gcloud@master
+ - name: Run iOS integration tests on Real Device via FTL
+ if: startsWith(matrix.ios_device, 'real')
+ run: |
+ python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
+ python scripts/gha/test_lab.py --testapp_dir testapps-macos-latest-iOS --ios_device "${{steps.get-device.outputs.device}}" --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
+
+ test_android:
+ name: ${{ matrix.build_os }}-Android-${{ matrix.android_device }}
+ needs: [prepare_matrix, tests]
+ runs-on: macos-latest
+ if: contains(needs.prepare_matrix.outputs.matrix_platform, 'Android') && needs.prepare_matrix.outputs.apis != '' && !cancelled() && !failure()
+ strategy:
+ fail-fast: false
+ matrix:
+ build_os: ${{ fromJson(needs.prepare_matrix.outputs.matrix_os) }}
+ android_device: ${{ fromJson(needs.prepare_matrix.outputs.android_device) }}
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup python
+ uses: actions/setup-python@v2
+ with:
+ python-version: '3.7'
+ - name: Install python deps
+ run: |
+ pip install -r scripts/gha/requirements.txt
+ - name: Install python deps
+ run: |
+ python scripts/gha/install_prereqs_desktop.py
+ - name: download artifact
+ uses: actions/download-artifact@v2.0.8
+ with:
+ path: testapps
+ name: testapps-${{ matrix.build_os }}-Android.zip
+ - name: unzip artifact
+ run: |
+ unzip testapps/testapps-${{ matrix.build_os }}-Android.zip
+ - id: get-device
+ run: |
+ device=$(echo "${{ matrix.android_device }}" | cut -d ":" -f2)
+ echo "::set-output name=device::$device"
+ - name: Run Android integration tests on Emulator locally
+ if: startsWith(matrix.android_device, 'virtual')
+ run: |
+ python scripts/gha/test_simulator.py --testapp_dir testapps-${{ matrix.build_os }}-Android --android_device "${{steps.get-device.outputs.device}}" --ci --artifact
+ - name: Install Cloud SDK
+ if: startsWith(matrix.android_device, 'real')
+ uses: google-github-actions/setup-gcloud@master
+ - name: Run Android integration tests on Real Device via FTL
+ if: startsWith(matrix.android_device, 'real')
+ run: |
+ python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
+ python scripts/gha/test_lab.py --testapp_dir testapps-${{ matrix.build_os }}-Android --android_device "${{steps.get-device.outputs.device}}" --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
+
  add_success_label:
  name: "add-success-label"
- needs: [check_trigger, tests]
+ needs: [check_trigger, tests, test_ios, test_android]
  runs-on: ubuntu-latest
  if: ${{ needs.check_trigger.outputs.should_update_labels && !cancelled() && !failure() }}
  steps:
@@ -468,7 +545,7 @@ jobs:
 
  add_failure_label:
  name: "add-failure-label"
- needs: [check_trigger, tests]
+ needs: [check_trigger, tests, test_ios, test_android]
  runs-on: ubuntu-latest
  if: ${{ needs.check_trigger.outputs.should_update_labels && !cancelled() && failure() }}
  steps:
@@ -521,7 +598,7 @@ jobs:
 
  remove_in_progress_label:
  name: "remove-in-progress-label"
- needs: [check_trigger, tests]
+ needs: [check_trigger, tests, test_ios, test_android]
  runs-on: ubuntu-latest
  if: ${{ needs.check_trigger.outputs.should_update_labels && !cancelled() }}
  steps:

scripts/gha/build_testapps.py

@@ -106,8 +106,8 @@
 _SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _DESKTOP)
 
 # Values for iOS SDK flag (where the iOS app will run)
-_IOS_SDK_DEVICE = "device"
-_IOS_SDK_SIMULATOR = "simulator"
+_IOS_SDK_DEVICE = "real"
+_IOS_SDK_SIMULATOR = "virtual"
 _SUPPORTED_IOS_SDK = (_IOS_SDK_DEVICE, _IOS_SDK_SIMULATOR)
 
 FLAGS = flags.FLAGS
@@ -120,6 +120,10 @@
  "output_directory", "~",
  "Build output will be placed in this directory.")
 
+flags.DEFINE_string(
+ "artifact_name", "~",
+ "integration_tests will be compressed with the name and placed in output_directory.") 
+
 flags.DEFINE_string(
  "repo_dir", os.getcwd(),
  "Firebase C++ SDK Git repository. Current directory by default.")
@@ -141,8 +145,8 @@
 
 flags.DEFINE_list(
  "ios_sdk", _IOS_SDK_DEVICE, 
- "(iOS only) Build for device (ipa), simulator (app), or both."
- " Building for both will produce both an .app and an .ipa.")
+ "(iOS only) Build for real device (.ipa), virtual device / simulator (.app), "
+ "or both. Building for both will produce both an .app and an .ipa.")
 
 flags.DEFINE_bool(
  "update_pod_repo", True,
@@ -186,7 +190,7 @@ def main(argv):
  testapps = FLAGS.testapps
 
  sdk_dir = _fix_path(FLAGS.packaged_sdk or FLAGS.repo_dir)
- output_dir = _fix_path(FLAGS.output_directory)
+ root_output_dir = _fix_path(FLAGS.output_directory)
  repo_dir = _fix_path(FLAGS.repo_dir)
 
  update_pod_repo = FLAGS.update_pod_repo
@@ -196,9 +200,9 @@ def main(argv):
  timestamp = ""
 
  if FLAGS.short_output_paths:
- output_dir = os.path.join(output_dir, "ta")
+ output_dir = os.path.join(root_output_dir, "ta")
  else:
- output_dir = os.path.join(output_dir, "testapps" + timestamp)
+ output_dir = os.path.join(root_output_dir, "testapps" + timestamp)
 
  ios_framework_dir = os.path.join(sdk_dir, "xcframeworks")
  ios_framework_exist = os.path.isdir(ios_framework_dir)
@@ -248,6 +252,9 @@ def main(argv):
  cmake_flags=cmake_flags,
  short_output_paths=FLAGS.short_output_paths)
  logging.info("END building for %s", testapp)
+
+ if _ANDROID in platforms or _IOS in platforms:
+ _zip_integration_tests(testapps, root_output_dir, FLAGS.artifact_name)
 
  _summarize_results(testapps, platforms, failures, output_dir)
  return 1 if failures else 0
@@ -321,6 +328,29 @@ def _build(
  return failures
 
 
+def _zip_integration_tests(testapps, output_dir, artifact_name):
+ testapp_paths = []
+ for file_dir, directories, file_names in os.walk(output_dir):
+ for directory in directories:
+ full_path = os.path.join(file_dir, directory)
+ if "simulator" in full_path and directory.endswith(".app"):
+ testapp_paths.append(full_path)
+ for file_name in file_names:
+ full_path = os.path.join(file_dir, file_name)
+ if file_name.endswith(".apk") or file_name.endswith(".ipa"):
+ testapp_paths.append(full_path)
+
+ artifact_path = os.path.join(output_dir, artifact_name)
+ for testapp in testapps:
+ os.makedirs(os.path.join(artifact_path, testapp))
+ for path in testapp_paths:
+ for testapp in testapps:
+ if testapp in path:
+ shutil.move(path, os.path.join(artifact_path, testapp))
+ break
+ shutil.make_archive(artifact_path, 'zip', root_dir=output_dir, base_dir=artifact_name)
+
+
 def _summarize_results(testapps, platforms, failures, output_dir):
  """Logs a readable summary of the results of the build."""
  summary = []

scripts/gha/integration_testing/xcodebuild.py

@@ -45,7 +45,7 @@ def get_args_for_build(
  scheme (str): Name of the scheme to build.
  output_dir (str): Directory for the resulting build artifacts. Will be
  created if it doesn't already exist.
- ios_sdk (str): Where this build will be run: device or simulator.
+ ios_sdk (str): Where this build will be run: real device or virtual device (simulator).
  configuration (str): Value for the -configuration flag.
 
  Returns:
@@ -77,9 +77,9 @@ def get_args_for_build(
 
 def _get_ios_env_from_target(ios_sdk):
  """Return a value for the -sdk flag based on the target (device/simulator)."""
- if ios_sdk == "device":
+ if ios_sdk == "real":
  return "iphoneos"
- elif ios_sdk == "simulator":
+ elif ios_sdk == "virtual":
  return "iphonesimulator"
  else:
  raise ValueError("Unrecognized ios_sdk: %s" % ios_sdk)

scripts/gha/print_matrix_configuration.py

@@ -99,15 +99,17 @@
  "matrix": {
  "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
  "platform": ["Desktop", "Android", "iOS"],
- "ssl_lib": ["openssl", "boringssl"]
+ "ssl_lib": ["openssl", "boringssl"],
+ "android_device": ["virtual:29;google_apis;x86;29.0.3", "real:flame;29"],
+ "ios_device": ["virtual:iPhone 11;14.4", "real:iphone8;12.4"],
+ EXPANDED_KEY: {
+ "android_device": ["virtual:24;google_apis;x86;24.0.3", "virtual:29;google_apis;x86;29.0.3", "real:Nexus6P;24","real:flame;29"],
+ "ios_device": ["virtual:iPhone 8;14.4", "virtual:iPhone 11;14.4", "real:iphone8;12.4", "real:iphonexr;13.2"]
+ }
  },
  "config": {
  "apis": "admob,analytics,auth,database,dynamic_links,firestore,functions,installations,instance_id,messaging,remote_config,storage",
- "mobile_test_on": "device,simulator",
- "android_device": "flame",
- "android_api": "29",
- "ios_device": "iphone8",
- "ios_version": "12.4"
+ "mobile_test_on": "real,virtual"
  }
  },
 
@@ -172,6 +174,18 @@ def get_value(workflow, use_expanded, parm_key, config_parms_only=False):
  use_expanded))
 
 
+def filter_value(workflow, parm_key, value, by):
+ """ Filter value by condtional
+
+ Currently only used for filter mobile device by device type.
+ """
+ if workflow == "integration_tests" and (parm_key == "android_device" or parm_key == "ios_device"):
+ filtered_value = filter(lambda device: device.split(":")[0] in by, value)
+ value = list(filtered_value)
+
+ return value 
+
+
 def print_value(value):
  """ Print Json formatted string that can be consumed in Github workflow."""
  # Eg: for lists,
@@ -276,6 +290,7 @@ def main():
  return
 
  value = get_value(args.workflow, args.expanded, args.parm_key, args.config)
+ value = filter_value(args.workflow, args.parm_key, value, by = args.device)
  if args.auto_diff:
  value = filter_values_on_diff(args.parm_key, value, args.auto_diff)
  print_value(value)
@@ -289,6 +304,7 @@ def parse_cmdline_args():
  parser.add_argument('-k', '--parm_key', required=True, help='Print the value of specified key from matrix or config maps.')
  parser.add_argument('-a', '--auto_diff', metavar='BRANCH', help='Compare with specified base branch to automatically set matrix options')
  parser.add_argument('-o', '--override', help='Override existing value with provided value')
+ parser.add_argument('-d', '--device', default=['real', 'virtual'], help='Test on which type of mobile devices')
  args = parser.parse_args()
  return args