Create build matrices & test matrices for integration test workflow

scripts/gha/build_testapps.py

@@ -106,8 +106,8 @@
 _SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _DESKTOP)
 
 # Values for iOS SDK flag (where the iOS app will run)
-_IOS_SDK_DEVICE = "device"
-_IOS_SDK_SIMULATOR = "simulator"
+_IOS_SDK_DEVICE = "real"
+_IOS_SDK_SIMULATOR = "virtual"
 _SUPPORTED_IOS_SDK = (_IOS_SDK_DEVICE, _IOS_SDK_SIMULATOR)
 
 FLAGS = flags.FLAGS
@@ -120,6 +120,12 @@
  "output_directory", "~",
  "Build output will be placed in this directory.")
 
+flags.DEFINE_string(
+ "artifact_name", "",
+ "artifacts will be created and placed in output_directory."
+ " testapps artifact is testapps-$artifact_name.zip;"
+ " build log artifact is build-results-$artifact_name.log.") 
+
 flags.DEFINE_string(
  "repo_dir", os.getcwd(),
  "Firebase C++ SDK Git repository. Current directory by default.")
@@ -141,8 +147,8 @@
 
 flags.DEFINE_list(
  "ios_sdk", _IOS_SDK_DEVICE, 
- "(iOS only) Build for device (ipa), simulator (app), or both."
- " Building for both will produce both an .app and an .ipa.")
+ "(iOS only) Build for real device (.ipa), virtual device / simulator (.app), "
+ "or both. Building for both will produce both an .app and an .ipa.")
 
 flags.DEFINE_bool(
  "update_pod_repo", True,
@@ -186,7 +192,7 @@ def main(argv):
  testapps = FLAGS.testapps
 
  sdk_dir = _fix_path(FLAGS.packaged_sdk or FLAGS.repo_dir)
- output_dir = _fix_path(FLAGS.output_directory)
+ root_output_dir = _fix_path(FLAGS.output_directory)
  repo_dir = _fix_path(FLAGS.repo_dir)
 
  update_pod_repo = FLAGS.update_pod_repo
@@ -196,9 +202,9 @@ def main(argv):
  timestamp = ""
 
  if FLAGS.short_output_paths:
- output_dir = os.path.join(output_dir, "ta")
+ output_dir = os.path.join(root_output_dir, "ta")
  else:
- output_dir = os.path.join(output_dir, "testapps" + timestamp)
+ output_dir = os.path.join(root_output_dir, "testapps" + timestamp)
 
  ios_framework_dir = os.path.join(sdk_dir, "xcframeworks")
  ios_framework_exist = os.path.isdir(ios_framework_dir)
@@ -248,8 +254,10 @@ def main(argv):
  cmake_flags=cmake_flags,
  short_output_paths=FLAGS.short_output_paths)
  logging.info("END building for %s", testapp)
+
+ _zip_integration_tests(testapps, root_output_dir, FLAGS.artifact_name)
 
- _summarize_results(testapps, platforms, failures, output_dir)
+ _summarize_results(testapps, platforms, failures, root_output_dir, FLAGS.artifact_name)
  return 1 if failures else 0
 
 
@@ -321,8 +329,45 @@ def _build(
  return failures
 
 
-def _summarize_results(testapps, platforms, failures, output_dir):
+def _zip_integration_tests(testapps, output_dir, artifact_name):
+ testapps_artifact_dir = "testapps-" + artifact_name
+ android_testapp_extension = ".apk"
+ ios_testapp_extension = ".ipa"
+ ios_simualtor_testapp_extension = ".app"
+ desktop_testapp_name = "integration_test" 
+ if platform.system() == "Windows":
+ desktop_testapp_name += ".exe"
+
+ testapp_paths = []
+ for file_dir, directories, file_names in os.walk(output_dir):
+ for directory in directories:
+ if directory.endswith(ios_simualtor_testapp_extension):
+ testapp_paths.append(os.path.join(file_dir, directory))
+ for file_name in file_names:
+ if ((file_name == desktop_testapp_name and "ios_build" not in file_dir) 
+ or file_name.endswith(android_testapp_extension) 
+ or file_name.endswith(ios_testapp_extension)):
+ testapp_paths.append(os.path.join(file_dir, file_name))
+
+ artifact_path = os.path.join(output_dir, testapps_artifact_dir)
+ for testapp in testapps:
+ os.makedirs(os.path.join(artifact_path, testapp))
+ for path in testapp_paths:
+ for testapp in testapps:
+ if testapp in path:
+ if os.path.isfile(path):
+ shutil.copy(path, os.path.join(artifact_path, testapp))
+ else:
+ dir_util.copy_tree(path, os.path.join(artifact_path, testapp, "integration_test.app"))
+ break
+ shutil.make_archive(artifact_path, 'zip', root_dir=output_dir, base_dir=testapps_artifact_dir)
+ logging.info("integration tests artifact: %s.zip", artifact_path)
+
+
+def _summarize_results(testapps, platforms, failures, output_dir, artifact_name):
  """Logs a readable summary of the results of the build."""
+ file_name = "build-results-" + artifact_name + ".log"
+
  summary = []
  summary.append("BUILD SUMMARY:")
  summary.append("TRIED TO BUILD: " + ",".join(testapps))
@@ -337,7 +382,7 @@ def _summarize_results(testapps, platforms, failures, output_dir):
  summary = "\n".join(summary)
 
  logging.info(summary)
- test_validation.write_summary(output_dir, summary)
+ test_validation.write_summary(output_dir, summary, file_name=file_name)
 
 
 def _build_desktop(sdk_dir, cmake_flags):

scripts/gha/desktop_tester.py

@@ -40,6 +40,10 @@
 
 flags.DEFINE_string("testapp_dir", None, "Look for testapps in this directory.")
 flags.DEFINE_string("testapp_name", "integration_test", "Name of the testapps.")
+flags.DEFINE_string(
+ "artifact_name", "",
+ "Create test log artifact is test-results-$artifact_name.log."
+ " artifacts will be created and placed in testapp_dir.") 
 
 
 def main(argv):
@@ -55,7 +59,7 @@ def main(argv):
  testapps = []
  for file_dir, _, file_names in os.walk(testapp_dir):
  for file_name in file_names:
- if file_name == testapp_name:
+ if file_name == testapp_name and "ios_build" not in file_dir:
  testapps.append(os.path.join(file_dir, file_name))
  if not testapps:
  logging.error("No testapps found.")
@@ -75,7 +79,10 @@ def main(argv):
  thread.join()
 
  return test_validation.summarize_test_results(
- tests, test_validation.CPP, testapp_dir)
+ tests, 
+ test_validation.CPP, 
+ testapp_dir, 
+ file_name="test-results-" + FLAGS.artifact_name + ".log")
 
 
 def _fix_path(path):

scripts/gha/integration_testing/gameloop/gameloop.xcodeproj/project.pbxproj

@@ -294,7 +294,7 @@
 GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 GCC_WARN_UNUSED_FUNCTION = YES;
 GCC_WARN_UNUSED_VARIABLE = YES;
-IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 MTL_FAST_MATH = YES;
 ONLY_ACTIVE_ARCH = YES;
@@ -349,7 +349,7 @@
 GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 GCC_WARN_UNUSED_FUNCTION = YES;
 GCC_WARN_UNUSED_VARIABLE = YES;
-IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 MTL_ENABLE_DEBUG_INFO = NO;
 MTL_FAST_MATH = YES;
 SDKROOT = iphoneos;
@@ -366,7 +366,7 @@
 ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 CODE_SIGN_STYLE = Automatic;
 INFOPLIST_FILE = gameloop/Info.plist;
-IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 LD_RUNPATH_SEARCH_PATHS = (
 "$(inherited)",
 "@executable_path/Frameworks",
@@ -385,7 +385,7 @@
 ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 CODE_SIGN_STYLE = Automatic;
 INFOPLIST_FILE = gameloop/Info.plist;
-IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 LD_RUNPATH_SEARCH_PATHS = (
 "$(inherited)",
 "@executable_path/Frameworks",

scripts/gha/integration_testing/gameloop_android/app/src/main/java/com/google/firebase/gameloop/MainActivity.kt

@@ -31,7 +31,7 @@ class MainActivity : AppCompatActivity(R.layout.activity_main) {
  val pkgAppsList: List<ResolveInfo> = packageManager.queryIntentActivities(intent, 0)
  val gamePackageName = pkgAppsList[0].activityInfo.packageName
 
- val dir = File(getExternalFilesDir(null), gamePackageName)
+ val dir = File(getFilesDir(), gamePackageName)
  if (!dir.exists()) dir.mkdirs()
  val filename = "Results1.json"
  val file = File(dir, filename)

scripts/gha/integration_testing/gameloop_android/app/src/main/res/xml/file_paths.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <paths>
- <external-path
+ <files-path
  name="files"
  path="/"/>
 </paths>

scripts/gha/integration_testing/test_validation.py

@@ -124,7 +124,7 @@ def validate_results_cpp(log_text):
  summary=result_summary)
 
 
-def summarize_test_results(tests, platform, summary_dir, extra_info=""):
+def summarize_test_results(tests, platform, summary_dir, file_name="summary.log", extra_info=""):
  """Summarizes and logs test results for multiple tests.
 
  Each 'test' should be an object with properties "testapp_path", which
@@ -199,12 +199,12 @@ def summarize_test_results(tests, platform, summary_dir, extra_info=""):
  % (len(tests), len(successes), len(failures), len(errors)))
  summary = "\n".join(summary)
  logging.info(summary)
- write_summary(summary_dir, summary)
+ write_summary(summary_dir, summary, file_name)
 
  return 0 if len(tests) == len(successes) else 1
 
 
-def write_summary(testapp_dir, summary):
+def write_summary(testapp_dir, summary, file_name="summary.log"):
  """Writes a summary of tests/builds to a file in the testapp directory.
 
  This will append the given summary to a file in the testapp directory,
@@ -217,7 +217,7 @@ def write_summary(testapp_dir, summary):
 
  """
  # This method serves as the source of truth on where to put the summary.
- with open(os.path.join(testapp_dir, "summary.log"), "a") as f:
+ with open(os.path.join(testapp_dir, file_name), "a") as f:
  # The timestamp mainly helps when running locally: if running multiple
  # tests on the same directory, the results will accumulate, with a timestamp
  # to help keep track of when a given test was run.

scripts/gha/integration_testing/xcodebuild.py

@@ -45,7 +45,7 @@ def get_args_for_build(
  scheme (str): Name of the scheme to build.
  output_dir (str): Directory for the resulting build artifacts. Will be
  created if it doesn't already exist.
- ios_sdk (str): Where this build will be run: device or simulator.
+ ios_sdk (str): Where this build will be run: real device or virtual device (simulator).
  configuration (str): Value for the -configuration flag.
 
  Returns:
@@ -77,16 +77,16 @@ def get_args_for_build(
 
 def _get_ios_env_from_target(ios_sdk):
  """Return a value for the -sdk flag based on the target (device/simulator)."""
- if ios_sdk == "device":
+ if ios_sdk == "real":
  return "iphoneos"
- elif ios_sdk == "simulator":
+ elif ios_sdk == "virtual":
  return "iphonesimulator"
  else:
  raise ValueError("Unrecognized ios_sdk: %s" % ios_sdk)
 
 
 def generate_unsigned_ipa(output_dir, configuration):
- """create unsigned .ipa from .app
+ """create unsigned .ipa from .app, then remove .app afterwards
 
  Args:
  output_dir (str): Same value as get_args_for_build. generated unsigned .ipa 
@@ -101,3 +101,4 @@ def generate_unsigned_ipa(output_dir, configuration):
  shutil.move(app_path, payload_path)
  shutil.make_archive(payload_path, 'zip', root_dir=iphone_build_dir, base_dir='Payload')
  shutil.move('%s.%s'%(payload_path, 'zip'), ipa_path)
+ shutil.rmtree(payload_path)

scripts/gha/print_matrix_configuration.py

@@ -99,15 +99,17 @@
  "matrix": {
  "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
  "platform": ["Desktop", "Android", "iOS"],
- "ssl_lib": ["openssl", "boringssl"]
+ "ssl_lib": ["openssl", "boringssl"],
+ "android_device": ["virtual:system-images;android-28;google_apis;x86_64+28.0.3", "real:Pixel2+28"],
+ "ios_device": ["virtual:iPhone 8+12.0", "real:iphone6s+12.0"],
+ EXPANDED_KEY: {
+ "android_device": ["virtual:system-images;android-18;default;x86+28.0.3", "virtual:system-images;android-28;google_apis;x86_64+28.0.3", "virtual:system-images;android-29;default;x86+28.0.3", "real:Nexus10+19", "real:Pixel2+28", "real:flame+29"],
+ "ios_device": ["virtual:iPhone 6+11.4", "virtual:iPhone 8+12.0", "virtual:iPhone 11+14.4", "real:iphone8+11.4", "real:iphone6s+12.0", "real:iphone11pro+14.1"]
+ }
  },
  "config": {
  "apis": "admob,analytics,auth,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage",
- "mobile_test_on": "device,simulator",
- "android_device": "flame",
- "android_api": "29",
- "ios_device": "iphone8",
- "ios_version": "12.4"
+ "mobile_test_on": "real,virtual"
  }
  },
 
@@ -172,6 +174,18 @@ def get_value(workflow, use_expanded, parm_key, config_parms_only=False):
  use_expanded))
 
 
+def filter_value(workflow, parm_key, value, by):
+ """ Filter value by condtional
+
+ Currently only used for filter mobile device by device type.
+ """
+ if workflow == "integration_tests" and (parm_key == "android_device" or parm_key == "ios_device"):
+ filtered_value = filter(lambda device: device.split(":")[0] in by, value)
+ value = list(filtered_value)
+
+ return value 
+
+
 def print_value(value):
  """ Print Json formatted string that can be consumed in Github workflow."""
  # Eg: for lists,
@@ -276,6 +290,7 @@ def main():
  return
 
  value = get_value(args.workflow, args.expanded, args.parm_key, args.config)
+ value = filter_value(args.workflow, args.parm_key, value, by = args.device)
  if args.auto_diff:
  value = filter_values_on_diff(args.parm_key, value, args.auto_diff)
  print_value(value)
@@ -289,6 +304,7 @@ def parse_cmdline_args():
  parser.add_argument('-k', '--parm_key', required=True, help='Print the value of specified key from matrix or config maps.')
  parser.add_argument('-a', '--auto_diff', metavar='BRANCH', help='Compare with specified base branch to automatically set matrix options')
  parser.add_argument('-o', '--override', help='Override existing value with provided value')
+ parser.add_argument('-d', '--device', default=['real', 'virtual'], help='Test on which type of mobile devices')
  args = parser.parse_args()
  return args
 

scripts/gha/restore_secrets.py

@@ -49,6 +49,7 @@
 flags.DEFINE_string("repo_dir", os.getcwd(), "Path to C++ SDK Github repo.")
 flags.DEFINE_string("passphrase", None, "The passphrase itself.")
 flags.DEFINE_string("passphrase_file", None, "Path to file with passphrase.")
+flags.DEFINE_string("artifact", None, "Artifact Path, google-services.json will be placed here.")
 
 
 def main(argv):
@@ -78,6 +79,13 @@ def main(argv):
  api = os.path.basename(os.path.dirname(path))
  file_name = os.path.basename(path).replace(".gpg", "")
  dest_paths = [os.path.join(repo_dir, api, "integration_test", file_name)]
+ if FLAGS.artifact:
+ # /<repo_dir>/<artifact>/auth/google-services.json
+ if "google-services" in path and os.path.isdir(os.path.join(repo_dir, FLAGS.artifact, api)):
+ dest_paths = [os.path.join(repo_dir, FLAGS.artifact, api, file_name)]
+ else:
+ continue
+
  # Some apis like Firestore also have internal integration tests.
  if os.path.exists( os.path.join(repo_dir, api, "integration_test_internal")):
  dest_paths.append(os.path.join(repo_dir, api,
@@ -93,6 +101,9 @@ def main(argv):
  _patch_reverse_id(dest_path)
  _patch_bundle_id(dest_path)
 
+ if FLAGS.artifact:
+ return
+
  print("Attempting to patch Dynamic Links uri prefix.")
  uri_path = os.path.join(secrets_dir, "dynamic_links", "uri_prefix.txt.gpg")
  uri_prefix = _decrypt(uri_path, passphrase)