Upload desktop integration artifacts to GCS

.github/workflows/integration_tests.yml

@@ -91,7 +91,7 @@ jobs:
  - name: Setup python
  uses: actions/setup-python@v2
  with:
- python-version: '3.x'
+ python-version: '3.7'
 
  - name: Install SDK Desktop prerequisites
  if: matrix.target_platform == 'Desktop'
@@ -126,9 +126,13 @@ jobs:
  - name: Set CLOUDSDK_PYTHON (Windows)
  if: startsWith(matrix.os, 'windows') && !cancelled()
  run: echo "::set-env name=CLOUDSDK_PYTHON::${{env.pythonLocation}}\python.exe"
- - name: Install Cloud SDK for mobile integration tests
- if: matrix.target_platform != 'Desktop' && !cancelled()
+ - name: Install Cloud SDK
+ if: ${{ !cancelled() }}
  uses: GoogleCloudPlatform/github-actions/setup-gcloud@master 
+ - name: Upload Desktop Artifacts to GCS
+ if: matrix.target_platform == 'Desktop' && !cancelled()
+ run: |
+ python scripts/gha/gcs_uploader.py --testapp_dir ${{ github.workspace }}/testapps --key_file ${{ github.workspace }}/scripts/gha-encrypted/gcs_key_file.json
  - name: Run mobile integration tests
  if: matrix.target_platform != 'Desktop' && !cancelled()
  run: |

scripts/gha/gcs_uploader.py

@@ -0,0 +1,126 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Command line tool for submitting artifacts to Google Cloud Storage.
+
+Example usage:
+ python gcs_uploader.py --testapp_dir <dir> --key_file <path>
+
+This will find desktop integration test artifacts in the testapp directory
+and upload them to a suitable location on GCS.
+
+The optional flag --extra_artifacts can be used to supply additional filenames
+to upload. e.g. "--extra_artifacts GoogleService-Info.plist,Info.plist"
+
+"""
+
+import os
+import subprocess
+
+from absl import app
+from absl import flags
+from absl import logging
+
+from integration_testing import gcs
+
+_DEFAULT_ARTIFACTS = (
+ "integration_test",
+ "integration_test.exe",
+ "google-services.json"
+)
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+ "testapp_dir", None,
+ "This directory will be searched for artifacts to upload.")
+flags.DEFINE_string(
+ "key_file", None, "Path to key file authorizing use of the GCS bucket.")
+flags.DEFINE_list(
+ "extra_artifacts", "",
+ "Additional file names to upload. The following are always uploaded: "
+ + ",".join(_DEFAULT_ARTIFACTS))
+
+
+def main(argv):
+ if len(argv) > 1:
+ raise app.UsageError("Too many command-line arguments.")
+
+ key_file_path = _fix_path(FLAGS.key_file)
+ testapp_dir = _fix_path(FLAGS.testapp_dir)
+
+ artifacts_to_upload = set(_DEFAULT_ARTIFACTS).union(FLAGS.extra_artifacts)
+
+ logging.info(
+ "Searching for the following artifacts:\n%s",
+ "\n".join(artifacts_to_upload))
+
+ artifacts = []
+ for file_dir, _, file_names in os.walk(testapp_dir):
+ for file_name in file_names:
+ if file_name in artifacts_to_upload:
+ artifacts.append(os.path.join(file_dir, file_name))
+
+ if artifacts:
+ logging.info("Found the following artifacts:\n%s", "\n".join(artifacts))
+ else:
+ logging.info("No artifacts found to upload")
+ return
+
+ gcs.authorize_gcs(key_file_path)
+
+ gcs_prefix = gcs.relative_path_to_gs_uri(gcs.get_unique_gcs_id())
+ logging.info("Uploading to %s", gcs_prefix)
+ for artifact in artifacts:
+ dest = _local_path_to_gcs_uri(gcs_prefix, artifact, testapp_dir)
+ logging.info("Creating %s", dest)
+ subprocess.run([gcs.GSUTIL, "cp", artifact, dest], check=True)
+ logging.info("Finished uploading to %s", gcs_prefix)
+ logging.info(
+ "Use 'gsutil cp <gs_uri> <local_path>' to copy an artifact locally.\n"
+ "Use 'gsutil -m cp -r %s <local_path>' to copy everything.", gcs_prefix)
+
+
+def _local_path_to_gcs_uri(gcs_prefix, path, testapp_dir):
+ """Converts full local path to a GCS URI for gsutil.
+
+ Replaces backslashes with forward slashes, since GCS expects the latter.
+
+ Example:
+ gcs_prefix: gs://<id>/<timestamp>
+ path: /Users/username/testapps/FirebaseAuth/integration_test
+ testapp_dir: /Users/username/testapps
+
+ Return value: gs://<id>/<timestamp>/FirebaseAuth/integration_test
+
+ Args:
+ gcs_prefix (str): Directory on GCS, of the form "gs://<id>/<etc>"
+ path (str): Full local path to artifact.
+ testapp_dir (str): Prefix to strip from all paths.
+
+ Returns:
+ (str): GCS URI for this artifact.
+
+ """
+ subdirectory = os.path.relpath(path, testapp_dir).replace("\\", "/")
+ return gcs_prefix + "/" + subdirectory
+
+
+def _fix_path(path):
+ """Expands ~, normalizes slashes, and converts relative paths to absolute."""
+ return os.path.abspath(os.path.expanduser(path))
+
+
+if __name__ == "__main__":
+ app.run(main)

scripts/gha/integration_testing/gcs.py

@@ -0,0 +1,102 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A library of functionality related to the Google Cloud SDK."""
+
+import datetime
+import random
+import shutil
+import string
+import subprocess
+
+from absl import logging
+
+# Full paths to the gCloud SDK tools. On Windows, subprocess.run does not check
+# the PATH, so we need to find and supply the full paths.
+# shutil.which returns None if it doesn't find a tool.
+# Note: this kind of thing (among others) could likely be simplified by using
+# the gCloud Python API instead of the command line tools.
+GCLOUD = shutil.which("gcloud")
+GSUTIL = shutil.which("gsutil")
+
+PROJECT_ID = "games-auto-release-testing"
+
+
+# This does not include the prefix "gs://<project_id>" because the gcloud
+# command for using Firebase Test Lab requires the results dir to be a relative
+# path within the bucket, not the full gs object URI.
+def get_unique_gcs_id():
+ """Defines an id usable for a unique object on GCS.
+
+ To avoid artifacts from parallel runs overwriting each other, this creates
+ a unique id. It's prefixed by a timestamp so that the artifact directories
+ end up sorted, and to make it easier to locate the artifacts for a particular
+ run.
+
+ Returns:
+ (str) A string of the form <timestamp>_<random_chars>.
+ """
+ # We generate a unique directory to store the results by appending 4
+ # random letters to a timestamp. Timestamps are useful so that the
+ # directories for different runs get sorted based on when they were run.
+ timestamp = datetime.datetime.now().strftime("%y%m%d-%H%M%S")
+ suffix = "".join(random.choice(string.ascii_letters) for _ in range(4))
+ return "%s_%s" % (timestamp, suffix)
+
+
+def relative_path_to_gs_uri(path):
+ """Converts a relative GCS path to a GS URI understood by gsutil.
+
+ This will prepend the gs prefix and project id to the path, i.e.
+ path -> gs://<project_id>/results_dir
+
+ Returns path unmodified if it already starts with the gs prefix.
+
+ Args:
+ path (str): A relative path.
+
+ Returns:
+ (str): The full GS URI for the given relative path.
+ """
+ if path.startswith("gs://"):
+ return path
+ else:
+ return "gs://%s/%s" % (PROJECT_ID, path)
+
+
+def authorize_gcs(key_file):
+ """Activates the service account on GCS and specifies the project to use."""
+ _verify_gcloud_sdk_command_line_tools()
+ subprocess.run(
+ args=[
+ GCLOUD, "auth", "activate-service-account", "--key-file", key_file
+ ],
+ check=True)
+ # Keep using this project for subsequent gcloud commands.
+ subprocess.run(
+ args=[GCLOUD, "config", "set", "project", PROJECT_ID],
+ check=True)
+
+
+def _verify_gcloud_sdk_command_line_tools():
+ """Verifies the presence of the gCloud SDK's command line tools."""
+ logging.info("Looking for gcloud and gsutil tools...")
+ if not GCLOUD:
+ logging.error("gcloud not on path")
+ if not GSUTIL:
+ logging.error("gsutil not on path")
+ if not GCLOUD or not GSUTIL:
+ raise RuntimeError("Could not find required gCloud SDK tool(s)")
+ subprocess.run([GCLOUD, "version"], check=True)
+ subprocess.run([GSUTIL, "version"], check=True)