Add workflow to automatically update iOS and Android dependencies. (#629)

This workflow will: - Check out the repo and create a branch - Update the iOS and/or Android dependencies in that branch - Push the branch to GitHub - Create a PR on the branch - Trigger integration tests to run on the PR (via label) For now, to use, go to "Actions", select the "Update Android and iOS dependencies" workflow, and use workflow_dispatch to run the workflow. In most cases, the default inputs should be fine.

Author: jonsimantov

.github/workflows/lint.yml

@@ -22,6 +22,10 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: 3.7
+ - name: Install prerequisites
+ run: |
+ cd firebase
+ python scripts/gha/install_prereqs_desktop.py
  - name: Install prerequisites
  run: |
  python3 -m pip install unidiff

.github/workflows/update-dependencies.yml

@@ -1,13 +1,137 @@
-name: Update Android and iOS dependencies (placeholder)
+name: Update Android and iOS dependencies
 on:
  workflow_dispatch:
  inputs:
+ updateAndroid:
+ description: 'update Android dependencies?'
+ default: 1
+ updateiOS:
+ description: 'update iOS dependencies?'
+ default: 1
+ triggerTests:
+ description: 'trigger tests on PR?'
+ default: 1
+ baseBranch:
+ description: 'create the new branch from this base'
+ default: 'main'
+
+env:
+ branchPrefix: "workflow/auto-update-deps-"
+ triggerTestsLabel: "tests-requested: quick"
 
 jobs:
  update_dependencies:
- name: update-dependencies
+ name: update-deps
  runs-on: ubuntu-latest
  steps:
- - name: Placeholder step
+ - name: Get token for firebase-workflow-trigger
+ uses: tibdex/github-app-token@v1
+ id: generate-token
+ with:
+ app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
+ private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
+
+ - name: Setup python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.7
+
+ - name: Check out base branch
+ uses: actions/checkout@v2.3.1
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.baseBranch }}
+
+ - name: Install prerequisites
+ run: |
+ python scripts/gha/install_prereqs_desktop.py
+ python -m pip install requests
+
+ - name: Name new branch
+ run: |
+ date_str=$(date "+%Y%m%d-%H%M%S")
+ echo "NEW_BRANCH=${{env.branchPrefix}}${{github.run_number}}-${date_str}" >> $GITHUB_ENV
+
+ - name: Create new branch
  run: |
- true
+ git remote update
+ git checkout -b "${NEW_BRANCH}"
+ echo "UPDATE_LOGFILE=update_log.txt" >> $GITHUB_ENV
+
+ - name: Run update script
+ run: |
+ if [[ ${{ github.event.inputs.updateiOS }} -eq 1 ]]; then
+ if [[ ${{ github.event.inputs.updateAndroid }} -eq 1 ]]; then
+ # Update both
+ echo "Updating all dependencies"
+ python scripts/update_android_ios_dependencies.py --logfile=${UPDATE_LOGFILE}
+ echo "CHOSEN_DEPS=mobile" >> $GITHUB_ENV
+ else
+ # Update iOS only
+ echo "Updating iOS dependencies only"
+ python scripts/update_android_ios_dependencies.py --skip_android --logfile=${UPDATE_LOGFILE}
+ echo "CHOSEN_DEPS=iOS" >> $GITHUB_ENV
+ fi
+ # iOS: Update Firestore external version to match Firestore Cocoapod version.
+ firestore_version=$(grep "pod 'Firebase/Firestore'" ios_pod/Podfile | sed "s/.*'\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)'.*/\1/")
+ sed -i~ "s/^set(version [^)]*)/set(version CocoaPods-${firestore_version})/i" cmake/external/firestore.cmake
+ elif [[ ${{ github.event.inputs.updateAndroid }} -eq 1 ]]; then
+ # Update Android only
+ echo "Updating Android dependencies only"
+ python scripts/update_android_ios_dependencies.py --skip_ios --logfile=${UPDATE_LOGFILE}
+ echo "CHOSEN_DEPS=Android" >> $GITHUB_ENV
+ else
+ echo "::error ::Neither Android nor iOS selected. Exiting."
+ exit 1
+ fi
+
+ - name: Push branch if there are changes
+ id: push-branch
+ run: |
+ if ! git update-index --refresh; then
+ # Do a bit of post-processing on the update log to split it by platform.
+ UPDATE_LOGFILE_PROCESSED=update_log_processed.txt
+ if grep -q ^Android: "${UPDATE_LOGFILE}"; then
+ echo "### Android" >> "${UPDATE_LOGFILE_PROCESSED}"
+ echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+ sed 's/^Android: /- /' ${UPDATE_LOGFILE} >> ${UPDATE_LOGFILE_PROCESSED}
+ echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+ fi
+ if grep -q ^iOS: "${UPDATE_LOGFILE}"; then
+ echo "### iOS" >> "${UPDATE_LOGFILE_PROCESSED}"
+ echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+ sed 's/^iOS: /- /' ${UPDATE_LOGFILE} >> ${UPDATE_LOGFILE_PROCESSED}
+ echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+ fi
+
+ date_str=$(date "+%a %b %d %Y")
+ commit_title="Update ${CHOSEN_DEPS} dependencies - ${date_str}"
+ commit_body="$(cat ${UPDATE_LOGFILE_PROCESSED})
+
+ > Created by [${{github.workflow}} workflow]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+ git config user.email "firebase-workflow-trigger-bot@google.com"
+ git config user.name "firebase-workflow-trigger-bot"
+ git config core.commentChar "%" # so we can use # in git commit messages
+ git commit -a -m "${commit_title}
+
+ ${commit_body}"
+ echo "::set-output name=branch_pushed::1"
+ # Show changes in git log
+ git diff
+ # Push branch
+ git push --set-upstream origin "${NEW_BRANCH}"
+ # Create pull request
+ pr_number=$(python scripts/gha/create_pull_request.py --token ${{ steps.generate-token.outputs.token }} --head "${NEW_BRANCH}" --base "${{ github.event.inputs.baseBranch }}" --title "${commit_title}" --body "${commit_body}")
+ echo "::set-output name=created_pr_number::${pr_number}"
+ else
+ echo "::warning ::No changes detected, won't create pull request."
+ echo "::set-output name=branch_pushed::0"
+ fi
+
+ - name: Set test trigger label.
+ uses: actions-ecosystem/action-add-labels@v1
+ if: ${{ github.event.inputs.triggerTests == 1 && steps.push-branch.outputs.branch_pushed == 1 }}
+ with:
+ github_token: ${{ steps.generate-token.outputs.token }}
+ number: ${{ steps.push-branch.outputs.created_pr_number }}
+ labels: "${{ env.triggerTestsLabel }}"

scripts/gha/create_pull_request.py

@@ -0,0 +1,75 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A utility to create pull requests.
+
+USAGE:
+ python scripts/gha/create_pull_request.py \
+ --token ${{github.token}} \
+ --head pr_branch \
+ --base main \
+ --title 'Title of the pull request' \
+ [--body 'Body text for the pull request']
+
+ Creates the pull request, and outputs the new PR number to stdout.
+"""
+
+import datetime
+import shutil
+
+from absl import app
+from absl import flags
+from absl import logging
+
+import github
+
+FLAGS = flags.FLAGS
+_DEFAULT_MESSAGE = "Creating pull request."
+
+flags.DEFINE_string(
+ "token", None,
+ "github.token: A token to authenticate on your repository.")
+
+flags.DEFINE_string(
+ "head", None,
+ "Head branch name.")
+
+flags.DEFINE_string(
+ "base", "main",
+ "Base branch name.")
+
+flags.DEFINE_string(
+ "title", None,
+ "Title for the pull request.")
+
+flags.DEFINE_string(
+ "body", "",
+ "Body text for the pull request.")
+
+def main(argv):
+ if len(argv) > 1:
+ raise app.UsageError("Too many command-line arguments.")
+ if github.create_pull_request(FLAGS.token, FLAGS.head, FLAGS.base, FLAGS.title, FLAGS.body, True):
+ # Find the most recent pull_request with the given base and head, that's ours.
+ pull_requests = github.list_pull_requests(FLAGS.token, "open", FLAGS.head, FLAGS.base)
+ print(pull_requests[0]['number'])
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ flags.mark_flag_as_required("token")
+ flags.mark_flag_as_required("head")
+ flags.mark_flag_as_required("title")
+ app.run(main)

scripts/gha/github.py

@@ -211,3 +211,39 @@ def workflow_dispatch(token, workflow_id, ref, inputs):
  with requests.post(url, headers=headers, data=json.dumps(data),
  stream=True, timeout=TIMEOUT) as response:
  logging.info("workflow_dispatch: %s response: %s", url, response)
+
+
+def create_pull_request(token, head, base, title, body, maintainer_can_modify):
+ """https://docs.github.com/en/rest/reference/pulls#create-a-pull-request"""
+ url = f'{FIREBASE_URL}/pulls'
+ headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+ data = {'head': head, 'base': base, 'title': title, 'body': body,
+ 'maintainer_can_modify': maintainer_can_modify}
+ with requests.post(url, headers=headers, data=json.dumps(data),
+ stream=True, timeout=TIMEOUT) as response:
+ logging.info("create_pull_request: %s response: %s", head, response)
+ return True if response.status_code == 201 else False
+
+def list_pull_requests(token, state, head, base):
+ """https://docs.github.com/en/rest/reference/pulls#list-pull-requests"""
+ url = f'{FIREBASE_URL}/pulls'
+ headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+ page = 1
+ per_page = 100
+ results = []
+ keep_going = True
+ while keep_going:
+ params = {'per_page': per_page, 'page': page}
+ if state: params.update({'state': state})
+ if head: params.update({'head': head})
+ if base: params.update({'base': base})
+ page = page + 1
+ keep_going = False
+ with requests_retry_session().get(url, headers=headers, params=params,
+ stream=True, timeout=TIMEOUT) as response:
+ logging.info("get_reviews: %s response: %s", url, response)
+ results = results + response.json()
+ # If exactly per_page results were retrieved, read the next page.
+ keep_going = (len(response.json()) == per_page)
+ return results
+

scripts/update_android_ios_dependencies.py

@@ -62,6 +62,9 @@
 --depfiles
 --readmefiles
 
+Log updated version numbers to a text file:
+--logfile=my_log_filename.txt
+
 These "files" flags can take a list of paths (files and directories).
 If directories are provided, they are scanned for known file types.
 """
@@ -321,6 +324,7 @@ def modify_pod_file(pod_file, pod_version_map, dryrun=True):
  dryrun (bool, optional): Just print the substitutions.
  Do not write to file. Defaults to True.
  """
+ global logfile_lines
  to_update = False
  existing_lines = []
  with open(pod_file, "r") as podfile:
@@ -345,6 +349,7 @@ def modify_pod_file(pod_file, pod_version_map, dryrun=True):
  substituted_pairs.append((line, substituted_line))
  existing_lines[idx] = substituted_line
  to_update = True
+ logfile_lines.add('iOS: %s → %s' % (pod_name, latest_version))
 
  if to_update:
  print('Updating contents of {0}'.format(pod_file))
@@ -491,6 +496,7 @@ def modify_dependency_file(dependency_filepath, version_map, dryrun=True):
  dryrun (bool, optional): Just print the substitutions.
  Do not write to file. Defaults to True.
  """
+ global logfile_lines
  logging.debug('Reading dependency file: {0}'.format(dependency_filepath))
  lines = None
  with open(dependency_filepath, "r") as dependency_file:
@@ -523,6 +529,9 @@ def replace_dependency(m):
  if substituted_line != line:
  substituted_pairs.append((line, substituted_line))
  to_update = True
+ log_match = re.search(RE_GENERIC_DEPENDENCY_MODULE, line)
+ log_pkg = log_match.group('pkg').replace('-', '_').replace(':', '.')
+ logfile_lines.add('Android: %s → %s' % (log_pkg, version_map[log_pkg]))
 
  if to_update:
  print('Updating contents of {0}'.format(dependency_filepath))
@@ -705,6 +714,7 @@ def parse_cmdline_args():
  default=('release_build_files/readme.md',),
  help= 'List of release readme markdown files or directories'
  'containing them.')
+ parser.add_argument('--logfile', help='Log to text file')
 
  args = parser.parse_args()
 
@@ -725,6 +735,7 @@ def parse_cmdline_args():
  logging.getLogger(__name__)
  return args
 
+logfile_lines = set()
 
 def main():
  args = parse_cmdline_args()
@@ -761,5 +772,9 @@ def main():
  for gradle_file in gradle_files:
  modify_gradle_file(gradle_file, latest_android_versions_map, args.dryrun)
 
+ if args.logfile:
+ with open(args.logfile, 'w') as logfile_file:
+ logfile_file.write("\n".join(sorted(list(logfile_lines))) + "\n")
+ 
 if __name__ == '__main__':
  main()