Add Firestore test history report (#1403)

* Add --firestore flag to report Firestore history. * Add Firestore report to nightly cron job. * Remove multiline commands. * Revert "Remove multiline commands." This reverts commit 6393ae9. * Fix tabbing. * Fix packaging test detection for Firestore.

.github/workflows/build-report.yml

@@ -13,8 +13,16 @@ env:
 
 jobs:
  generate-report:
+ strategy:
+ fail-fast: false
+ matrix:
+ test_set: [ general, firestore ]
  runs-on: ubuntu-20.04
+ name: generate-report-${{ matrix.test_set }}
  steps:
+ - name: Pause 5 minutes to offset Firestore fetches.
+ if: ${{ matrix.test_set == 'firestore' }}
+ run: sleep 300
  - name: Setup python
  uses: actions/setup-python@v4
  with:
@@ -36,14 +44,24 @@ jobs:
  timeout_minutes: 20
  max_attempts: 3
  shell: bash
- command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache
+ command: |
+ extra_flags=
+ if [[ "${{ matrix.test_set }}" == "firestore" ]]; then
+ extra_flags=--firestore
+ fi
+ python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache ${extra_flags}
  - name: Fetch extended GitHub jobs (with retry)
  uses: nick-invision/retry@v2
  with:
  timeout_minutes: 80
  max_attempts: 3
  shell: bash
- command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache
+ command: |
+ extra_flags=
+ if [[ "${{ matrix.test_set }}" == "firestore" ]]; then
+ extra_flags=--firestore
+ fi
+ python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache ${extra_flags}
  - name: Generate report files
  run: |
  python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md
@@ -88,4 +106,9 @@ jobs:
  timeout_minutes: 5
  max_attempts: 3
  shell: bash
- command: python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title '[C++] Nightly Integration Testing Report' --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md
+ command: |
+ issue_title='[C++] Nightly Integration Testing Report'
+ if [[ "${{ matrix.test_set }}" == "firestore" ]]; then
+ issue_title='[C++] Nightly Integration Testing Report for Firestore'
+ fi
+ python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title "${issue_title}" --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md

scripts/gha/report_build_status.py

@@ -112,6 +112,10 @@
  "summary_include_crashes", True,
  "Whether to include CRASH/TIMEOUT in the test summary.")
 
+flags.DEFINE_bool(
+ "firestore", False,
+ "Report on Firestore tests rather than on general tests.")
+
 _WORKFLOW_TESTS = 'integration_tests.yml'
 _WORKFLOW_PACKAGING = 'cpp-packaging.yml'
 _TRIGGER_USER = 'firebase-workflow-trigger[bot]'
@@ -380,11 +384,10 @@ def main(argv):
  if run['status'] != 'completed': continue
  if run['day'] < start_date or run['day'] > end_date: continue
  run['duration'] = dateutil.parser.parse(run['updated_at'], ignoretz=True) - run['date']
- if general_test_time in str(run['date']):
+ compare_test_time = firestore_test_time if FLAGS.firestore else general_test_time
+ if compare_test_time in str(run['date']):
  source_tests[day] = run
  all_days.add(day)
- # elif firestore_test_time in str(run['date']):
- # firestore_tests[day] = run
 
  workflow_id = _WORKFLOW_PACKAGING
  all_runs = firebase_github.list_workflow_runs(FLAGS.token, workflow_id, _BRANCH, 'schedule', _LIMIT)
@@ -426,9 +429,15 @@ def main(argv):
  with progress.bar.Bar('Downloading triggered workflow logs...', max=len(package_tests_all)) as bar:
  for run in package_tests_all:
  day = str(run['date'].date())
- if day in package_tests and int(package_tests[day]['id']) < int(run['id']):
- bar.next()
- continue
+ if day in package_tests:
+ # Packaging triggers two tests. For Firestore, we want the larger run ID (the second run triggered).
+ if FLAGS.firestore and int(package_tests[day]['id']) > int(run['id']):
+ bar.next()
+ continue
+ # For general tests we want the smaller run ID (the first run triggered).
+ if not FLAGS.firestore and int(package_tests[day]['id']) < int(run['id']):
+ bar.next()
+ continue
 
  packaging_run = 0