Setup Desktop test workflow that detects C++ SDK breakage caused by iOS SDK changes (#1162)

Author: sunmou99

.github/workflows/integration_tests.yml

@@ -3,7 +3,8 @@ name: Integration tests
 on:
  schedule:
  - cron: "0 9 * * *" # 9am UTC = 1am PST / 2am PDT. for all testapps except firestore
- - cron: "0 10 * * *" # 10am UTC = 2am PST / 3am PDT. for firestore test only
+ - cron: "0 10 * * *" # 10am UTC = 2am PST / 3am PDT. for firestore test against release iOS/Android SDK
+ - cron: "0 11 * * *" # 11am UTC = 3am PST / 4am PDT. for firestore desktop test aginst tip-of-tree iOS repo
 
  pull_request:
  types: [ labeled, closed ]
@@ -183,17 +184,25 @@ jobs:
  # To feed input into the job matrix, we first need to convert to a JSON
  # list. Then we can use fromJson to define the field in the matrix for the tests job.
  if [[ "${{ github.event.schedule }}" == "0 9 * * *" ]]; then
- # at 1am PST / 2am PDT. Running integration tests and generate test report for all testapps except firestore
+ # at 1am PST/2am PDT. Running integration tests and generate test report for all testapps except firestore
  apis="admob,analytics,auth,database,dynamic_links,functions,gma,installations,messaging,remote_config,storage"
- elif [[ "${{ github.event.schedule }}" == "0 10 * * *" ]]; then
- # at 2am PST / 3am PDT. Running integration tests for firestore and generate test report
+ elif [[ "${{ github.event.schedule }}" == "0 10 * * *" || "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+ # at 2am PST/3am PDT and 3am PST/4am PDT. Running integration tests for firestore and generate test report.
  apis="firestore"
  else
  apis=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -o "${{github.event.inputs.apis}}" ${TEST_MATRIX_PARAM} )
  fi
+ if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+ # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
+ matrix_platform="Desktop"
+ matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "ubuntu-20.04,macos-12")
+ else
+ matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )
+ matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")
+ fi
  echo "apis=${apis}" >> $GITHUB_OUTPUT
- echo "matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )" >> $GITHUB_OUTPUT
- echo "matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")" >> $GITHUB_OUTPUT
+ echo "matrix_platform=${matrix_platform}" >> $GITHUB_OUTPUT
+ echo "matrix_os=${matrix_os}" >> $GITHUB_OUTPUT
  echo "matrix_arch_macos=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" >> $GITHUB_OUTPUT
  echo "matrix_arch_windows_linux=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" >> $GITHUB_OUTPUT
  # Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix.
@@ -318,6 +327,10 @@ jobs:
  additional_flags+=(--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON)
  fi
  fi
+ if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+ # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
+ additional_flags+=(--cmake_flag=-DFIRESTORE_DEP_SOURCE=TIP)
+ fi
  python scripts/gha/build_testapps.py --p Desktop \
  --t ${{ needs.check_and_prepare.outputs.apis }} \
  --output_directory "${{ github.workspace }}" \
@@ -327,6 +340,13 @@ jobs:
  --gha_build \
  --arch ${{ matrix.arch }} \
  ${additional_flags[*]}
+ - name: Upload Desktop Cmake
+ uses: actions/upload-artifact@v3
+ if: ${{ !cancelled() }}
+ with:
+ name: cmake-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
+ path: D:/a/firebase-cpp-sdk/firebase-cpp-sdk/ta/firestore/iti/CMakeFiles/
+ retention-days: 1
  - name: Prepare results summary artifact
  if: ${{ !cancelled() }}
  shell: bash
@@ -1361,7 +1381,12 @@ jobs:
  if [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
  additional_flags=(--build_against sdk)
  else
- additional_flags=(--build_against repo)
+ if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
+ # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
+ additional_flags=(--build_against tip)
+ else
+ additional_flags=(--build_against repo)
+ fi
  fi
  if [[ "${{ needs.check_and_prepare.outputs.apis }}" == "firestore" ]]; then
  additional_flags+=(--build_apis firestore)

scripts/gha/build_desktop.py

@@ -230,7 +230,7 @@ def cmake_configure(build_dir, arch, msvc_runtime_library='static', linux_abi='l
  if verbose:
  cmd.append('-DCMAKE_VERBOSE_MAKEFILE=1')
 
- utils.run_command(cmd)
+ utils.run_command(cmd, check=True)
 
 def main():
  args = parse_cmdline_args()
@@ -269,7 +269,7 @@ def main():
  # Example: cmake --build build -j 8 --target firebase_app firebase_auth
  cmd.append('--target')
  cmd.extend(args.target)
- utils.run_command(cmd)
+ utils.run_command(cmd, check=True)
 
 
 def parse_cmdline_args():

scripts/gha/it_workflow.py

@@ -66,17 +66,20 @@
 _COMMENT_TITLE_FLAKY = "### Integration test with FLAKINESS (succeeded after retry)\n"
 _COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
-_COMMENT_TITLE_FLAKY_SDK = "\n***\n### [build against SDK] Integration test with FLAKINESS (succeeded after retry)\n"
-_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  [build against SDK] Integration test FAILED\n"
-_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  [build against SDK] Integration test succeeded!\n"
 _COMMENT_TITLE_FLAKY_REPO = "### [build against repo] Integration test with FLAKINESS (succeeded after retry)\n"
 _COMMENT_TITLE_FAIL_REPO = "### ❌  [build against repo] Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED_REPO = "### ✅  [build against repo] Integration test succeeded!\n"
+_COMMENT_TITLE_FLAKY_SDK = "\n***\n### [build against SDK] Integration test with FLAKINESS (succeeded after retry)\n"
+_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  [build against SDK] Integration test FAILED\n"
+_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  [build against SDK] Integration test succeeded!\n"
+_COMMENT_TITLE_FLAKY_TIP = "\n***\n### [build against tip] Integration test with FLAKINESS (succeeded after retry)\n"
+_COMMENT_TITLE_FAIL_TIP = "\n***\n### ❌  [build against tip] Integration test FAILED\n"
+_COMMENT_TITLE_SUCCEED_TIP = "\n***\n### ✅  [build against tip] Integration test succeeded!\n"
 
 _COMMENT_FLAKY_TRACKER = "\n\nAdd flaky tests to **[go/fpl-cpp-flake-tracker](http://go/fpl-cpp-flake-tracker)**\n"
 
 _COMMENT_IDENTIFIER = "integration-test-status-comment"
-_COMMENT_HIDDEN_DIVIDER = f'\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\n'
+_COMMENT_HIDDEN_DIVIDER = f'\r\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\r\n'
 
 _LOG_ARTIFACT_NAME = "log-artifact"
 _LOG_OUTPUT_DIR = "test_results"
@@ -89,6 +92,7 @@
 
 _BUILD_AGAINST_SDK = "sdk"
 _BUILD_AGAINST_REPO = "repo"
+_BUILD_AGAINST_TIP = "tip"
 
 _BUILD_API_ALL = "all"
 _BUILD_API_FIRESTORE = "firestore"
@@ -222,27 +226,44 @@ def test_report(token, actor, commit, run_id, build_against, build_apis):
 
  issue_number = _get_issue_number(token, report_title, _REPORT_LABEL)
  previous_comment = github.get_issue_body(token, issue_number)
- [_, previous_comment_repo, previous_comment_sdk] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
+ [_, previous_comment_repo, previous_comment_sdk, previous_comment_tip] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
  success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
  if success_or_only_flakiness and not log_summary:
  # succeeded (without flakiness)
- title = _COMMENT_TITLE_SUCCEED_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_SUCCEED_SDK
+ if build_against==_BUILD_AGAINST_REPO:
+ title = _COMMENT_TITLE_SUCCEED_REPO 
+ elif build_against==_BUILD_AGAINST_SDK:
+ title = _COMMENT_TITLE_SUCCEED_SDK
+ else:
+ title = _COMMENT_TITLE_SUCCEED_TIP
  comment = title + _get_description(actor, commit, run_id)
  else:
  if success_or_only_flakiness:
  # all failures/errors are due to flakiness (succeeded after retry)
- title = _COMMENT_TITLE_FLAKY_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FLAKY_SDK
+ if build_against==_BUILD_AGAINST_REPO:
+ title = _COMMENT_TITLE_FLAKY_REPO 
+ elif build_against==_BUILD_AGAINST_SDK:
+ title = _COMMENT_TITLE_FLAKY_SDK
+ else:
+ title = _COMMENT_TITLE_FLAKY_TIP
  else:
  # failures/errors still exist after retry
- title = _COMMENT_TITLE_FAIL_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FAIL_SDK
+ if build_against==_BUILD_AGAINST_REPO:
+ title = _COMMENT_TITLE_FAIL_REPO 
+ elif build_against==_BUILD_AGAINST_SDK:
+ title = _COMMENT_TITLE_FAIL_SDK
+ else:
+ title = _COMMENT_TITLE_FAIL_TIP
  comment = title + _get_description(actor, commit, run_id) + log_summary + _COMMENT_FLAKY_TRACKER
 
  if build_against==_BUILD_AGAINST_REPO:
- comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk
+ comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
  elif build_against==_BUILD_AGAINST_SDK:
- comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + comment
+ comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
+ else:
+ comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + comment
 
- if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment):
+ if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment) and (build_apis != _BUILD_API_FIRESTORE or _COMMENT_TITLE_SUCCEED_TIP in comment):
  github.close_issue(token, issue_number)
  else:
  github.open_issue(token, issue_number)