test: upgrading the ci/cd pipeline to reduce turnover time. (#783)

* adds decorator and prerelease tweaks * removes the duplicative prerelease-deps * remove reference to unused nox session/kokoro check * adds two import statements * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * adds exclusion to owlbot.py * corrects syntax error in owlbot.py file --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: chalmerlowe

noxfile.py

@@ -18,11 +18,13 @@
 
 from __future__ import absolute_import
 
+from functools import wraps
 import os
 import pathlib
 import re
 import shutil
 import subprocess
+import time
 import warnings
 
 import nox
@@ -76,6 +78,27 @@
 
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
+
+def _calculate_duration(func):
+ """This decorator prints the execution time for the decorated function."""
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ start = time.monotonic()
+ result = func(*args, **kwargs)
+ end = time.monotonic()
+ total_seconds = round(end - start)
+ hours = total_seconds // 3600 # Integer division to get hours
+ remaining_seconds = total_seconds % 3600 # Modulo to find remaining seconds
+ minutes = remaining_seconds // 60
+ seconds = remaining_seconds % 60
+ human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}"
+ print(f"Session ran in {total_seconds} seconds ({human_time})")
+ return result
+
+ return wrapper
+
+
 # 'docfx' is excluded since it only needs to run in 'docs-presubmit'
 nox.options.sessions = [
  "unit",
@@ -92,6 +115,7 @@
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def lint(session):
  """Run linters.
 
@@ -108,6 +132,7 @@ def lint(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def blacken(session):
  """Run black. Format code to uniform standard."""
  session.install(BLACK_VERSION)
@@ -118,6 +143,7 @@ def blacken(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def format(session):
  """
  Run isort to sort imports. Then run black
@@ -138,6 +164,7 @@ def format(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def lint_setup_py(session):
  """Verify that setup.py is valid (including RST check)."""
  session.install("docutils", "pygments")
@@ -199,6 +226,7 @@ def default(session):
 
 
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
+@_calculate_duration
 def unit(session):
  """Run the unit test suite."""
  default(session)
@@ -235,6 +263,7 @@ def install_systemtest_dependencies(session, *constraints):
 
 
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+@_calculate_duration
 def system(session):
  """Run the system test suite."""
  constraints_path = str(
@@ -281,6 +310,7 @@ def system(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def prerelease(session):
  session.install(
  "--extra-index-url",
@@ -334,7 +364,7 @@ def prerelease(session):
  constraints_text = constraints_file.read()
 
  # Ignore leading whitespace and comment lines.
- deps = [
+ constraints_deps = [
  match.group(1)
  for match in re.finditer(
  r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
@@ -343,7 +373,7 @@ def prerelease(session):
 
  # We use --no-deps to ensure that pre-release versions aren't overwritten
  # by the version ranges in setup.py.
- session.install(*deps)
+ session.install(*constraints_deps)
  session.install("--no-deps", "-e", ".[all]")
 
  # Print out prerelease package versions.
@@ -368,6 +398,7 @@ def prerelease(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def cover(session):
  """Run the final coverage report.
 
@@ -390,6 +421,7 @@ def cover(session):
 
 
 @nox.session(python="3.9")
+@_calculate_duration
 def docs(session):
  """Build the docs for this library."""
 
@@ -425,6 +457,7 @@ def docs(session):
 
 
 @nox.session(python="3.10")
+@_calculate_duration
 def docfx(session):
  """Build the docfx yaml files for this library."""
 
@@ -470,92 +503,6 @@ def docfx(session):
  )
 
 
-@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
-def prerelease_deps(session):
- """Run all tests with prerelease versions of dependencies installed."""
-
- # Install all dependencies
- session.install("-e", ".[all, tests, tracing]")
- unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES
- session.install(*unit_deps_all)
- system_deps_all = (
- SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES
- )
- session.install(*system_deps_all)
-
- # Because we test minimum dependency versions on the minimum Python
- # version, the first version we test with in the unit tests sessions has a
- # constraints file containing all dependencies and extras.
- with open(
- CURRENT_DIRECTORY
- / "testing"
- / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
- encoding="utf-8",
- ) as constraints_file:
- constraints_text = constraints_file.read()
-
- # Ignore leading whitespace and comment lines.
- constraints_deps = [
- match.group(1)
- for match in re.finditer(
- r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
- )
- ]
-
- session.install(*constraints_deps)
-
- prerel_deps = [
- # "protobuf",
- # dependency of grpc
- "six",
- "googleapis-common-protos",
- # Exclude version 1.52.0rc1 which has a known issue. See https://github.com/grpc/grpc/issues/32163
- "grpcio!=1.52.0rc1",
- "grpcio-status",
- "google-api-core",
- "google-auth",
- "proto-plus",
- "google-cloud-testutils",
- # dependencies of google-cloud-testutils"
- "click",
- ]
-
- for dep in prerel_deps:
- session.install("--pre", "--no-deps", "--upgrade", dep)
-
- # Remaining dependencies
- other_deps = [
- "requests",
- ]
- session.install(*other_deps)
-
- # Print out package versions.
- session.run("python", "-m", "pip", "freeze")
-
- session.run("py.test", "tests/unit")
-
- system_test_path = os.path.join("tests", "system.py")
- system_test_folder_path = os.path.join("tests", "system")
-
- # Only run system tests if found.
- if os.path.exists(system_test_path):
- session.run(
- "py.test",
- "--verbose",
- f"--junitxml=system_{session.python}_sponge_log.xml",
- system_test_path,
- *session.posargs,
- )
- if os.path.exists(system_test_folder_path):
- session.run(
- "py.test",
- "--verbose",
- f"--junitxml=system_{session.python}_sponge_log.xml",
- system_test_folder_path,
- *session.posargs,
- )
-
-
 def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages):
  """Installs packages from conda forge, pypi, and locally."""
 
@@ -573,6 +520,7 @@ def install_conda_unittest_dependencies(session, standard_deps, conda_forge_pack
 
 
 @nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba")
+@_calculate_duration
 def conda_test(session):
  """Run test suite in a conda virtual environment.
 

owlbot.py

@@ -56,9 +56,13 @@
  "docs/multiprocessing.rst",
  "noxfile.py",
  "README.rst",
+
  # exclude .kokoro/build.sh which is customized due to support for conda
  ".kokoro/build.sh",
- ],
+ 
+ # exclude this file as we have an alternate prerelease.cfg
+ ".kokoro/presubmit/prerelease-deps.cfg",
+ ]
 )
 
 # ----------------------------------------------------------------------------