fix: update minimum dependency versions (#263)

This PR updates the minimum dependency versions to match those that I found to be actually runnable. Updates tests to use constraint files so that at least one test session uses these minimum versions. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [x] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! - Based on internal Python Client Library Testing Improvements docs. - In response to internal bug 166792569 covering insufficient pyarrow minimum dependency. - [x] Ensure the tests and linter pass - [x] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary)

Author: tswast

noxfile.py

@@ -14,6 +14,7 @@
 
 from __future__ import absolute_import
 
+import pathlib
 import os
 import shutil
 
@@ -22,6 +23,7 @@
 
 BLACK_VERSION = "black==19.10b0"
 BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py")
+CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
 
 def default(session):
@@ -32,27 +34,33 @@ def default(session):
  Python corresponding to the ``nox`` binary the ``PATH`` can
  run the tests.
  """
+ constraints_path = str(
+ CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+ )
+
  # Install all test dependencies, then install local packages in-place.
  session.install(
- "mock", "pytest", "google-cloud-testutils", "pytest-cov", "freezegun"
+ "mock",
+ "pytest",
+ "google-cloud-testutils",
+ "pytest-cov",
+ "freezegun",
+ "-c",
+ constraints_path,
  )
- session.install("grpcio")
-
- # fastparquet is not included in .[all] because, in general, it's redundant
- # with pyarrow. We still want to run some unit tests with fastparquet
- # serialization, though.
- session.install("-e", ".[all,fastparquet]")
 
- # IPython does not support Python 2 after version 5.x
  if session.python == "2.7":
- session.install("ipython==5.5")
+ # The [all] extra is not installable on Python 2.7.
+ session.install("-e", ".[pandas,pyarrow]", "-c", constraints_path)
+ elif session.python == "3.5":
+ session.install("-e", ".[all]", "-c", constraints_path)
  else:
- session.install("ipython")
+ # fastparquet is not included in .[all] because, in general, it's
+ # redundant with pyarrow. We still want to run some unit tests with
+ # fastparquet serialization, though.
+ session.install("-e", ".[all,fastparquet]", "-c", constraints_path)
 
- # opentelemetry was not added to [all] because opentelemetry does not support Python 2.
- # Exporter does not need to be in nox thus it has been added to README documentation
- if session.python != "2.7":
- session.install("-e", ".[opentelemetry]")
+ session.install("ipython", "-c", constraints_path)
 
  # Run py.test against the unit tests.
  session.run(
@@ -79,6 +87,10 @@ def unit(session):
 def system(session):
  """Run the system test suite."""
 
+ constraints_path = str(
+ CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+ )
+
  # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
  if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
  session.skip("RUN_SYSTEM_TESTS is set to false, skipping")
@@ -88,18 +100,21 @@ def system(session):
  session.skip("Credentials must be set via environment variable.")
 
  # Use pre-release gRPC for system tests.
- session.install("--pre", "grpcio")
+ session.install("--pre", "grpcio", "-c", constraints_path)
 
  # Install all test dependencies, then install local packages in place.
- session.install("mock", "pytest", "psutil", "google-cloud-testutils")
- session.install("google-cloud-storage")
- session.install("-e", ".[all]")
+ session.install(
+ "mock", "pytest", "psutil", "google-cloud-testutils", "-c", constraints_path
+ )
+ session.install("google-cloud-storage", "-c", constraints_path)
 
- # IPython does not support Python 2 after version 5.x
  if session.python == "2.7":
- session.install("ipython==5.5")
+ # The [all] extra is not installable on Python 2.7.
+ session.install("-e", ".[pandas]", "-c", constraints_path)
  else:
- session.install("ipython")
+ session.install("-e", ".[all]", "-c", constraints_path)
+
+ session.install("ipython", "-c", constraints_path)
 
  # Run py.test against the system tests.
  session.run(
@@ -111,15 +126,24 @@ def system(session):
 def snippets(session):
  """Run the snippets test suite."""
 
+ constraints_path = str(
+ CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+ )
+
  # Sanity check: Only run snippets tests if the environment variable is set.
  if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
  session.skip("Credentials must be set via environment variable.")
 
  # Install all test dependencies, then install local packages in place.
- session.install("mock", "pytest", "google-cloud-testutils")
- session.install("google-cloud-storage")
- session.install("grpcio")
- session.install("-e", ".[all]")
+ session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path)
+ session.install("google-cloud-storage", "-c", constraints_path)
+ session.install("grpcio", "-c", constraints_path)
+
+ if session.python == "2.7":
+ # The [all] extra is not installable on Python 2.7.
+ session.install("-e", ".[pandas]", "-c", constraints_path)
+ else:
+ session.install("-e", ".[all]", "-c", constraints_path)
 
  # Run py.test against the snippets tests.
  # Skip tests in samples/snippets, as those are run in a different session

samples/snippets/jupyter_tutorial_test.py

@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import IPython
-from IPython.terminal import interactiveshell
-from IPython.testing import tools
-import matplotlib
 import pytest
 
+IPython = pytest.importorskip("IPython")
+interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell")
+tools = pytest.importorskip("IPython.testing.tools")
+matplotlib = pytest.importorskip("matplotlib")
 
 # Ignore semicolon lint warning because semicolons are used in notebooks
 # flake8: noqa E703

samples/tests/test_download_public_data.py

@@ -14,8 +14,12 @@
 
 import logging
 
+import pytest
+
 from .. import download_public_data
 
+pytest.importorskip("google.cloud.bigquery_storage_v1")
+
 
 def test_download_public_data(caplog, capsys):
  # Enable debug-level logging to verify the BigQuery Storage API is used.

samples/tests/test_download_public_data_sandbox.py

@@ -14,8 +14,12 @@
 
 import logging
 
+import pytest
+
 from .. import download_public_data_sandbox
 
+pytest.importorskip("google.cloud.bigquery_storage_v1")
+
 
 def test_download_public_data_sandbox(caplog, capsys):
  # Enable debug-level logging to verify the BigQuery Storage API is used.

samples/tests/test_query_to_arrow.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pyarrow
+import pytest
 
 from .. import query_to_arrow
 
+pyarrow = pytest.importorskip("pyarrow")
+
 
 def test_query_to_arrow(capsys,):
 

setup.py

@@ -32,7 +32,7 @@
  'enum34; python_version < "3.4"',
  "google-api-core >= 1.21.0, < 2.0dev",
  "google-cloud-core >= 1.4.1, < 2.0dev",
- "google-resumable-media >= 0.5.0, < 2.0dev",
+ "google-resumable-media >= 0.6.0, < 2.0dev",
  "six >=1.13.0,< 2.0.0dev",
 ]
 extras = {
@@ -41,18 +41,19 @@
  # Due to an issue in pip's dependency resolver, the `grpc` extra is not
  # installed, even though `google-cloud-bigquery-storage` specifies it
  # as `google-api-core[grpc]`. We thus need to explicitly specify it here.
- # See: https://github.com/googleapis/python-bigquery/issues/83
- "grpcio >= 1.8.2, < 2.0dev",
- "pyarrow >= 1.0.0, < 2.0dev; python_version >= '3.5'",
+ # See: https://github.com/googleapis/python-bigquery/issues/83 The
+ # grpc.Channel.close() method isn't added until 1.32.0.
+ # https://github.com/grpc/grpc/pull/15254
+ "grpcio >= 1.32.0, < 2.0dev",
+ "pyarrow >= 1.0.0, < 2.0dev",
  ],
- "pandas": ["pandas>=0.17.1"],
- # Exclude PyArrow dependency from Windows Python 2.7.
+ "pandas": ["pandas>=0.23.0"],
  "pyarrow": [
- "pyarrow >= 1.0.0, < 2.0dev; python_version >= '3.5'",
- # Pyarrow >= 0.17.0 is not compatible with Python 2 anymore.
- "pyarrow < 0.17.0; python_version < '3.0' and platform_system != 'Windows'",
+ # pyarrow 1.0.0 is required for the use of timestamp_as_object keyword.
+ "pyarrow >= 1.0.0, < 2.0de ; python_version>='3.5'",
+ "pyarrow >= 0.16.0, < 0.17.0dev ; python_version<'3.5'",
  ],
- "tqdm": ["tqdm >= 4.0.0, <5.0.0dev"],
+ "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"],
  "fastparquet": [
  "fastparquet",
  "python-snappy",
@@ -77,8 +78,6 @@
  # creates a dependency on pre-release versions of numpy. See:
  # https://github.com/googleapis/google-cloud-python/issues/8549
  "fastparquet",
- # Skip opentelemetry because the library is not compatible with Python 2.
- "opentelemetry",
  ):
  continue
  all_extras.extend(extras[extra])

testing/constraints-2.7.txt

@@ -0,0 +1,9 @@
+google-api-core==1.21.0
+google-cloud-core==1.4.1
+google-cloud-storage==1.30.0
+google-resumable-media==0.6.0
+ipython==5.5
+pandas==0.23.0
+pyarrow==0.16.0
+six==1.13.0
+tqdm==4.7.4

testing/constraints-3.5.txt

@@ -0,0 +1,12 @@
+google-api-core==1.21.0
+google-cloud-bigquery-storage==1.0.0
+google-cloud-core==1.4.1
+google-resumable-media==0.6.0
+google-cloud-storage==1.30.0
+grpcio==1.32.0
+ipython==5.5
+# pandas 0.23.0 is the first version to work with pyarrow to_pandas.
+pandas==0.23.0
+pyarrow==1.0.0
+six==1.13.0
+tqdm==4.7.4