fix: remove utcnow usage (#1215)

* fix: replace utcnow in Bucket * replace utcnow in signing * update constant and tests * update datetime in post policy * replace utcnow in system tests * replace tzinfo

Author: cojenco

google/cloud/storage/_helpers.py

@@ -18,6 +18,7 @@
 """
 
 import base64
+import datetime
 from hashlib import md5
 import os
 from urllib.parse import urlsplit
@@ -77,6 +78,12 @@
  "object, or None, instead."
 )
 
+# _NOW() returns the current local date and time.
+# It is preferred to use timezone-aware datetimes _NOW(_UTC),
+# which returns the current UTC date and time.
+_NOW = datetime.datetime.now
+_UTC = datetime.timezone.utc
+
 
 def _get_storage_emulator_override():
  return os.environ.get(STORAGE_EMULATOR_ENV_VAR, None)

google/cloud/storage/_signing.py

@@ -28,9 +28,13 @@
 from google.auth import exceptions
 from google.auth.transport import requests
 from google.cloud import _helpers
+from google.cloud.storage._helpers import _NOW
+from google.cloud.storage._helpers import _UTC
 
 
-NOW = datetime.datetime.utcnow # To be replaced by tests.
+# `google.cloud.storage._signing.NOW` is deprecated.
+# Use `_NOW(_UTC)` instead.
+NOW = datetime.datetime.utcnow
 
 SERVICE_ACCOUNT_URL = (
  "https://googleapis.dev/python/google-api-core/latest/"
@@ -103,7 +107,7 @@ def get_expiration_seconds_v2(expiration):
  """
  # If it's a timedelta, add it to `now` in UTC.
  if isinstance(expiration, datetime.timedelta):
- now = NOW().replace(tzinfo=_helpers.UTC)
+ now = _NOW(_UTC)
  expiration = now + expiration
 
  # If it's a datetime, convert to a timestamp.
@@ -141,15 +145,14 @@ def get_expiration_seconds_v4(expiration):
  "timedelta. Got %s" % type(expiration)
  )
 
- now = NOW().replace(tzinfo=_helpers.UTC)
+ now = _NOW(_UTC)
 
  if isinstance(expiration, int):
  seconds = expiration
 
  if isinstance(expiration, datetime.datetime):
  if expiration.tzinfo is None:
  expiration = expiration.replace(tzinfo=_helpers.UTC)
-
  expiration = expiration - now
 
  if isinstance(expiration, datetime.timedelta):
@@ -638,7 +641,7 @@ def get_v4_now_dtstamps():
  :rtype: str, str
  :returns: Current timestamp, datestamp.
  """
- now = NOW()
+ now = _NOW(_UTC).replace(tzinfo=None)
  timestamp = now.strftime("%Y%m%dT%H%M%SZ")
  datestamp = now.date().strftime("%Y%m%d")
  return timestamp, datestamp

google/cloud/storage/bucket.py

@@ -23,14 +23,15 @@
 
 from google.api_core import datetime_helpers
 from google.cloud._helpers import _datetime_to_rfc3339
-from google.cloud._helpers import _NOW
 from google.cloud._helpers import _rfc3339_nanos_to_datetime
 from google.cloud.exceptions import NotFound
 from google.api_core.iam import Policy
 from google.cloud.storage import _signing
 from google.cloud.storage._helpers import _add_etag_match_headers
 from google.cloud.storage._helpers import _add_generation_match_parameters
+from google.cloud.storage._helpers import _NOW
 from google.cloud.storage._helpers import _PropertyMixin
+from google.cloud.storage._helpers import _UTC
 from google.cloud.storage._helpers import _scalar_property
 from google.cloud.storage._helpers import _validate_name
 from google.cloud.storage._signing import generate_signed_url_v2
@@ -3186,7 +3187,7 @@ def generate_upload_policy(self, conditions, expiration=None, client=None):
  _signing.ensure_signed_credentials(credentials)
 
  if expiration is None:
- expiration = _NOW() + datetime.timedelta(hours=1)
+ expiration = _NOW(_UTC).replace(tzinfo=None) + datetime.timedelta(hours=1)
 
  conditions = conditions + [{"bucket": self.name}]
 

google/cloud/storage/client.py

@@ -26,19 +26,21 @@
 from google.auth.credentials import AnonymousCredentials
 
 from google.api_core import page_iterator
-from google.cloud._helpers import _LocalStack, _NOW
+from google.cloud._helpers import _LocalStack
 from google.cloud.client import ClientWithProject
 from google.cloud.exceptions import NotFound
 
+from google.cloud.storage._helpers import _bucket_bound_hostname_url
+from google.cloud.storage._helpers import _get_api_endpoint_override
 from google.cloud.storage._helpers import _get_environ_project
-from google.cloud.storage._helpers import _use_client_cert
 from google.cloud.storage._helpers import _get_storage_emulator_override
-from google.cloud.storage._helpers import _get_api_endpoint_override
-from google.cloud.storage._helpers import _STORAGE_HOST_TEMPLATE
-from google.cloud.storage._helpers import _bucket_bound_hostname_url
+from google.cloud.storage._helpers import _use_client_cert
+from google.cloud.storage._helpers import _virtual_hosted_style_base_url
 from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
 from google.cloud.storage._helpers import _DEFAULT_SCHEME
-from google.cloud.storage._helpers import _virtual_hosted_style_base_url
+from google.cloud.storage._helpers import _STORAGE_HOST_TEMPLATE
+from google.cloud.storage._helpers import _NOW
+from google.cloud.storage._helpers import _UTC
 
 from google.cloud.storage._http import Connection
 from google.cloud.storage._signing import (
@@ -1707,7 +1709,7 @@ def generate_signed_post_policy_v4(
  conditions += required_conditions
 
  # calculate policy expiration time
- now = _NOW()
+ now = _NOW(_UTC).replace(tzinfo=None)
  if expiration is None:
  expiration = now + datetime.timedelta(hours=1)
 

tests/system/test__signing.py

@@ -22,6 +22,8 @@
 
 from google.api_core import path_template
 from google.cloud import iam_credentials_v1
+from google.cloud.storage._helpers import _NOW
+from google.cloud.storage._helpers import _UTC
 from . import _helpers
 
 
@@ -63,7 +65,7 @@ def test_create_signed_list_blobs_url_v2(storage_client, signing_bucket, no_mtls
 def test_create_signed_list_blobs_url_v2_w_expiration(
  storage_client, signing_bucket, no_mtls
 ):
- now = datetime.datetime.utcnow()
+ now = _NOW(_UTC).replace(tzinfo=None)
  delta = datetime.timedelta(seconds=10)
 
  _create_signed_list_blobs_url_helper(
@@ -85,7 +87,7 @@ def test_create_signed_list_blobs_url_v4(storage_client, signing_bucket, no_mtls
 def test_create_signed_list_blobs_url_v4_w_expiration(
  storage_client, signing_bucket, no_mtls
 ):
- now = datetime.datetime.utcnow()
+ now = _NOW(_UTC).replace(tzinfo=None)
  delta = datetime.timedelta(seconds=10)
  _create_signed_list_blobs_url_helper(
  storage_client,
@@ -158,7 +160,7 @@ def test_create_signed_read_url_v4(storage_client, signing_bucket, no_mtls):
 def test_create_signed_read_url_v2_w_expiration(
  storage_client, signing_bucket, no_mtls
 ):
- now = datetime.datetime.utcnow()
+ now = _NOW(_UTC).replace(tzinfo=None)
  delta = datetime.timedelta(seconds=10)
 
  _create_signed_read_url_helper(
@@ -169,7 +171,7 @@ def test_create_signed_read_url_v2_w_expiration(
 def test_create_signed_read_url_v4_w_expiration(
  storage_client, signing_bucket, no_mtls
 ):
- now = datetime.datetime.utcnow()
+ now = _NOW(_UTC).replace(tzinfo=None)
  delta = datetime.timedelta(seconds=10)
  _create_signed_read_url_helper(
  storage_client, signing_bucket, expiration=now + delta, version="v4"
@@ -391,14 +393,15 @@ def test_generate_signed_post_policy_v4(
  with open(blob_name, "wb") as f:
  f.write(payload)
 
+ now = _NOW(_UTC).replace(tzinfo=None)
  policy = storage_client.generate_signed_post_policy_v4(
  bucket_name,
  blob_name,
  conditions=[
  {"bucket": bucket_name},
  ["starts-with", "$Content-Type", "text/pla"],
  ],
- expiration=datetime.datetime.utcnow() + datetime.timedelta(hours=1),
+ expiration=now + datetime.timedelta(hours=1),
  fields={"content-type": "text/plain"},
  )
  with open(blob_name, "r") as f:
@@ -424,14 +427,15 @@ def test_generate_signed_post_policy_v4_invalid_field(
  with open(blob_name, "wb") as f:
  f.write(payload)
 
+ now = _NOW(_UTC).replace(tzinfo=None)
  policy = storage_client.generate_signed_post_policy_v4(
  bucket_name,
  blob_name,
  conditions=[
  {"bucket": bucket_name},
  ["starts-with", "$Content-Type", "text/pla"],
  ],
- expiration=datetime.datetime.utcnow() + datetime.timedelta(hours=1),
+ expiration=now + datetime.timedelta(hours=1),
  fields={"x-goog-random": "invalid_field", "content-type": "text/plain"},
  )
  with open(blob_name, "r") as f:

tests/system/test_blob.py

@@ -1120,6 +1120,9 @@ def test_blob_update_storage_class_large_file(
 
 
 def test_object_retention_lock(storage_client, buckets_to_delete, blobs_to_delete):
+ from google.cloud.storage._helpers import _NOW
+ from google.cloud.storage._helpers import _UTC
+
  # Test bucket created with object retention enabled
  new_bucket_name = _helpers.unique_name("object-retention")
  created_bucket = _helpers.retry_429_503(storage_client.create_bucket)(
@@ -1131,7 +1134,7 @@ def test_object_retention_lock(storage_client, buckets_to_delete, blobs_to_delet
  # Test create object with object retention enabled
  payload = b"Hello World"
  mode = "Unlocked"
- current_time = datetime.datetime.utcnow()
+ current_time = _NOW(_UTC).replace(tzinfo=None)
  expiration_time = current_time + datetime.timedelta(seconds=10)
  blob = created_bucket.blob("object-retention-lock")
  blob.retention.mode = mode

tests/system/test_hmac_key_metadata.py

@@ -16,8 +16,6 @@
 
 import pytest
 
-from google.cloud import _helpers as _cloud_helpers
-
 from . import _helpers
 
 
@@ -32,9 +30,12 @@ def ensure_hmac_key_deleted(hmac_key):
 
 @pytest.fixture
 def scrubbed_hmac_keys(storage_client):
+ from google.cloud.storage._helpers import _NOW
+ from google.cloud.storage._helpers import _UTC
+
  before_hmac_keys = set(storage_client.list_hmac_keys())
 
- now = datetime.datetime.utcnow().replace(tzinfo=_cloud_helpers.UTC)
+ now = _NOW(_UTC)
  yesterday = now - datetime.timedelta(days=1)
 
  # Delete any HMAC keys older than a day.

tests/system/test_transfer_manager.py

@@ -267,10 +267,10 @@ def test_upload_chunks_concurrently(shared_bucket, file_data, blobs_to_delete):
 def test_upload_chunks_concurrently_with_metadata(
  shared_bucket, file_data, blobs_to_delete
 ):
- import datetime
- from google.cloud._helpers import UTC
+ from google.cloud.storage._helpers import _NOW
+ from google.cloud.storage._helpers import _UTC
 
- now = datetime.datetime.utcnow().replace(tzinfo=UTC)
+ now = _NOW(_UTC)
  custom_metadata = {"key_a": "value_a", "key_b": "value_b"}
 
  METADATA = {