Merge branch 'main' of https://github.com/googleapis/python-storage into transfer_manager_usage_metrics

Author: MiaCY

.github/.OwlBot.lock.yaml

@@ -13,4 +13,5 @@
 # limitations under the License.
 docker:
  image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
- digest: sha256:2e247c7bf5154df7f98cce087a20ca7605e236340c7d6d1a14447e5c06791bd6
+ digest: sha256:240b5bcc2bafd450912d2da2be15e62bc6de2cf839823ae4bf94d4f392b451dc
+# created: 2023-06-03T21:25:37.968717478Z

.kokoro/requirements.txt

@@ -113,28 +113,26 @@ commonmark==0.9.1 \
  --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \
  --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9
  # via rich
-cryptography==39.0.1 \
- --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \
- --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \
- --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \
- --hash=sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41 \
- --hash=sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965 \
- --hash=sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e \
- --hash=sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc \
- --hash=sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad \
- --hash=sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505 \
- --hash=sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388 \
- --hash=sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6 \
- --hash=sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2 \
- --hash=sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac \
- --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \
- --hash=sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6 \
- --hash=sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336 \
- --hash=sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0 \
- --hash=sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c \
- --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \
- --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \
- --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8
+cryptography==41.0.0 \
+ --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \
+ --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \
+ --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \
+ --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \
+ --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \
+ --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \
+ --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \
+ --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \
+ --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \
+ --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \
+ --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \
+ --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \
+ --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \
+ --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \
+ --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \
+ --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \
+ --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \
+ --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \
+ --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be
  # via
  # gcp-releasetool
  # secretstorage
@@ -419,9 +417,9 @@ readme-renderer==37.3 \
  --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \
  --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343
  # via twine
-requests==2.28.1 \
- --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \
- --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349
+requests==2.31.0 \
+ --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
+ --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
  # via
  # gcp-releasetool
  # google-api-core

google/cloud/storage/batch.py

@@ -133,18 +133,27 @@ class Batch(Connection):
 
  :type client: :class:`google.cloud.storage.client.Client`
  :param client: The client to use for making connections.
+
+ :type raise_exception: bool
+ :param raise_exception:
+ (Optional) Defaults to True. If True, instead of adding exceptions
+ to the list of return responses, the final exception will be raised.
+ Note that exceptions are unwrapped after all operations are complete
+ in success or failure, and only the last exception is raised.
  """
 
  _MAX_BATCH_SIZE = 1000
 
- def __init__(self, client):
+ def __init__(self, client, raise_exception=True):
  api_endpoint = client._connection.API_BASE_URL
  client_info = client._connection._client_info
  super(Batch, self).__init__(
  client, client_info=client_info, api_endpoint=api_endpoint
  )
  self._requests = []
  self._target_objects = []
+ self._responses = []
+ self._raise_exception = raise_exception
 
  def _do_request(
  self, method, url, headers, data, target_object, timeout=_DEFAULT_TIMEOUT
@@ -219,24 +228,34 @@ def _prepare_batch_request(self):
  _, body = payload.split("\n\n", 1)
  return dict(multi._headers), body, timeout
 
- def _finish_futures(self, responses):
+ def _finish_futures(self, responses, raise_exception=True):
  """Apply all the batch responses to the futures created.
 
  :type responses: list of (headers, payload) tuples.
  :param responses: List of headers and payloads from each response in
  the batch.
 
+ :type raise_exception: bool
+ :param raise_exception:
+ (Optional) Defaults to True. If True, instead of adding exceptions
+ to the list of return responses, the final exception will be raised.
+ Note that exceptions are unwrapped after all operations are complete
+ in success or failure, and only the last exception is raised.
+
  :raises: :class:`ValueError` if no requests have been deferred.
  """
  # If a bad status occurs, we track it, but don't raise an exception
  # until all futures have been populated.
+ # If raise_exception=False, we add exceptions to the list of responses.
  exception_args = None
 
  if len(self._target_objects) != len(responses): # pragma: NO COVER
  raise ValueError("Expected a response for every request.")
 
  for target_object, subresponse in zip(self._target_objects, responses):
- if not 200 <= subresponse.status_code < 300:
+ # For backwards compatibility, only the final exception will be raised.
+ # Set raise_exception=False to include all exceptions to the list of return responses.
+ if not 200 <= subresponse.status_code < 300 and raise_exception:
  exception_args = exception_args or subresponse
  elif target_object is not None:
  try:
@@ -247,9 +266,16 @@ def _finish_futures(self, responses):
  if exception_args is not None:
  raise exceptions.from_http_response(exception_args)
 
- def finish(self):
+ def finish(self, raise_exception=True):
  """Submit a single `multipart/mixed` request with deferred requests.
 
+ :type raise_exception: bool
+ :param raise_exception:
+ (Optional) Defaults to True. If True, instead of adding exceptions
+ to the list of return responses, the final exception will be raised.
+ Note that exceptions are unwrapped after all operations are complete
+ in success or failure, and only the last exception is raised.
+
  :rtype: list of tuples
  :returns: one ``(headers, payload)`` tuple per deferred request.
  """
@@ -269,7 +295,8 @@ def finish(self):
  raise exceptions.from_http_response(response)
 
  responses = list(_unpack_batch_response(response))
- self._finish_futures(responses)
+ self._finish_futures(responses, raise_exception=raise_exception)
+ self._responses = responses
  return responses
 
  def current(self):
@@ -283,7 +310,7 @@ def __enter__(self):
  def __exit__(self, exc_type, exc_val, exc_tb):
  try:
  if exc_type is None:
- self.finish()
+ self.finish(raise_exception=self._raise_exception)
  finally:
  self._client._pop_batch()
 

google/cloud/storage/client.py

@@ -302,17 +302,24 @@ def bucket(self, bucket_name, user_project=None):
  """
  return Bucket(client=self, name=bucket_name, user_project=user_project)
 
- def batch(self):
+ def batch(self, raise_exception=True):
  """Factory constructor for batch object.
 
  .. note::
  This will not make an HTTP request; it simply instantiates
  a batch object owned by this client.
 
+ :type raise_exception: bool
+ :param raise_exception:
+ (Optional) Defaults to True. If True, instead of adding exceptions
+ to the list of return responses, the final exception will be raised.
+ Note that exceptions are unwrapped after all operations are complete
+ in success or failure, and only the last exception is raised.
+
  :rtype: :class:`google.cloud.storage.batch.Batch`
  :returns: The batch object created.
  """
- return Batch(client=self)
+ return Batch(client=self, raise_exception=raise_exception)
 
  def _get_resource(
  self,

samples/snippets/requirements.txt

@@ -1,4 +1,4 @@
-google-cloud-pubsub==2.16.1
+google-cloud-pubsub==2.17.1
 google-cloud-storage==2.9.0
 pandas===1.3.5; python_version == '3.7'
-pandas==2.0.1; python_version >= '3.8'
+pandas==2.0.2; python_version >= '3.8'

tests/perf/README.md

@@ -32,6 +32,7 @@ $ python3 benchmarking.py --num_samples 10000 --object_size 5120..16384 --output
 | --test_type | test type to run benchmarking | `w1r3`, `range` | `w1r3` |
 | --output_file | file to output results to | any file path | `output_bench<TIMESTAMP>.csv` |
 | --tmp_dir | temp directory path on file system | any file path | `tm-perf-metrics` |
+| --delete_bucket | whether or not to delete GCS bucket used for benchmarking| bool | `False` |
 
 
 ## Workload definition and CSV headers

tests/perf/_perf_utils.py

@@ -193,19 +193,20 @@ def get_bucket_instance(bucket_name):
  return bucket
 
 
-def cleanup_bucket(bucket):
+def cleanup_bucket(bucket, delete_bucket=False):
  # Delete blobs first as the bucket may contain more than 256 blobs.
  try:
  blobs = bucket.list_blobs()
  for blob in blobs:
  blob.delete()
  except Exception as e:
  logging.exception(f"Caught an exception while deleting blobs\n {e}")
- # Delete bucket.
- try:
- bucket.delete(force=True)
- except Exception as e:
- logging.exception(f"Caught an exception while deleting bucket\n {e}")
+ # Delete bucket if delete_bucket is set to True
+ if delete_bucket:
+ try:
+ bucket.delete(force=True)
+ except Exception as e:
+ logging.exception(f"Caught an exception while deleting bucket\n {e}")
 
 
 def get_min_max_size(object_size):

tests/perf/benchmarking.py

@@ -80,7 +80,7 @@ def main(args):
  )
 
  # Cleanup and delete blobs.
- _pu.cleanup_bucket(bucket)
+ _pu.cleanup_bucket(bucket, delete_bucket=args.delete_bucket)
 
  # BBMC will not surface errors unless the process is terminated with a non zero code.
  if counter.count.errors != 0:
@@ -173,6 +173,12 @@ def main(args):
  default=_pu.DEFAULT_BASE_DIR,
  help="Temp directory path on file system",
  )
+ parser.add_argument(
+ "--delete_bucket",
+ type=bool,
+ default=False,
+ help="Whether or not to delete GCS bucket used for benchmarking",
+ )
  args = parser.parse_args()
 
  main(args)

tests/system/_helpers.py

@@ -111,8 +111,12 @@ def delete_bucket(bucket):
  retry(bucket.delete)(force=True)
 
 
-def await_config_changes_propagate(sec=3):
+def await_config_changes_propagate(sec=12):
  # Changes to the bucket will be readable immediately after writing,
  # but configuration changes may take time to propagate.
  # See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch
+ #
+ # The default was changed from 3 to 12 in May 2023 due to changes in bucket
+ # metadata handling. Note that the documentation recommends waiting "30
+ # seconds".
  time.sleep(sec)

tests/system/conftest.py

@@ -44,6 +44,8 @@
  "parent/child/other/file32.txt",
 ]
 
+ebh_bucket_iteration = 0
+
 
 @pytest.fixture(scope="session")
 def storage_client():
@@ -165,12 +167,20 @@ def signing_bucket(storage_client, signing_bucket_name):
  _helpers.delete_bucket(bucket)
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def default_ebh_bucket_name():
- return _helpers.unique_name("gcp-systest-default-ebh")
+ # Keep track of how many ebh buckets have been created so we can get a
+ # clean one each rerun. "unique_name" is unique per test iteration, not
+ # per test rerun.
+ global ebh_bucket_iteration
+ ebh_bucket_iteration += 1
+ return _helpers.unique_name("gcp-systest-default-ebh") + "-{}".format(
+ ebh_bucket_iteration
+ )
 
 
-@pytest.fixture(scope="session")
+# ebh_bucket/name are not scope=session because the bucket is modified in test.
+@pytest.fixture(scope="function")
 def default_ebh_bucket(storage_client, default_ebh_bucket_name):
  bucket = storage_client.bucket(default_ebh_bucket_name)
  bucket.default_event_based_hold = True