Storage: replace httplib2 with Requests

Author: Jon Wayne Parrott

storage/google/cloud/storage/batch.py

@@ -23,10 +23,10 @@
 import io
 import json
 
-import httplib2
+import requests
 import six
 
-from google.cloud.exceptions import make_exception
+from google.cloud import exceptions
 from google.cloud.storage._http import Connection
 
 
@@ -70,11 +70,6 @@ def __init__(self, method, uri, headers, body):
  super_init(payload, 'http', encode_noop)
 
 
-class NoContent(object):
- """Emulate an HTTP '204 No Content' response."""
- status = 204
-
-
 class _FutureDict(object):
  """Class to hold a future value for a deferred request.
 
@@ -123,6 +118,19 @@ def __setitem__(self, key, value):
  raise KeyError('Cannot set %r -> %r on a future' % (key, value))
 
 
+class _FutureResponse(requests.Response):
+ def __init__(self, future_dict):
+ super(_FutureResponse, self).__init__()
+ self._future_dict = future_dict
+ self.status_code = 204
+
+ def json(self):
+ raise ValueError()
+
+ def content(self):
+ return self._future_dict
+
+
 class Batch(Connection):
  """Proxy an underlying connection, batching up change operations.
 
@@ -171,7 +179,7 @@ def _do_request(self, method, url, headers, data, target_object):
  self._target_objects.append(target_object)
  if target_object is not None:
  target_object._properties = result
- return NoContent(), result
+ return _FutureResponse(result)
 
  def _prepare_batch_request(self):
  """Prepares headers and body for a batch request.
@@ -218,17 +226,18 @@ def _finish_futures(self, responses):
  if len(self._target_objects) != len(responses):
  raise ValueError('Expected a response for every request.')
 
- for target_object, sub_response in zip(self._target_objects,
- responses):
- resp_headers, sub_payload = sub_response
- if not 200 <= resp_headers.status < 300:
- exception_args = exception_args or (resp_headers,
- sub_payload)
+ for target_object, subresponse in zip(
+ self._target_objects, responses):
+ if not 200 <= subresponse.status_code < 300:
+ exception_args = exception_args or subresponse
  elif target_object is not None:
- target_object._properties = sub_payload
+ try:
+ target_object._properties = subresponse.json()
+ except ValueError:
+ target_object._properties = subresponse.content
 
  if exception_args is not None:
- raise make_exception(*exception_args)
+ raise exceptions.from_http_response(exception_args)
 
  def finish(self):
  """Submit a single `multipart/mixed` request with deferred requests.
@@ -243,9 +252,9 @@ def finish(self):
  # Use the private ``_base_connection`` rather than the property
  # ``_connection``, since the property may be this
  # current batch.
- response, content = self._client._base_connection._make_request(
+ response = self._client._base_connection._make_request(
  'POST', url, data=body, headers=headers)
- responses = list(_unpack_batch_response(response, content))
+ responses = list(_unpack_batch_response(response))
  self._finish_futures(responses)
  return responses
 
@@ -265,24 +274,23 @@ def __exit__(self, exc_type, exc_val, exc_tb):
  self._client._pop_batch()
 
 
-def _generate_faux_mime_message(parser, response, content):
+def _generate_faux_mime_message(parser, response):
  """Convert response, content -> (multipart) email.message.
 
  Helper for _unpack_batch_response.
  """
  # We coerce to bytes to get consistent concat across
  # Py2 and Py3. Percent formatting is insufficient since
  # it includes the b in Py3.
- if not isinstance(content, six.binary_type):
- content = content.encode('utf-8')
- content_type = response['content-type']
+ content_type = response.headers.get('content-type', '')
  if not isinstance(content_type, six.binary_type):
  content_type = content_type.encode('utf-8')
+
  faux_message = b''.join([
  b'Content-Type: ',
  content_type,
  b'\nMIME-Version: 1.0\n\n',
- content,
+ response.content,
  ])
 
  if six.PY2:
@@ -291,20 +299,17 @@ def _generate_faux_mime_message(parser, response, content):
  return parser.parsestr(faux_message.decode('utf-8'))
 
 
-def _unpack_batch_response(response, content):
- """Convert response, content -> [(headers, payload)].
+def _unpack_batch_response(response):
+ """Convert requests.Response -> [(headers, payload)].
 
  Creates a generator of tuples of emulating the responses to
  :meth:`httplib2.Http.request` (a pair of headers and payload).
 
- :type response: :class:`httplib2.Response`
+ :type response: :class:`requests.Response`
  :param response: HTTP response / headers from a request.
-
- :type content: str
- :param content: Response payload with a batch response.
  """
  parser = Parser()
- message = _generate_faux_mime_message(parser, response, content)
+ message = _generate_faux_mime_message(parser, response)
 
  if not isinstance(message._payload, list):
  raise ValueError('Bad response: not multi-part')
@@ -314,10 +319,15 @@ def _unpack_batch_response(response, content):
  _, status, _ = status_line.split(' ', 2)
  sub_message = parser.parsestr(rest)
  payload = sub_message._payload
- ctype = sub_message['Content-Type']
  msg_headers = dict(sub_message._headers)
- msg_headers['status'] = status
- headers = httplib2.Response(msg_headers)
- if ctype and ctype.startswith('application/json'):
- payload = json.loads(payload)
- yield headers, payload
+ content_id = msg_headers.get('Content-ID')
+
+ subresponse = requests.Response()
+ subresponse.request = requests.Request(
+ method='BATCH',
+ url='contentid://{}'.format(content_id)).prepare()
+ subresponse.status_code = int(status)
+ subresponse.headers.update(msg_headers)
+ subresponse._content = payload.encode('utf-8')
+
+ yield subresponse

storage/google/cloud/storage/blob.py

@@ -34,7 +34,6 @@
 import time
 import warnings
 
-import httplib2
 from six.moves.urllib.parse import quote
 
 import google.auth.transport.requests
@@ -44,11 +43,11 @@
 from google.resumable_media.requests import MultipartUpload
 from google.resumable_media.requests import ResumableUpload
 
+from google.cloud import exceptions
 from google.cloud._helpers import _rfc3339_to_datetime
 from google.cloud._helpers import _to_bytes
 from google.cloud._helpers import _bytes_to_unicode
 from google.cloud.exceptions import NotFound
-from google.cloud.exceptions import make_exception
 from google.cloud.iam import Policy
 from google.cloud.storage._helpers import _PropertyMixin
 from google.cloud.storage._helpers import _scalar_property
@@ -469,7 +468,7 @@ def download_to_file(self, file_obj, client=None):
  try:
  self._do_download(transport, file_obj, download_url, headers)
  except resumable_media.InvalidResponse as exc:
- _raise_from_invalid_response(exc, download_url)
+ _raise_from_invalid_response(exc)
 
  def download_to_filename(self, filename, client=None):
  """Download the contents of this blob into a named file.
@@ -1598,20 +1597,14 @@ def _maybe_rewind(stream, rewind=False):
  stream.seek(0, os.SEEK_SET)
 
 
-def _raise_from_invalid_response(error, error_info=None):
+def _raise_from_invalid_response(error):
  """Re-wrap and raise an ``InvalidResponse`` exception.
 
  :type error: :exc:`google.resumable_media.InvalidResponse`
  :param error: A caught exception from the ``google-resumable-media``
  library.
 
- :type error_info: str
- :param error_info: (Optional) Extra information about the failed request.
-
  :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding
  to the failed status code
  """
- response = error.response
- faux_response = httplib2.Response({'status': response.status_code})
- raise make_exception(faux_response, response.content,
- error_info=error_info, use_json=False)
+ raise exceptions.from_http_response(error.response)

storage/tests/unit/test__http.py

@@ -29,13 +29,17 @@ def _make_one(self, *args, **kw):
  return self._get_target_class()(*args, **kw)
 
  def test_extra_headers(self):
+ import requests
+
  from google.cloud import _http as base_http
  from google.cloud.storage import _http as MUT
 
- http = mock.Mock(spec=['request'])
- response = mock.Mock(status=200, spec=['status'])
+ http = mock.create_autospec(requests.Session, instance=True)
+ response = requests.Response()
+ response.status_code = 200
  data = b'brent-spiner'
- http.request.return_value = response, data
+ response._content = data
+ http.request.return_value = response
  client = mock.Mock(_http=http, spec=['_http'])
 
  conn = self._make_one(client)
@@ -45,17 +49,16 @@ def test_extra_headers(self):
  self.assertEqual(result, data)
 
  expected_headers = {
- 'Content-Length': str(len(req_data)),
  'Accept-Encoding': 'gzip',
  base_http.CLIENT_INFO_HEADER: MUT._CLIENT_INFO,
  'User-Agent': conn.USER_AGENT,
  }
  expected_uri = conn.build_api_url('/rainbow')
  http.request.assert_called_once_with(
- body=req_data,
+ data=req_data,
  headers=expected_headers,
  method='GET',
- uri=expected_uri,
+ url=expected_uri,
  )
 
  def test_build_api_url_no_extra_query_params(self):