BigQuery: replace httplib2 with Requests

Author: Jon Wayne Parrott

bigquery/google/cloud/bigquery/client.py

@@ -64,10 +64,10 @@ class Client(ClientWithProject):
  passed), falls back to the default inferred from the
  environment.
 
- :type _http: :class:`~httplib2.Http`
+ :type _http: :class:`~requests.Session`
  :param _http: (Optional) HTTP object to make requests. Can be any object
  that defines ``request()`` with the same interface as
- :meth:`~httplib2.Http.request`. If not passed, an
+ :meth:`requests.Session.request`. If not passed, an
  ``_http`` object is created that is bound to the
  ``credentials`` for the current object.
  This parameter should be considered private, and could

bigquery/google/cloud/bigquery/dataset.py

@@ -364,7 +364,7 @@ def _parse_access_grants(access):
  def _set_properties(self, api_response):
  """Update properties from resource in body of ``api_response``
 
- :type api_response: httplib2.Response
+ :type api_response: dict
  :param api_response: response returned from an API call.
  """
  self._properties.clear()

bigquery/google/cloud/bigquery/job.py

@@ -14,7 +14,6 @@
 
 """Define API Jobs."""
 
-import collections
 import threading
 
 import six
@@ -58,8 +57,6 @@
  'tableUnavailable': http_client.BAD_REQUEST,
 }
 
-_FakeResponse = collections.namedtuple('_FakeResponse', ['status'])
-
 
 def _error_result_to_exception(error_result):
  """Maps BigQuery error reasons to an exception.
@@ -80,12 +77,8 @@ def _error_result_to_exception(error_result):
  status_code = _ERROR_REASON_TO_EXCEPTION.get(
  reason, http_client.INTERNAL_SERVER_ERROR)
  # make_exception expects an httplib2 response object.
- fake_response = _FakeResponse(status=status_code)
- return exceptions.make_exception(
- fake_response,
- error_result.get('message', ''),
- error_info=error_result,
- use_json=False)
+ return exceptions.from_http_status(
+ status_code, error_result.get('message', ''), errors=[error_result])
 
 
 class Compression(_EnumProperty):
@@ -307,7 +300,7 @@ def _scrub_local_properties(self, cleaned):
  def _set_properties(self, api_response):
  """Update properties from resource in body of ``api_response``
 
- :type api_response: httplib2.Response
+ :type api_response: dict
  :param api_response: response returned from an API call
  """
  cleaned = api_response.copy()

bigquery/google/cloud/bigquery/query.py

@@ -310,7 +310,7 @@ def schema(self):
  def _set_properties(self, api_response):
  """Update properties from resource in body of ``api_response``
 
- :type api_response: httplib2.Response
+ :type api_response: dict
  :param api_response: response returned from an API call
  """
  self._properties.clear()

bigquery/google/cloud/bigquery/table.py

@@ -17,18 +17,16 @@
 import datetime
 import os
 
-import httplib2
 import six
 
 import google.auth.transport.requests
 from google import resumable_media
 from google.resumable_media.requests import MultipartUpload
 from google.resumable_media.requests import ResumableUpload
 
+from google.cloud import exceptions
 from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud._helpers import _millis_from_datetime
-from google.cloud.exceptions import NotFound
-from google.cloud.exceptions import make_exception
 from google.cloud.iterator import HTTPIterator
 from google.cloud.bigquery.schema import SchemaField
 from google.cloud.bigquery._helpers import _item_to_row
@@ -474,7 +472,7 @@ def _require_client(self, client):
  def _set_properties(self, api_response):
  """Update properties from resource in body of ``api_response``
 
- :type api_response: httplib2.Response
+ :type api_response: dict
  :param api_response: response returned from an API call
  """
  self._properties.clear()
@@ -563,7 +561,7 @@ def exists(self, client=None):
  try:
  client._connection.api_request(method='GET', path=self.path,
  query_params={'fields': 'id'})
- except NotFound:
+ except exceptions.NotFound:
  return False
  else:
  return True
@@ -1113,7 +1111,7 @@ def upload_from_file(self,
  client, file_obj, metadata, size, num_retries)
  return client.job_from_resource(created_json)
  except resumable_media.InvalidResponse as exc:
- _raise_from_invalid_response(exc)
+ raise exceptions.from_http_response(exc.response)
  # pylint: enable=too-many-arguments,too-many-locals
 
 
@@ -1298,22 +1296,3 @@ def _get_upload_metadata(source_format, schema, dataset, name):
  'load': load_config,
  },
  }
-
-
-def _raise_from_invalid_response(error, error_info=None):
- """Re-wrap and raise an ``InvalidResponse`` exception.
-
- :type error: :exc:`google.resumable_media.InvalidResponse`
- :param error: A caught exception from the ``google-resumable-media``
- library.
-
- :type error_info: str
- :param error_info: (Optional) Extra information about the failed request.
-
- :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding
- to the failed status code
- """
- response = error.response
- faux_response = httplib2.Response({'status': response.status_code})
- raise make_exception(faux_response, response.content,
- error_info=error_info, use_json=False)

bigquery/tests/unit/test__http.py

@@ -15,6 +15,7 @@
 import unittest
 
 import mock
+import requests
 
 
 class TestConnection(unittest.TestCase):
@@ -55,10 +56,12 @@ def test_extra_headers(self):
  from google.cloud import _http as base_http
  from google.cloud.bigquery import _http as MUT
 
- http = mock.Mock(spec=['request'])
- response = mock.Mock(status=200, spec=['status'])
+ http = mock.create_autospec(requests.Session, instance=True)
+ response = requests.Response()
+ response.status_code = 200
  data = b'brent-spiner'
- http.request.return_value = response, data
+ response._content = data
+ http.request.return_value = response
  client = mock.Mock(_http=http, spec=['_http'])
 
  conn = self._make_one(client)
@@ -68,15 +71,14 @@ def test_extra_headers(self):
  self.assertEqual(result, data)
 
  expected_headers = {
- 'Content-Length': str(len(req_data)),
  'Accept-Encoding': 'gzip',
  base_http.CLIENT_INFO_HEADER: MUT._CLIENT_INFO,
  'User-Agent': conn.USER_AGENT,
  }
  expected_uri = conn.build_api_url('/rainbow')
  http.request.assert_called_once_with(
- body=req_data,
+ data=req_data,
  headers=expected_headers,
  method='GET',
- uri=expected_uri,
+ url=expected_uri,
  )

bigquery/tests/unit/test_job.py

@@ -31,7 +31,7 @@ def test_simple(self):
  exception = self._call_fut(error_result)
  self.assertEqual(exception.code, http_client.BAD_REQUEST)
  self.assertTrue(exception.message.startswith('bad request'))
- self.assertIn("'reason': 'invalid'", exception.message)
+ self.assertIn(error_result, exception.errors)
 
  def test_missing_reason(self):
  error_result = {}

bigquery/tests/unit/test_table.py

@@ -1744,9 +1744,11 @@ def _make_table():
  def _make_response(status_code, content='', headers={}):
  """Make a mock HTTP response."""
  import requests
- response = mock.create_autospec(requests.Response, instance=True)
- response.content = content.encode('utf-8')
- response.headers = headers
+ response = requests.Response()
+ response.request = requests.Request(
+ 'POST', 'http://example.com').prepare()
+ response._content = content.encode('utf-8')
+ response.headers.update(headers)
  response.status_code = status_code
  return response
 
@@ -1921,7 +1923,7 @@ def test_upload_from_file_failure(self):
  table.upload_from_file(
  file_obj, source_format='CSV', rewind=True)
 
- assert exc_info.value.message == response.content.decode('utf-8')
+ assert response.text in exc_info.value.message
  assert exc_info.value.errors == []
 
  def test_upload_from_file_bad_mode(self):