Add customer-supplied encryption to storage (#1844)

* Add customer-supplied encryption to storage.

Author: daspecster

gcloud/_helpers.py

@@ -394,6 +394,27 @@ def _to_bytes(value, encoding='ascii'):
  raise TypeError('%r could not be converted to bytes' % (value,))
 
 
+def _bytes_to_unicode(value):
+ """Converts bytes to a unicode value, if necessary.
+
+ :type value: bytes
+ :param value: bytes value to attempt string conversion on.
+
+ :rtype: str
+ :returns: The original value converted to unicode (if bytes) or as passed
+ in if it started out as unicode.
+
+ :raises: :class:`ValueError` if the value could not be converted to
+ unicode.
+ """
+ result = (value.decode('utf-8')
+ if isinstance(value, six.binary_type) else value)
+ if isinstance(result, six.text_type):
+ return result
+ else:
+ raise ValueError('%r could not be converted to unicode' % (value,))
+
+
 def _pb_timestamp_to_datetime(timestamp):
  """Convert a Timestamp protobuf to a datetime object.
 

gcloud/storage/blob.py

@@ -14,7 +14,9 @@
 
 """Create / interact with Google Cloud Storage blobs."""
 
+import base64
 import copy
+import hashlib
 from io import BytesIO
 import json
 import mimetypes
@@ -26,6 +28,8 @@
 from six.moves.urllib.parse import quote
 
 from gcloud._helpers import _rfc3339_to_datetime
+from gcloud._helpers import _to_bytes
+from gcloud._helpers import _bytes_to_unicode
 from gcloud.credentials import generate_signed_url
 from gcloud.exceptions import NotFound
 from gcloud.exceptions import make_exception
@@ -276,17 +280,41 @@ def delete(self, client=None):
  """
  return self.bucket.delete_blob(self.name, client=client)
 
- def download_to_file(self, file_obj, client=None):
+ def download_to_file(self, file_obj, encryption_key=None, client=None):
  """Download the contents of this blob into a file-like object.
 
  .. note::
 
  If the server-set property, :attr:`media_link`, is not yet
  initialized, makes an additional API request to load it.
 
+ Downloading a file that has been encrypted with a `customer-supplied`_
+ encryption key::
+
+ >>> from gcloud import storage
+ >>> from gcloud.storage import Blob
+
+ >>> client = storage.Client(project='my-project')
+ >>> bucket = client.get_bucket('my-bucket')
+ >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
+ >>> blob = Blob('secure-data', bucket)
+ >>> with open('/tmp/my-secure-file', 'wb') as file_obj:
+ >>> blob.download_to_file(file_obj,
+ ... encryption_key=encryption_key)
+
+ The ``encryption_key`` should be a str or bytes with a length of at
+ least 32.
+
+ .. _customer-supplied: https://cloud.google.com/storage/docs/\
+ encryption#customer-supplied
+
  :type file_obj: file
  :param file_obj: A file handle to which to write the blob's data.
 
+ :type encryption_key: str or bytes
+ :param encryption_key: Optional 32 byte encryption key for
+ customer-supplied encryption.
+
  :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the blob's bucket.
@@ -305,7 +333,11 @@ def download_to_file(self, file_obj, client=None):
  if self.chunk_size is not None:
  download.chunksize = self.chunk_size
 
- request = Request(download_url, 'GET')
+ headers = {}
+ if encryption_key:
+ _set_encryption_headers(encryption_key, headers)
+
+ request = Request(download_url, 'GET', headers)
 
  # Use the private ``_connection`` rather than the public
  # ``.connection``, since the public connection may be a batch. A
@@ -315,27 +347,36 @@ def download_to_file(self, file_obj, client=None):
  # it has all three (http, API_BASE_URL and build_api_url).
  download.initialize_download(request, client._connection.http)
 
- def download_to_filename(self, filename, client=None):
+ def download_to_filename(self, filename, encryption_key=None, client=None):
  """Download the contents of this blob into a named file.
 
  :type filename: string
  :param filename: A filename to be passed to ``open``.
 
+ :type encryption_key: str or bytes
+ :param encryption_key: Optional 32 byte encryption key for
+ customer-supplied encryption.
+
  :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the blob's bucket.
 
  :raises: :class:`gcloud.exceptions.NotFound`
  """
  with open(filename, 'wb') as file_obj:
- self.download_to_file(file_obj, client=client)
+ self.download_to_file(file_obj, encryption_key=encryption_key,
+ client=client)
 
  mtime = time.mktime(self.updated.timetuple())
  os.utime(file_obj.name, (mtime, mtime))
 
- def download_as_string(self, client=None):
+ def download_as_string(self, encryption_key=None, client=None):
  """Download the contents of this blob as a string.
 
+ :type encryption_key: str or bytes
+ :param encryption_key: Optional 32 byte encryption key for
+ customer-supplied encryption.
+
  :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the blob's bucket.
@@ -345,7 +386,8 @@ def download_as_string(self, client=None):
  :raises: :class:`gcloud.exceptions.NotFound`
  """
  string_buffer = BytesIO()
- self.download_to_file(string_buffer, client=client)
+ self.download_to_file(string_buffer, encryption_key=encryption_key,
+ client=client)
  return string_buffer.getvalue()
 
  @staticmethod
@@ -358,8 +400,10 @@ def _check_response_error(request, http_response):
  raise make_exception(faux_response, http_response.content,
  error_info=request.url)
 
+ # pylint: disable=too-many-locals
  def upload_from_file(self, file_obj, rewind=False, size=None,
- content_type=None, num_retries=6, client=None):
+ encryption_key=None, content_type=None, num_retries=6,
+ client=None):
  """Upload the contents of this blob from a file-like object.
 
  The content type of the upload will either be
@@ -378,6 +422,25 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
  `lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_
  API documents for details.
 
+ Uploading a file with a `customer-supplied`_ encryption key::
+
+ >>> from gcloud import storage
+ >>> from gcloud.storage import Blob
+
+ >>> client = storage.Client(project='my-project')
+ >>> bucket = client.get_bucket('my-bucket')
+ >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
+ >>> blob = Blob('secure-data', bucket)
+ >>> with open('my-file', 'rb') as my_file:
+ >>> blob.upload_from_file(my_file,
+ ... encryption_key=encryption_key)
+
+ The ``encryption_key`` should be a str or bytes with a length of at
+ least 32.
+
+ .. _customer-supplied: https://cloud.google.com/storage/docs/\
+ encryption#customer-supplied
+
  :type file_obj: file
  :param file_obj: A file handle open for reading.
 
@@ -391,6 +454,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
  :func:`os.fstat`. (If the file handle is not from the
  filesystem this won't be possible.)
 
+ :type encryption_key: str or bytes
+ :param encryption_key: Optional 32 byte encryption key for
+ customer-supplied encryption.
+
  :type content_type: string or ``NoneType``
  :param content_type: Optional type of content being uploaded.
 
@@ -434,6 +501,9 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
  'User-Agent': connection.USER_AGENT,
  }
 
+ if encryption_key:
+ _set_encryption_headers(encryption_key, headers)
+
  upload = Upload(file_obj, content_type, total_bytes,
  auto_transfer=False)
 
@@ -473,9 +543,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
  six.string_types): # pragma: NO COVER Python3
  response_content = response_content.decode('utf-8')
  self._set_properties(json.loads(response_content))
+ # pylint: enable=too-many-locals
 
  def upload_from_filename(self, filename, content_type=None,
- client=None):
+ encryption_key=None, client=None):
  """Upload this blob's contents from the content of a named file.
 
  The content type of the upload will either be
@@ -500,6 +571,10 @@ def upload_from_filename(self, filename, content_type=None,
  :type content_type: string or ``NoneType``
  :param content_type: Optional type of content being uploaded.
 
+ :type encryption_key: str or bytes
+ :param encryption_key: Optional 32 byte encryption key for
+ customer-supplied encryption.
+
  :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the blob's bucket.
@@ -510,10 +585,10 @@ def upload_from_filename(self, filename, content_type=None,
 
  with open(filename, 'rb') as file_obj:
  self.upload_from_file(file_obj, content_type=content_type,
- client=client)
+ encryption_key=encryption_key, client=client)
 
  def upload_from_string(self, data, content_type='text/plain',
- client=None):
+ encryption_key=None, client=None):
  """Upload contents of this blob from the provided string.
 
  .. note::
@@ -535,6 +610,10 @@ def upload_from_string(self, data, content_type='text/plain',
  :param content_type: Optional type of content being uploaded. Defaults
  to ``'text/plain'``.
 
+ :type encryption_key: str or bytes
+ :param encryption_key: Optional 32 byte encryption key for
+ customer-supplied encryption.
+
  :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
  :param client: Optional. The client to use. If not passed, falls back
  to the ``client`` stored on the blob's bucket.
@@ -545,7 +624,7 @@ def upload_from_string(self, data, content_type='text/plain',
  string_buffer.write(data)
  self.upload_from_file(file_obj=string_buffer, rewind=True,
  size=len(data), content_type=content_type,
- client=client)
+ encryption_key=encryption_key, client=client)
 
  def make_public(self, client=None):
  """Make this blob public giving all users read access.
@@ -838,3 +917,21 @@ def __init__(self, bucket_name, object_name):
  self.query_params = {'name': object_name}
  self._bucket_name = bucket_name
  self._relative_path = ''
+
+
+def _set_encryption_headers(key, headers):
+ """Builds customer encyrption key headers
+
+ :type key: str or bytes
+ :param key: 32 byte key to build request key and hash.
+
+ :type headers: dict
+ :param headers: dict of HTTP headers being sent in request.
+ """
+ key = _to_bytes(key)
+ sha256_key = hashlib.sha256(key).digest()
+ key_hash = base64.b64encode(sha256_key).rstrip()
+ encoded_key = base64.b64encode(key).rstrip()
+ headers['X-Goog-Encryption-Algorithm'] = 'AES256'
+ headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key)
+ headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash)