feature: V4 Post policies (#87)

* feat: add POST policies building method * add comments, ignoring x-ignore fields and required fields validation * fix docs style, add virtual hosted style URLs * add bucket_bound_hostname support * cosmetic changes * add unit tests * Revert "add unit tests" This reverts commit f56440b. * add few lines from the old implementation for consistency * add some system tests * move system tests into separate class * fix credentials scope URL mistake * fix unit tests * fix algorithm name * add an example * add access token support * add credentials as an argument * rename method * add conformance tests into client unit tests * align conformance tests with test data * add an ability to set expiration as integer * update conformance tests to avoid problems with json spaces and timestamp Z-symbol violation * update implementation to avoid Z symbol isoformat violation and json whitespaces encoding * fix error with bounded hostnames * fix problem with bounded hostnames in implementation * fix conformance tests * fix problems: ascii encoding of signature and fields order * change asserts order * fix conformance tests * fix encoding issues * cosmetic changes and adding conformance tests * fix russion "C" letter in comment * add conformance tests data * cosmetic changes * cosmetic changes * add fields sorting * fix system tests Co-authored-by: Frank Natividad <frankyn@users.noreply.github.com>

Author: Gurov Ilya

google/cloud/storage/_signing.py

@@ -531,9 +531,7 @@ def generate_signed_url_v4(
  expiration_seconds = get_expiration_seconds_v4(expiration)
 
  if _request_timestamp is None:
- now = NOW()
- request_timestamp = now.strftime("%Y%m%dT%H%M%SZ")
- datestamp = now.date().strftime("%Y%m%d")
+ request_timestamp, datestamp = get_v4_now_dtstamps()
  else:
  request_timestamp = _request_timestamp
  datestamp = _request_timestamp[:8]
@@ -629,6 +627,18 @@ def generate_signed_url_v4(
  )
 
 
+def get_v4_now_dtstamps():
+ """Get current timestamp and datestamp in V4 valid format.
+
+ :rtype: str, str
+ :returns: Current timestamp, datestamp.
+ """
+ now = NOW()
+ timestamp = now.strftime("%Y%m%dT%H%M%SZ")
+ datestamp = now.date().strftime("%Y%m%d")
+ return timestamp, datestamp
+
+
 def _sign_message(message, access_token, service_account_email):
 
  """Signs a message.

google/cloud/storage/client.py

@@ -14,18 +14,28 @@
 
 """Client for interacting with the Google Cloud Storage API."""
 
-import warnings
+import base64
+import binascii
+import datetime
 import functools
+import json
+import warnings
 import google.api_core.client_options
 
 from google.auth.credentials import AnonymousCredentials
 
 from google.api_core import page_iterator
-from google.cloud._helpers import _LocalStack
+from google.cloud._helpers import _LocalStack, _NOW
 from google.cloud.client import ClientWithProject
 from google.cloud.exceptions import NotFound
 from google.cloud.storage._helpers import _get_storage_host
 from google.cloud.storage._http import Connection
+from google.cloud.storage._signing import (
+ get_expiration_seconds_v4,
+ get_v4_now_dtstamps,
+ ensure_signed_credentials,
+ _sign_message,
+)
 from google.cloud.storage.batch import Batch
 from google.cloud.storage.bucket import Bucket
 from google.cloud.storage.blob import Blob
@@ -836,6 +846,174 @@ def get_hmac_key_metadata(
  metadata.reload(timeout=timeout) # raises NotFound for missing key
  return metadata
 
+ def generate_signed_post_policy_v4(
+ self,
+ bucket_name,
+ blob_name,
+ expiration,
+ conditions=None,
+ fields=None,
+ credentials=None,
+ virtual_hosted_style=False,
+ bucket_bound_hostname=None,
+ scheme=None,
+ service_account_email=None,
+ access_token=None,
+ ):
+ """Generate a V4 signed policy object.
+
+ .. note::
+
+ Assumes ``credentials`` implements the
+ :class:`google.auth.credentials.Signing` interface. Also assumes
+ ``credentials`` has a ``service_account_email`` property which
+ identifies the credentials.
+
+ Generated policy object allows user to upload objects with a POST request.
+
+ :type bucket_name: str
+ :param bucket_name: Bucket name.
+
+ :type blob_name: str
+ :param blob_name: Object name.
+
+ :type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
+ :param expiration: Policy expiration time.
+
+ :type conditions: list
+ :param conditions: (Optional) List of POST policy conditions, which are
+ used to restrict what is allowed in the request.
+
+ :type fields: dict
+ :param fields: (Optional) Additional elements to include into request.
+
+ :type credentials: :class:`google.auth.credentials.Signing`
+ :param credentials: (Optional) Credentials object with an associated private
+ key to sign text.
+
+ :type virtual_hosted_style: bool
+ :param virtual_hosted_style: (Optional) If True, construct the URL relative to the bucket
+ virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
+
+ :type bucket_bound_hostname: str
+ :param bucket_bound_hostname:
+ (Optional) If passed, construct the URL relative to the bucket-bound hostname.
+ Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'.
+ See: https://cloud.google.com/storage/docs/request-endpoints#cname
+
+ :type scheme: str
+ :param scheme:
+ (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
+ this value as a scheme. ``https`` will work only when using a CDN.
+ Defaults to ``"http"``.
+
+ :type service_account_email: str
+ :param service_account_email: (Optional) E-mail address of the service account.
+
+ :type access_token: str
+ :param access_token: (Optional) Access token for a service account.
+
+ :rtype: dict
+ :returns: Signed POST policy.
+
+ Example:
+ Generate signed POST policy and upload a file.
+
+ >>> from google.cloud import storage
+ >>> client = storage.Client()
+ >>> policy = client.generate_signed_post_policy_v4(
+ "bucket-name",
+ "blob-name",
+ expiration=datetime.datetime(2020, 3, 17),
+ conditions=[
+ ["content-length-range", 0, 255]
+ ],
+ fields=[
+ "x-goog-meta-hello" => "world"
+ ],
+ )
+ >>> with open("bucket-name", "rb") as f:
+ files = {"file": ("bucket-name", f)}
+ requests.post(policy["url"], data=policy["fields"], files=files)
+ """
+ credentials = self._credentials if credentials is None else credentials
+ ensure_signed_credentials(credentials)
+
+ # prepare policy conditions and fields
+ timestamp, datestamp = get_v4_now_dtstamps()
+
+ x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
+ email=credentials.signer_email, datestamp=datestamp
+ )
+ required_conditions = [
+ {"key": blob_name},
+ {"x-goog-date": timestamp},
+ {"x-goog-credential": x_goog_credential},
+ {"x-goog-algorithm": "GOOG4-RSA-SHA256"},
+ ]
+
+ conditions = conditions or []
+ policy_fields = {}
+ for key, value in sorted((fields or {}).items()):
+ if not key.startswith("x-ignore-"):
+ policy_fields[key] = value
+ conditions.append({key: value})
+
+ conditions += required_conditions
+
+ # calculate policy expiration time
+ now = _NOW()
+ if expiration is None:
+ expiration = now + datetime.timedelta(hours=1)
+
+ policy_expires = now + datetime.timedelta(
+ seconds=get_expiration_seconds_v4(expiration)
+ )
+
+ # encode policy for signing
+ policy = json.dumps(
+ {"conditions": conditions, "expiration": policy_expires.isoformat() + "Z"},
+ separators=(",", ":"),
+ )
+ str_to_sign = base64.b64encode(policy.encode("utf-8"))
+
+ # sign the policy and get its cryptographic signature
+ if access_token and service_account_email:
+ signature = _sign_message(str_to_sign, access_token, service_account_email)
+ signature_bytes = base64.b64decode(signature)
+ else:
+ signature_bytes = credentials.sign_bytes(str_to_sign)
+
+ # get hexadecimal representation of the signature
+ signature = binascii.hexlify(signature_bytes).decode("utf-8")
+
+ policy_fields.update(
+ {
+ "key": blob_name,
+ "x-goog-algorithm": "GOOG4-RSA-SHA256",
+ "x-goog-credential": x_goog_credential,
+ "x-goog-date": timestamp,
+ "x-goog-signature": signature,
+ "policy": str_to_sign,
+ }
+ )
+ # designate URL
+ if virtual_hosted_style:
+ url = "https://{}.storage.googleapis.com/".format(bucket_name)
+
+ elif bucket_bound_hostname:
+ if ":" in bucket_bound_hostname: # URL includes scheme
+ url = bucket_bound_hostname
+
+ else: # scheme is given separately
+ url = "{scheme}://{host}/".format(
+ scheme=scheme, host=bucket_bound_hostname
+ )
+ else:
+ url = "https://storage.googleapis.com/{}/".format(bucket_name)
+
+ return {"url": url, "fields": policy_fields}
+
 
 def _item_to_bucket(iterator, item):
  """Convert a JSON bucket to the native object.

tests/system/test_system.py

@@ -1955,3 +1955,67 @@ def test_ubla_set_unset_preserves_acls(self):
 
  self.assertEqual(bucket_acl_before, bucket_acl_after)
  self.assertEqual(blob_acl_before, blob_acl_after)
+
+
+class TestV4POSTPolicies(unittest.TestCase):
+ def setUp(self):
+ self.case_buckets_to_delete = []
+
+ def tearDown(self):
+ for bucket_name in self.case_buckets_to_delete:
+ bucket = Config.CLIENT.bucket(bucket_name)
+ retry_429_harder(bucket.delete)(force=True)
+
+ def test_get_signed_policy_v4(self):
+ bucket_name = "post_policy" + unique_resource_id("-")
+ self.assertRaises(exceptions.NotFound, Config.CLIENT.get_bucket, bucket_name)
+ retry_429_503(Config.CLIENT.create_bucket)(bucket_name)
+ self.case_buckets_to_delete.append(bucket_name)
+
+ blob_name = "post_policy_obj.txt"
+ with open(blob_name, "w") as f:
+ f.write("DEADBEEF")
+
+ policy = Config.CLIENT.generate_signed_post_policy_v4(
+ bucket_name,
+ blob_name,
+ conditions=[
+ {"bucket": bucket_name},
+ ["starts-with", "$Content-Type", "text/pla"],
+ ],
+ expiration=datetime.datetime.now() + datetime.timedelta(hours=1),
+ fields={"content-type": "text/plain"},
+ )
+ with open(blob_name, "r") as f:
+ files = {"file": (blob_name, f)}
+ response = requests.post(policy["url"], data=policy["fields"], files=files)
+
+ os.remove(blob_name)
+ self.assertEqual(response.status_code, 204)
+
+ def test_get_signed_policy_v4_invalid_field(self):
+ bucket_name = "post_policy" + unique_resource_id("-")
+ self.assertRaises(exceptions.NotFound, Config.CLIENT.get_bucket, bucket_name)
+ retry_429_503(Config.CLIENT.create_bucket)(bucket_name)
+ self.case_buckets_to_delete.append(bucket_name)
+
+ blob_name = "post_policy_obj.txt"
+ with open(blob_name, "w") as f:
+ f.write("DEADBEEF")
+
+ policy = Config.CLIENT.generate_signed_post_policy_v4(
+ bucket_name,
+ blob_name,
+ conditions=[
+ {"bucket": bucket_name},
+ ["starts-with", "$Content-Type", "text/pla"],
+ ],
+ expiration=datetime.datetime.now() + datetime.timedelta(hours=1),
+ fields={"x-goog-random": "invalid_field", "content-type": "text/plain"},
+ )
+ with open(blob_name, "r") as f:
+ files = {"file": (blob_name, f)}
+ response = requests.post(policy["url"], data=policy["fields"], files=files)
+
+ os.remove(blob_name)
+ self.assertEqual(response.status_code, 400)

tests/unit/__init__.py

@@ -11,3 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import io
+import json
+import os
+
+
+def _read_local_json(json_file):
+ here = os.path.dirname(__file__)
+ json_path = os.path.abspath(os.path.join(here, json_file))
+ with io.open(json_path, "r", encoding="utf-8-sig") as fileobj:
+ return json.load(fileobj)

tests/unit/test__signing.py

@@ -18,9 +18,7 @@
 import binascii
 import calendar
 import datetime
-import io
 import json
-import os
 import time
 import unittest
 
@@ -29,12 +27,7 @@
 import six
 from six.moves import urllib_parse
 
-
-def _read_local_json(json_file):
- here = os.path.dirname(__file__)
- json_path = os.path.abspath(os.path.join(here, json_file))
- with io.open(json_path, "r", encoding="utf-8-sig") as fileobj:
- return json.load(fileobj)
+from . import _read_local_json
 
 
 _SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json")
@@ -762,6 +755,22 @@ def test_bytes(self):
  self.assertEqual(encoded_param, "bytes")
 
 
+class TestV4Stamps(unittest.TestCase):
+ def test_get_v4_now_dtstamps(self):
+ import datetime
+ from google.cloud.storage._signing import get_v4_now_dtstamps
+
+ with mock.patch(
+ "google.cloud.storage._signing.NOW",
+ return_value=datetime.datetime(2020, 3, 12, 13, 14, 15),
+ ) as now_mock:
+ timestamp, datestamp = get_v4_now_dtstamps()
+ now_mock.assert_called_once()
+
+ self.assertEqual(timestamp, "20200312T131415Z")
+ self.assertEqual(datestamp, "20200312")
+
+
 _DUMMY_SERVICE_ACCOUNT = None