feat: track invocation id for retry metrics (#741)

* feat: track invocation id for retry metrics * woops, fix the context * slight adjustment to stringify * old tests pass * lint * adjust based on new python-cloud-core changes * updated cloud core, and all working * test that invocation id changes between calls * lint and fix test name Co-authored-by: Anthonios Partheniou <partheniou@google.com>

Author: Aaron Gabriel Neyer

google/cloud/storage/_helpers.py

@@ -21,6 +21,7 @@
 from hashlib import md5
 import os
 from urllib.parse import urlsplit
+from uuid import uuid4
 
 from google import resumable_media
 from google.auth import environment_vars
@@ -584,6 +585,10 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
  return resumable_media.RetryStrategy(max_retries=0)
 
 
+def _get_invocation_id():
+ return "gccl-invocation-id/" + str(uuid4())
+
+
 def _get_default_headers(
  user_agent,
  content_type="application/json; charset=UTF-8",
@@ -600,7 +605,7 @@ def _get_default_headers(
  "Accept": "application/json",
  "Accept-Encoding": "gzip, deflate",
  "User-Agent": user_agent,
- "x-goog-api-client": user_agent,
+ "X-Goog-API-Client": f"{user_agent} {_get_invocation_id()}",
  "content-type": content_type,
  "x-upload-content-type": x_upload_content_type or content_type,
  }

google/cloud/storage/_http.py

@@ -18,6 +18,7 @@
 
 from google.cloud import _http
 from google.cloud.storage import __version__
+from google.cloud.storage import _helpers
 
 
 class Connection(_http.JSONConnection):
@@ -59,6 +60,7 @@ def __init__(self, client, client_info=None, api_endpoint=None):
 
  def api_request(self, *args, **kwargs):
  retry = kwargs.pop("retry", None)
+ kwargs["extra_api_info"] = _helpers._get_invocation_id()
  call = functools.partial(super(Connection, self).api_request, *args, **kwargs)
  if retry:
  # If this is a ConditionalRetryPolicy, check conditions.

setup.py

@@ -30,7 +30,7 @@
 dependencies = [
  "google-auth >= 1.25.0, < 3.0dev",
  "google-api-core >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0",
- "google-cloud-core >= 1.6.0, < 3.0dev",
+ "google-cloud-core >= 2.3.0, < 3.0dev",
  "google-resumable-media >= 2.3.2",
  "requests >= 2.18.0, < 3.0.0dev",
  "protobuf",

tests/unit/test__helpers.py

@@ -19,6 +19,8 @@
 from google.cloud.storage.retry import DEFAULT_RETRY
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
 
+GCCL_INVOCATION_TEST_CONST = "gccl-invocation-id/test-invocation-123"
+
 
 class Test__get_storage_host(unittest.TestCase):
  @staticmethod

tests/unit/test__http.py

@@ -13,9 +13,13 @@
 # limitations under the License.
 
 import unittest
+from unittest.mock import patch
 
 import mock
 
+from google.cloud.storage import _helpers
+from tests.unit.test__helpers import GCCL_INVOCATION_TEST_CONST
+
 
 class TestConnection(unittest.TestCase):
  @staticmethod
@@ -44,12 +48,17 @@ def test_extra_headers(self):
 
  conn = self._make_one(client)
  req_data = "hey-yoooouuuuu-guuuuuyyssss"
- result = conn.api_request("GET", "/rainbow", data=req_data, expect_json=False)
+ with patch.object(
+ _helpers, "_get_invocation_id", return_value=GCCL_INVOCATION_TEST_CONST
+ ):
+ result = conn.api_request(
+ "GET", "/rainbow", data=req_data, expect_json=False
+ )
  self.assertEqual(result, data)
 
  expected_headers = {
  "Accept-Encoding": "gzip",
- base_http.CLIENT_INFO_HEADER: conn.user_agent,
+ base_http.CLIENT_INFO_HEADER: f"{conn.user_agent} {GCCL_INVOCATION_TEST_CONST}",
  "User-Agent": conn.user_agent,
  }
  expected_uri = conn.build_api_url("/rainbow")