feat: allow custom labels with standard library logging (#264)

Author: daniel-sanche

google/cloud/logging_v2/client.py

@@ -362,9 +362,9 @@ def get_default_handler(self, **kw):
  ):
  # Cloud Functions with runtimes > 3.8 supports structured logs on standard out
  # 3.7 should use the standard CloudLoggingHandler, which sends logs over the network.
- return StructuredLogHandler(**kw, project=self.project)
+ return StructuredLogHandler(**kw, project_id=self.project)
  elif monitored_resource.type == _RUN_RESOURCE_TYPE:
- return StructuredLogHandler(**kw, project=self.project)
+ return StructuredLogHandler(**kw, project_id=self.project)
  return CloudLoggingHandler(self, resource=monitored_resource, **kw)
 
  def setup_logging(

google/cloud/logging_v2/handlers/handlers.py

@@ -37,8 +37,9 @@ class CloudLoggingFilter(logging.Filter):
  the `extras` argument when writing logs.
  """
 
- def __init__(self, project=None):
+ def __init__(self, project=None, default_labels=None):
  self.project = project
+ self.default_labels = default_labels if default_labels else {}
 
  def filter(self, record):
  # ensure record has all required fields set
@@ -61,6 +62,12 @@ def filter(self, record):
  inferred_http, inferred_trace = get_request_data()
  if inferred_trace is not None and self.project is not None:
  inferred_trace = f"projects/{self.project}/traces/{inferred_trace}"
+ # set labels
+ user_labels = getattr(record, "labels", {})
+ record.total_labels = {**self.default_labels, **user_labels}
+ record.total_labels_str = ", ".join(
+ [f'"{k}": "{v}"' for k, v in record.total_labels.items()]
+ )
 
  record.trace = getattr(record, "trace", inferred_trace) or ""
  record.http_request = getattr(record, "http_request", inferred_http) or {}
@@ -126,8 +133,7 @@ def __init__(
  option is :class:`.SyncTransport`.
  resource (~logging_v2.resource.Resource):
  Resource for this Handler. Defaults to ``global``.
- labels (Optional[dict]): Monitored resource of the entry, defaults
- to the global resource type.
+ labels (Optional[dict]): Additional labels to attach to logs.
  stream (Optional[IO]): Stream to be used by the handler.
  """
  super(CloudLoggingHandler, self).__init__(stream)
@@ -138,7 +144,8 @@ def __init__(
  self.resource = resource
  self.labels = labels
  # add extra keys to log record
- self.addFilter(CloudLoggingFilter(self.project_id))
+ log_filter = CloudLoggingFilter(project=self.project_id, default_labels=labels)
+ self.addFilter(log_filter)
 
  def emit(self, record):
  """Actually log the specified logging record.
@@ -151,22 +158,16 @@ def emit(self, record):
  record (logging.LogRecord): The record to be logged.
  """
  message = super(CloudLoggingHandler, self).format(record)
- user_labels = getattr(record, "labels", {})
- # merge labels
- total_labels = self.labels if self.labels is not None else {}
- total_labels.update(user_labels)
- if len(total_labels) == 0:
- total_labels = None
  # send off request
  self.transport.send(
  record,
  message,
  resource=getattr(record, "resource", self.resource),
- labels=total_labels,
- trace=getattr(record, "trace", None),
- span_id=getattr(record, "span_id", None),
- http_request=getattr(record, "http_request", None),
- source_location=getattr(record, "source_location", None),
+ labels=getattr(record, "total_labels", None) or None,
+ trace=getattr(record, "trace", None) or None,
+ span_id=getattr(record, "span_id", None) or None,
+ http_request=getattr(record, "http_request", None) or None,
+ source_location=getattr(record, "source_location", None) or None,
  )
 
 

google/cloud/logging_v2/handlers/structured_log.py

@@ -19,26 +19,34 @@
 
 from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter
 
-GCP_FORMAT = '{"message": "%(message)s", "severity": "%(levelname)s", "logging.googleapis.com/trace": "%(trace)s", "logging.googleapis.com/sourceLocation": { "file": "%(file)s", "line": "%(line)d", "function": "%(function)s"}, "httpRequest": {"requestMethod": "%(request_method)s", "requestUrl": "%(request_url)s", "userAgent": "%(user_agent)s", "protocol": "%(protocol)s"} }'
+GCP_FORMAT = (
+ '{"message": "%(message)s", '
+ '"severity": "%(levelname)s", '
+ '"logging.googleapis.com/labels": { %(total_labels_str)s }, '
+ '"logging.googleapis.com/trace": "%(trace)s", '
+ '"logging.googleapis.com/sourceLocation": { "file": "%(file)s", "line": "%(line)d", "function": "%(function)s"}, '
+ '"httpRequest": {"requestMethod": "%(request_method)s", "requestUrl": "%(request_url)s", "userAgent": "%(user_agent)s", "protocol": "%(protocol)s"} }'
+)
 
 
 class StructuredLogHandler(logging.StreamHandler):
  """Handler to format logs into the Cloud Logging structured log format,
  and write them to standard output
  """
 
- def __init__(self, *, name=None, stream=None, project=None):
+ def __init__(self, *, labels=None, stream=None, project_id=None):
  """
  Args:
- name (Optional[str]): The name of the custom log in Cloud Logging.
+ labels (Optional[dict]): Additional labels to attach to logs.
  stream (Optional[IO]): Stream to be used by the handler.
+ project (Optional[str]): Project Id associated with the logs.
  """
  super(StructuredLogHandler, self).__init__(stream=stream)
- self.name = name
- self.project_id = project
+ self.project_id = project_id
 
  # add extra keys to log record
- self.addFilter(CloudLoggingFilter(project))
+ log_filter = CloudLoggingFilter(project=project_id, default_labels=labels)
+ self.addFilter(log_filter)
 
  # make logs appear in GCP structured logging format
  self.formatter = logging.Formatter(GCP_FORMAT)

tests/environment

@@ -268,22 +268,27 @@ def test_emit(self):
  )
  logname = "loggername"
  message = "hello world"
- labels = {"test-key": "test-value"}
  record = logging.LogRecord(logname, logging, None, None, message, None, None)
- record.labels = labels
- handler.emit(record)
+ handler.handle(record)
  self.assertEqual(
  handler.transport.send_called_with,
- (record, message, _GLOBAL_RESOURCE, labels, None, None, None, None),
+ (record, message, _GLOBAL_RESOURCE, None, None, None, None, None),
  )
 
  def test_emit_manual_field_override(self):
  from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE
  from google.cloud.logging_v2.resource import Resource
 
  client = _Client(self.PROJECT)
+ default_labels = {
+ "default_key": "default-value",
+ "overwritten_key": "bad_value",
+ }
  handler = self._make_one(
- client, transport=_Transport, resource=_GLOBAL_RESOURCE
+ client,
+ transport=_Transport,
+ resource=_GLOBAL_RESOURCE,
+ labels=default_labels,
  )
  logname = "loggername"
  message = "hello world"
@@ -299,9 +304,14 @@ def test_emit_manual_field_override(self):
  setattr(record, "source_location", expected_source)
  expected_resource = Resource(type="test", labels={})
  setattr(record, "resource", expected_resource)
- expected_labels = {"test-label": "manual"}
- setattr(record, "labels", expected_labels)
- handler.emit(record)
+ added_labels = {"added_key": "added_value", "overwritten_key": "new_value"}
+ expected_labels = {
+ "default_key": "default-value",
+ "overwritten_key": "new_value",
+ "added_key": "added_value",
+ }
+ setattr(record, "labels", added_labels)
+ handler.handle(record)
 
  self.assertEqual(
  handler.transport.send_called_with,

tests/unit/handlers/test_handlers.py

@@ -40,17 +40,18 @@ def index():
 
  def test_ctor_defaults(self):
  handler = self._make_one()
- self.assertIsNone(handler.name)
+ self.assertIsNone(handler.project_id)
 
- def test_ctor_w_name(self):
- handler = self._make_one(name="foo")
- self.assertEqual(handler.name, "foo")
+ def test_ctor_w_project(self):
+ handler = self._make_one(project_id="foo")
+ self.assertEqual(handler.project_id, "foo")
 
  def test_format(self):
  import logging
  import json
 
- handler = self._make_one()
+ labels = {"default_key": "default-value"}
+ handler = self._make_one(labels=labels)
  logname = "loggername"
  message = "hello world，嗨 世界"
  pathname = "testpath"
@@ -74,6 +75,7 @@ def test_format(self):
  "userAgent": "",
  "protocol": "",
  },
+ "logging.googleapis.com/labels": labels,
  }
  handler.filter(record)
  result = json.loads(handler.format(record))
@@ -106,6 +108,7 @@ def test_format_minimal(self):
  "userAgent": "",
  "protocol": "",
  },
+ "logging.googleapis.com/labels": {},
  }
  handler.filter(record)
  result = json.loads(handler.format(record))
@@ -160,7 +163,11 @@ def test_format_overrides(self):
  import logging
  import json
 
- handler = self._make_one()
+ default_labels = {
+ "default_key": "default-value",
+ "overwritten_key": "bad_value",
+ }
+ handler = self._make_one(labels=default_labels)
  logname = "loggername"
  message = "hello world，嗨 世界"
  record = logging.LogRecord(logname, logging.INFO, "", 0, message, None, None)
@@ -172,6 +179,8 @@ def test_format_overrides(self):
  record.http_request = {"requestUrl": overwrite_path}
  record.source_location = {"file": overwrite_file}
  record.trace = overwrite_trace
+ added_labels = {"added_key": "added_value", "overwritten_key": "new_value"}
+ record.labels = added_labels
  expected_payload = {
  "logging.googleapis.com/trace": overwrite_trace,
  "logging.googleapis.com/sourceLocation": {
@@ -185,6 +194,11 @@ def test_format_overrides(self):
  "userAgent": "",
  "protocol": "",
  },
+ "logging.googleapis.com/labels": {
+ "default_key": "default-value",
+ "overwritten_key": "new_value",
+ "added_key": "added_value",
+ },
  }
 
  app = self.create_app()