Add auto_ecs_logging config (#1058)

* Implement auto_ecs_logging config and docs * Add a test for auto_ecs_logging * Changelog

Author: basepi

CHANGELOG.asciidoc

@@ -38,6 +38,7 @@ endif::[]
 ===== Features
 
 * Add global access to Client singleton object at `elasticapm.get_client()` {pull}1043[#1043]
+* Add `auto_ecs_logging` config option {pull}1058[#1058]
 
 
 [float]

docs/configuration.asciidoc

@@ -200,6 +200,9 @@ file will automatically rotate.
 Note that setting <<config-log_level>> is required for this setting to do
 anything.
 
+If https://github.com/elastic/ecs-logging-python[`ecs_logging`] is installed,
+the logs will automatically be formatted as ecs-compatible json.
+
 [float]
 [[config-log_file_size]]
 ==== `log_file_size`
@@ -215,6 +218,36 @@ The size of the log file, if <<config-log_file>> is set.
 The agent always keeps one backup file when rotating, so the max space that
 the log files will consume is twice the value of this setting.
 
+[float]
+[[config-auto_ecs_logging]]
+==== `auto_ecs_logging`
+
+[options="header"]
+|============
+| Environment | Django/Flask | Default
+| `ELASTIC_APM_AUTO_ECS_LOGGING` | `AUTO_ECS_LOGGING` | `False`
+|============
+
+If https://github.com/elastic/ecs-logging-python[`ecs_logging`] is installed,
+setting this to `True` will cause the agent to automatically attempt to enable
+ecs-formatted logging.
+
+For base `logging` from the standard library, the agent will get the root
+logger, find any attached handlers, and for each, set the formatter to
+`ecs_logging.StdlibFormatter()`.
+
+If `structlog` is installed, the agent will override any configured processors
+with `ecs_logging.StructlogFormatter()`.
+
+Note that this is a very blunt instrument that could have unintended side effects.
+If problems arise, please apply these formatters manually and leave this setting
+as `False`. See the
+https://www.elastic.co/guide/en/ecs-logging/python/current/installation.html[`ecs_logging` docs]
+for more information about using these formatters.
+
+Also note that this setting does not facilitate shipping logs to Elasticsearch.
+We recommend https://www.elastic.co/beats/filebeat[Filebeat] for that purpose.
+
 [float]
 [[other-options]]
 === Other options

elasticapm/conf/__init__.py

@@ -367,9 +367,43 @@ def _log_level_callback(dict_key, old_value, new_value, config_instance):
  filehandler = logging.handlers.RotatingFileHandler(
  config_instance.log_file, maxBytes=config_instance.log_file_size, backupCount=1
  )
+ try:
+ import ecs_logging
+
+ filehandler.setFormatter(ecs_logging.StdlibFormatter())
+ except ImportError:
+ pass
  elasticapm_logger.addHandler(filehandler)
 
 
+def _auto_ecs_logging_callback(dict_key, old_value, new_value, config_instance):
+ """
+ If ecs_logging is installed and auto_ecs_logging is set to True, we should
+ set the ecs_logging.StdlibFormatter as the formatted for every handler in
+ the root logger, and set the default processor for structlog to the
+ ecs_logging.StructlogFormatter.
+ """
+ if new_value:
+ try:
+ import ecs_logging
+ except ImportError:
+ return
+
+ # Stdlib
+ root_logger = logging.getLogger()
+ formatter = ecs_logging.StdlibFormatter()
+ for handler in root_logger.handlers:
+ handler.setFormatter(formatter)
+
+ # Structlog
+ try:
+ import structlog
+
+ structlog.configure(processors=[ecs_logging.StructlogFormatter()])
+ except ImportError:
+ pass
+
+
 class _ConfigBase(object):
  _NO_VALUE = object() # sentinel object
 
@@ -582,6 +616,7 @@ class Config(_ConfigBase):
  )
  log_file = _ConfigValue("LOG_FILE", default="")
  log_file_size = _ConfigValue("LOG_FILE_SIZE", validators=[size_validator], type=int, default=50 * 1024 * 1024)
+ auto_ecs_logging = _BoolConfigValue("AUTO_ECS_LOGGING", callbacks=[_auto_ecs_logging_callback], default=False)
 
  @property
  def is_recording(self):

tests/fixtures.py

@@ -202,6 +202,10 @@ def elasticapm_client_log_file(request):
  client_config.setdefault("cloud_provider", False)
  client_config.setdefault("log_level", "warning")
 
+ root_logger = logging.getLogger()
+ handler = logging.StreamHandler()
+ root_logger.addHandler(handler)
+
  tmp = tempfile.NamedTemporaryFile(delete=False)
  tmp.close()
  client_config["log_file"] = tmp.name
@@ -217,6 +221,9 @@ def elasticapm_client_log_file(request):
  handler.close()
  os.unlink(tmp.name)
 
+ # Remove our streamhandler
+ root_logger.removeHandler(handler)
+
  # clear any execution context that might linger around
  sys.excepthook = original_exceptionhook
  execution_context.set_transaction(None)

tests/handlers/logging/logging_tests.py

@@ -34,7 +34,9 @@
 import warnings
 from logging import LogRecord
 
+import ecs_logging
 import pytest
+import structlog
 
 from elasticapm.conf import Config
 from elasticapm.conf.constants import ERROR
@@ -394,3 +396,10 @@ def test_log_file(elasticapm_client_log_file):
  if isinstance(handler, logging.handlers.RotatingFileHandler):
  found = True
  assert found
+
+
+@pytest.mark.parametrize("elasticapm_client_log_file", [{"auto_ecs_logging": True}], indirect=True)
+def test_auto_ecs_logging(elasticapm_client_log_file):
+ logger = logging.getLogger()
+ assert isinstance(logger.handlers[0].formatter, ecs_logging.StdlibFormatter)
+ assert isinstance(structlog.get_config()["processors"][-1], ecs_logging.StructlogFormatter)

tests/requirements/reqs-base.txt

@@ -46,6 +46,8 @@ mock
 msgpack-python
 pep8
 pytz
+ecs_logging
+structlog
 
 
 pytest-asyncio==0.14.0 ; python_version >= '3.7'