Merge pull request DataDog#174 from DataDog/gaetan.deputier/disable-ssl-validation-lambda

[Lambda]: Add DD_SKIP_SSL_VALIDATION option to disable hostname validation for HTTP forwarding

Author: gaetan-deputier

aws/logs_monitoring/README.md

@@ -181,3 +181,8 @@ If there are multiline logs in s3, set `DD_MULTILINE_LOG_REGEX_PATTERN` environm
 ### 10. (optional) Disable log forwarding
 
 The datadog forwarder **ALWAYS** forwards logs by default. If you do NOT use the Datadog log management product, you **MUST** set environment variable `DD_FORWARD_LOG` to `False`, to avoid sending logs to Datadog. The forwarder will then only forward other observability data, such as metrics.
+
+### 11. (optional) Disable SSL validation
+
+If you need to ignore SSL certificate validation when forwarding logs using HTTPS, you can set the environment variable `DD_SKIP_SSL_VALIDATION` to `True`.
+This will still encrypt the traffic between the forwarder and the endpoint provided with `DD_URL` but will not check if the destination SSL certificate is valid. 

aws/logs_monitoring/lambda_function.py

@@ -111,6 +111,11 @@ def get_bool_env_var(envvar, default):
 #
 DD_NO_SSL = get_bool_env_var("DD_NO_SSL", "false")
 
+## @param DD_SKIP_SSL_VALIDATION - boolean - optional -default: false
+## Disable SSL certificate validation when forwarding logs via HTTP.
+#
+DD_SKIP_SSL_VALIDATION = get_bool_env_var("DD_SKIP_SSL_VALIDATION", "false")
+
 ## @param DD_SITE - String - optional -default: datadoghq.com
 ## Define the Datadog Site to send your logs and metrics to.
 ## Set it to `datadoghq.eu` to send your logs and metrics to Datadog EU site.
@@ -250,7 +255,7 @@ def compileRegex(rule, pattern):
 DD_CUSTOM_TAGS = "ddtags"
 DD_SERVICE = "service"
 DD_HOST = "host"
-DD_FORWARDER_VERSION = "2.3.1"
+DD_FORWARDER_VERSION = "2.3.2"
 
 class RetriableException(Exception):
  pass
@@ -346,12 +351,13 @@ class DatadogHTTPClient(object):
  _POST = "POST"
  _HEADERS = {"Content-type": "application/json"}
 
- def __init__(self, host, port, no_ssl, api_key, scrubber, timeout=10):
+ def __init__(self, host, port, no_ssl, skip_ssl_validation, api_key, scrubber, timeout=10):
  protocol = "http" if no_ssl else "https"
  self._url = "{}://{}:{}/v1/input/{}".format(protocol, host, port, api_key)
  self._scrubber = scrubber
  self._timeout = timeout
  self._session = None
+ self._ssl_validation = not skip_ssl_validation
 
  def _connect(self):
  self._session = requests.Session()
@@ -369,6 +375,7 @@ def send(self, logs):
  self._url,
  data=self._scrubber.scrub("[{}]".format(",".join(logs))),
  timeout=self._timeout,
+ verify=self._ssl_validation
  )
  except ScrubbingException:
  raise Exception("could not scrub the payload")
@@ -500,7 +507,7 @@ def forward_logs(logs):
  cli = DatadogTCPClient(DD_URL, DD_PORT, DD_NO_SSL, DD_API_KEY, scrubber)
  else:
  batcher = DatadogBatcher(256 * 1000, 2 * 1000 * 1000, 200)
- cli = DatadogHTTPClient(DD_URL, DD_PORT, DD_NO_SSL, DD_API_KEY, scrubber)
+ cli = DatadogHTTPClient(DD_URL, DD_PORT, DD_NO_SSL, DD_SKIP_SSL_VALIDATION, DD_API_KEY, scrubber)
 
  with DatadogClient(cli) as client:
  for batch in batcher.batch(logs):