Create internals for testing IPv6 network between lib and agent

.github/workflows/run-end-to-end.yml

@@ -100,6 +100,7 @@ jobs:
  - name: Build weblog
  id: build
  run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog }}
+
  - name: Run APPSEC_STANDALONE scenario
  if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE"')
  run: ./run.sh APPSEC_STANDALONE
@@ -115,6 +116,9 @@ jobs:
  - name: Run DEFAULT scenario
  if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEFAULT"')
  run: ./run.sh DEFAULT
+ - name: Run IPV6 scenario
+ if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"IPV6"') && inputs.library != 'ruby'
+ run: ./run.sh IPV6
  - name: Run CROSSED_TRACING_LIBRARIES scenario
  if: always() && steps.build.outcome == 'success' && matrix.weblog != 'python3.12' && contains(inputs.scenarios, '"CROSSED_TRACING_LIBRARIES"')
  run: ./run.sh CROSSED_TRACING_LIBRARIES

.vscode/launch.json

@@ -144,6 +144,16 @@
  "justMyCode": true,
  "python": "${workspaceFolder}/venv/bin/python"
  },
+ {
+ "name": "Run IPV6 scenario",
+ "type": "python",
+ "request": "launch",
+ "module": "pytest",
+ "args": ["-S", "IPV6", "-p", "no:warnings"],
+ "console": "integratedTerminal",
+ "justMyCode": true,
+ "python": "${workspaceFolder}/venv/bin/python"
+ },
  {
  "name": "Python: Current File",
  "type": "python",

docs/scenarios/IPv6.md

@@ -0,0 +1,5 @@
+The `IPV6` scenario setup an IPv6 docker network and use an IPv6 address as DD_AGENT_HOST to verify that the library is able to communicate to the agent using an IPv6 address. It does not use a proxy between the lib and the agent to not interfer at any point here, so all assertions must be done on the outgoing traffic from the agent.
+
+Please note that it requires the docker daemon to supports IPv6. It should be ok on Linux CI and Mac OS, but has not be tested on Windows.
+
+A user has seen his network function altered after running it on a linux latptop (to be investigated). If it happen, `docker network prune` may sole the issue.

manifests/cpp.yml

@@ -252,6 +252,7 @@ tests/:
  Test_Span_Links_From_Conflicting_Contexts_Datadog_Precedence: missing_feature
  Test_Span_Links_Omit_Tracestate_From_Conflicting_Contexts: missing_feature (implementation specs have not been determined)
  test_identify.py: irrelevant
+ test_ipv6.py: missing_feature (APMAPI-869)
  test_library_conf.py: irrelevant
  test_miscs.py: irrelevant
  test_scrubbing.py: irrelevant

manifests/dotnet.yml

@@ -503,6 +503,7 @@ tests/:
  Test_Basic: v2.7.0
  Test_Propagate: v2.27.0
  Test_Propagate_Legacy: v2.26.0
+ test_ipv6.py: missing_feature (APMAPI-869)
  test_library_conf.py:
  Test_HeaderTags: v2.27.0
  Test_HeaderTags_Colon_Leading: v2.1.0

manifests/java.yml

@@ -1594,6 +1594,7 @@ tests/:
  Test_Basic: missing_feature
  Test_Propagate: missing_feature
  Test_Propagate_Legacy: missing_feature
+ test_ipv6.py: missing_feature (APMAPI-869)
  test_library_conf.py:
  Test_HeaderTags: missing_feature
  Test_HeaderTags_Colon_Leading: v0.102.0

manifests/ruby.yml

@@ -502,6 +502,7 @@ tests/:
  Test_Basic: v1.0.0
  Test_Propagate: missing_feature
  Test_Propagate_Legacy: missing_feature
+ test_ipv6.py: missing_feature (APMAPI-869)
  test_library_conf.py:
  Test_HeaderTags: v1.13.0
  Test_HeaderTags_Colon_Leading: v1.13.0

run.sh

@@ -139,13 +139,12 @@ function activate_venv() {
  source venv/bin/activate
 }
 
-function network_name() {
- perl -ne '/_DEFAULT_NETWORK_NAME = "(.*)"/ and print "$1\n"' utils/_context/containers.py
-}
-
 function ensure_network() {
  local network_name
- network_name="$(network_name)"
+
+ # limited support of docker mode: it can't control test targets, so going for the most common use case
+ # reminder : this mode is unofficial and not supported (for the exact reason it can't control test targets...)
+ network_name="system-tests-ipv4"
 
  if docker network ls | grep -q "${network_name}"; then
  : # network exists
@@ -184,7 +183,7 @@ function run_scenario() {
 
  cmd+=(
  docker run
- --network system-tests_default
+ --network system-tests-ipv4
  --rm -i
  )
  if [ -t 1 ]; then

tests/test_ipv6.py

@@ -0,0 +1,11 @@
+from utils import scenarios, weblog, interfaces, features
+
+
+@features.agent_host_ipv6
+@scenarios.ipv6
+class Test_Basic:
+ def setup_main(self):
+ self.r = weblog.get("/")
+
+ def test_main(self):
+ interfaces.agent.assert_trace_exists(self.r)

utils/_context/_scenarios/core.py

@@ -16,6 +16,7 @@ class ScenarioGroup(Enum):
  END_TO_END = "end-to-end"
  GRAPHQL = "graphql"
  INTEGRATIONS = "integrations"
+ IPV6 = "ipv6"
  LIB_INJECTION = "lib-injection"
  OPEN_TELEMETRY = "open-telemetry"
  PARAMETRIC = "parametric"

utils/_context/_scenarios/endtoend.py

@@ -1,7 +1,9 @@
 import os
+import sys
 import pytest
 
 from docker.models.networks import Network
+from docker.types import IPAMConfig, IPAMPool
 
 from watchdog.observers.polling import PollingObserver
 from watchdog.events import FileSystemEventHandler
@@ -42,6 +44,7 @@ def __init__(
  github_workflow,
  doc,
  scenario_groups=None,
+ enable_ipv6: bool = False,
  use_proxy=True,
  rc_api_enabled=False,
  meta_structs_disabled=False,
@@ -57,6 +60,7 @@ def __init__(
  super().__init__(name, doc=doc, github_workflow=github_workflow, scenario_groups=scenario_groups)
 
  self.use_proxy = use_proxy
+ self.enable_ipv6 = enable_ipv6
  self.rc_api_enabled = rc_api_enabled
  self.meta_structs_disabled = False
  self.span_events = span_events
@@ -73,6 +77,7 @@ def __init__(
  rc_api_enabled=rc_api_enabled,
  meta_structs_disabled=meta_structs_disabled,
  span_events=span_events,
+ enable_ipv6=enable_ipv6,
  )
 
  self._required_containers.append(self.proxy_container)
@@ -142,14 +147,28 @@ def _ingest(self, event):
  observer.start()
 
  def _create_network(self) -> None:
- name = "system-tests_default"
+ name = "system-tests-ipv6" if self.enable_ipv6 else "system-tests-ipv4"
 
  for network in get_docker_client().networks.list(names=[name]):
- logger.debug(f"Network {name} still exists")
  self._network = network
+ logger.debug(f"Network {name} still exists")
  return
 
- self._network = get_docker_client().networks.create(name, check_duplicate=True)
+ logger.debug(f"Create network {name}")
+
+ if self.enable_ipv6:
+ self._network = get_docker_client().networks.create(
+ name=name,
+ driver="bridge",
+ enable_ipv6=True,
+ ipam=IPAMConfig(
+ driver="default",
+ pool_configs=[IPAMPool(subnet="2001:db8:1::/64")],
+ ),
+ )
+ assert self._network.attrs["EnableIPv6"] is True, self._network.attrs
+ else:
+ self._network = get_docker_client().networks.create(name, check_duplicate=True)
 
  def get_warmups(self):
  warmups = super().get_warmups()
@@ -198,13 +217,15 @@ def __init__(
  weblog_env=None,
  weblog_volumes=None,
  agent_env=None,
+ enable_ipv6: bool = False,
  tracer_sampling_rate=None,
  appsec_enabled=True,
  iast_enabled=True,
  additional_trace_header_tags=(),
  library_interface_timeout=None,
  agent_interface_timeout=5,
- use_proxy=True,
+ use_proxy_for_weblog: bool = True,
+ use_proxy_for_agent: bool = True,
  rc_api_enabled=False,
  meta_structs_disabled=False,
  span_events=True,
@@ -229,7 +250,8 @@ def __init__(
  doc=doc,
  github_workflow=github_workflow,
  scenario_groups=scenario_groups,
- use_proxy=use_proxy,
+ enable_ipv6=enable_ipv6,
+ use_proxy=use_proxy_for_agent or use_proxy_for_weblog,
  rc_api_enabled=rc_api_enabled,
  meta_structs_disabled=meta_structs_disabled,
  span_events=span_events,
@@ -242,13 +264,16 @@ def __init__(
  include_sqlserver=include_sqlserver,
  )
 
+ self._use_proxy_for_agent = use_proxy_for_agent
+ self._use_proxy_for_weblog = use_proxy_for_weblog
+
  self._require_api_key = require_api_key
 
  self.agent_container = AgentContainer(
- host_log_folder=self.host_log_folder, use_proxy=use_proxy, environment=agent_env
+ host_log_folder=self.host_log_folder, use_proxy=use_proxy_for_agent, environment=agent_env
  )
 
- if self.use_proxy:
+ if use_proxy_for_agent:
  self.agent_container.depends_on.append(self.proxy_container)
 
  weblog_env = dict(weblog_env) if weblog_env else {}
@@ -272,7 +297,7 @@ def __init__(
  appsec_enabled=appsec_enabled,
  iast_enabled=iast_enabled,
  additional_trace_header_tags=additional_trace_header_tags,
- use_proxy=use_proxy,
+ use_proxy=use_proxy_for_weblog,
  volumes=weblog_volumes,
  )
 
@@ -389,18 +414,36 @@ def _start_interface_watchdog(self):
  [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies]
  )
 
+ def _set_weblog_domain(self):
+ if self.enable_ipv6:
+ from utils import weblog # TODO better interface
+
+ if sys.platform == "linux":
+ # on Linux, with ipv6 mode, we can't use localhost anymore for a reason I ignore
+ # To fix, we use the container ipv4 address as weblog doamin, as it's accessible from host
+ weblog.domain = self.weblog_container.network_ip(self._network)
+ logger.info(f"Linux => Using Container IPv6 address [{weblog.domain}] as weblog domain")
+
+ elif sys.platform == "darwin":
+ # on Mac, this ipv4 address can't be used, because container IP are not accessible from host
+ # as they are on an network intermal to the docker VM. But we can still use localhost.
+ logger.info("Mac => Using localhost as weblog domain")
+ else:
+ pytest.exit(f"Unsupported platform {sys.platform} with ipv6 enabled", 1)
+
  def get_warmups(self):
  warmups = super().get_warmups()
 
  if not self.replay:
  warmups.insert(1, self._start_interface_watchdog)
  warmups.append(self._get_weblog_system_info)
  warmups.append(self._wait_for_app_readiness)
+ warmups.append(self._set_weblog_domain)
 
  return warmups
 
  def _wait_for_app_readiness(self):
- if self.use_proxy:
+ if self._use_proxy_for_weblog:
  logger.debug("Wait for app readiness")
 
  if not interfaces.library.ready.wait(40):
@@ -414,6 +457,7 @@ def _wait_for_app_readiness(self):
 
  logger.debug(f"{container.name} ready")
 
+ if self._use_proxy_for_agent:
  if not interfaces.agent.ready.wait(40):
  raise Exception("Datadog agent not ready")
  logger.debug("Agent ready")
@@ -443,13 +487,15 @@ def _wait_and_stop_containers(self):
 
  interfaces.backend.load_data_from_logs()
 
- elif self.use_proxy:
+ else:
  self._wait_interface(interfaces.library, self.library_interface_timeout)
 
  if self.library in ("nodejs",):
+ from utils import weblog # TODO better interface
+
  # for weblogs who supports it, call the flush endpoint
  try:
- r = self.weblog_container.request("GET", "/flush", timeout=10)
+ r = weblog.get("/flush", timeout=10)
  assert r.status_code == 200
  except:
  self.weblog_container.healthy = False

utils/_context/_scenarios/ipv6.py

@@ -1,6 +1,30 @@
+from utils.tools import logger
+
+from .core import ScenarioGroup
 from .endtoend import EndToEndScenario
 
 
 class IPV6Scenario(EndToEndScenario):
- def __init__(self, name):
- super().__init__(name, doc="Library use a DD_AGENT_HOST using an IPv6 address")
+ def __init__(self, name) -> None:
+ super().__init__(
+ name,
+ enable_ipv6=True,
+ scenario_groups=[ScenarioGroup.IPV6],
+ use_proxy_for_agent=True,
+ use_proxy_for_weblog=False,
+ doc=(
+ "Test the agent/lib communication using an IPv6 address. We do not use the proxy between "
+ "the lib and the agent to check that the agent correctly accept IPv6 traffic"
+ ),
+ )
+
+ def _start_containers(self):
+ self.proxy_container.start(self._network)
+ self.agent_container.start(self._network)
+
+ agent_container_ip = self.agent_container.network_ipv6(self._network)
+ logger.stdout(f"Lib is configured with DD_AGENT_HOST: {agent_container_ip}")
+
+ self.weblog_container.environment["DD_AGENT_HOST"] = agent_container_ip
+
+ super()._start_containers()

utils/_context/_scenarios/performance.py

@@ -6,7 +6,9 @@
 
 class PerformanceScenario(EndToEndScenario):
  def __init__(self, name, doc) -> None:
- super().__init__(name, doc=doc, appsec_enabled=self.appsec_enabled, use_proxy=False)
+ super().__init__(
+ name, doc=doc, appsec_enabled=self.appsec_enabled, use_proxy_for_agent=False, use_proxy_for_weblog=False
+ )
 
  @property
  def appsec_enabled(self):