cisco_meraki: improve handling of flows events (#4352)

Simplify the handling of the line by using grok from the outset, reducing the number of machines needing to be instantiated and increasing the flexibility of the inputs that can be parsed.

Author: efd6

packages/cisco_meraki/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.3"
+ changes:
+ - description: Improve handling of flows events.
+ type: bugfix
+ link: https://github.com/elastic/integrations/issues/4352
 - version: "1.2.2"
  changes:
  - description: Remove duplicate fields.

packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-flows.log

@@ -1,3 +1,6 @@
 <134>1 1647478988.289402144 MX84_4 flows allow src=10.0.2.170 dst=10.0.0.34 mac=00:7C:2D:BD:76:F2 protocol=udp sport=54841 dport=15600
 <134>1 1647478988.476061795 MX84 flows src=216.160.83.57 dst=216.160.83.61 protocol=tcp sport=54445 dport=44210 pattern: 1 all
 <134>1 1647478988.596151424 MX84_7 flows allow src=10.0.0.34 dst=10.0.0.234 mac=64:1C:B0:BA:F0:EC protocol=tcp sport=49761 dport=15500
+<134>1 1664382879.496990921 AP_XXXX flows allow src=fe80::1021:83ca:b68:4cd8 dst=ff02::1:ffb6:a227 mac=28:FF:3C:AB:DB:AA protocol=icmp6 type=135
+<134>1 1664385452.707589827 AP_XXXX flows allow src=172.16.12.23 dst=224.0.0.2 mac=4C:AB:4F:0D:3D:AA protocol=2
+<134>1 1664385453.129104346 AP_XXXX flows allow src=172.16.10.14 dst=81.2.69.144 mac=EC:63:D7:0F:6B:AA protocol=icmp type=8

packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-flows.log-expected.json

@@ -157,6 +157,144 @@
  "forwarded",
  "preserve_original_event"
  ]
+ },
+ {
+ "@timestamp": "2022-09-28T16:34:39.496Z",
+ "cisco_meraki": {
+ "event_subtype": "flow_allowed",
+ "event_type": "flows",
+ "flows": {
+ "op": "allow"
+ }
+ },
+ "destination": {
+ "ip": "ff02::1:ffb6:a227"
+ },
+ "ecs": {
+ "version": "8.4.0"
+ },
+ "event": {
+ "action": "layer3-firewall-allowed-flow",
+ "category": [
+ "network"
+ ],
+ "original": "\u003c134\u003e1 1664382879.496990921 AP_XXXX flows allow src=fe80::1021:83ca:b68:4cd8 dst=ff02::1:ffb6:a227 mac=28:FF:3C:AB:DB:AA protocol=icmp6 type=135",
+ "type": [
+ "info",
+ "connection",
+ "start"
+ ]
+ },
+ "network": {
+ "protocol": "icmp6"
+ },
+ "observer": {
+ "hostname": "AP_XXXX"
+ },
+ "source": {
+ "ip": "fe80::1021:83ca:b68:4cd8",
+ "mac": "28-FF-3C-AB-DB-AA"
+ },
+ "tags": [
+ "forwarded",
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2022-09-28T17:17:32.707Z",
+ "cisco_meraki": {
+ "event_subtype": "flow_allowed",
+ "event_type": "flows",
+ "flows": {
+ "op": "allow"
+ }
+ },
+ "destination": {
+ "ip": "224.0.0.2"
+ },
+ "ecs": {
+ "version": "8.4.0"
+ },
+ "event": {
+ "action": "layer3-firewall-allowed-flow",
+ "category": [
+ "network"
+ ],
+ "original": "\u003c134\u003e1 1664385452.707589827 AP_XXXX flows allow src=172.16.12.23 dst=224.0.0.2 mac=4C:AB:4F:0D:3D:AA protocol=2",
+ "type": [
+ "info",
+ "connection",
+ "start"
+ ]
+ },
+ "network": {
+ "protocol": "2"
+ },
+ "observer": {
+ "hostname": "AP_XXXX"
+ },
+ "source": {
+ "ip": "172.16.12.23",
+ "mac": "4C-AB-4F-0D-3D-AA"
+ },
+ "tags": [
+ "forwarded",
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2022-09-28T17:17:33.129Z",
+ "cisco_meraki": {
+ "event_subtype": "flow_allowed",
+ "event_type": "flows",
+ "flows": {
+ "op": "allow"
+ }
+ },
+ "destination": {
+ "geo": {
+ "city_name": "London",
+ "continent_name": "Europe",
+ "country_iso_code": "GB",
+ "country_name": "United Kingdom",
+ "location": {
+ "lat": 51.5142,
+ "lon": -0.0931
+ },
+ "region_iso_code": "GB-ENG",
+ "region_name": "England"
+ },
+ "ip": "81.2.69.144"
+ },
+ "ecs": {
+ "version": "8.4.0"
+ },
+ "event": {
+ "action": "layer3-firewall-allowed-flow",
+ "category": [
+ "network"
+ ],
+ "original": "\u003c134\u003e1 1664385453.129104346 AP_XXXX flows allow src=172.16.10.14 dst=81.2.69.144 mac=EC:63:D7:0F:6B:AA protocol=icmp type=8",
+ "type": [
+ "info",
+ "connection",
+ "start"
+ ]
+ },
+ "network": {
+ "protocol": "icmp"
+ },
+ "observer": {
+ "hostname": "AP_XXXX"
+ },
+ "source": {
+ "ip": "172.16.10.14",
+ "mac": "EC-63-D7-0F-6B-AA"
+ },
+ "tags": [
+ "forwarded",
+ "preserve_original_event"
+ ]
  }
  ]
 }

packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/flows.yml

@@ -1,72 +1,24 @@
 ---
 description: Pipeline for Cisco Meraki flows message type
 processors:
-- dissect:
- description: Determine if the token is src= or operation
- field: event.original
- pattern: "%{} %{} %{} %{} %{_temp.token} %{}"
-- dissect:
- description: Case for src= follows flows keyword
- field: event.original
- pattern: "%{} flows %{*src}=%{&src} %{*dst}=%{&dst} %{*prot}=%{&prot} %{*sport}=%{&sport} %{*dport}=%{&dport} %{}"
- if: ctx._temp.token.startsWith("src=") == true
-- dissect:
- description: Case for firewall action prepends src=
- field: event.original
- pattern: "%{} flows %{cisco_meraki.flows.op} %{*src}=%{&src} %{*dst}=%{&dst} %{*mac}=%{&mac} %{*prot}=%{&prot} %{*sport}=%{&sport} %{*dport}=%{&dport}"
- if: ctx._temp.token.startsWith("src=") == false
 - grok:
- field: src
- patterns:
- - "^%{IPV4:src}$"
- - "^%{IPV6:src}$"
- if: ctx?.src != null
-- convert:
- type: ip
- field: src
- target_field: source.ip
- ignore_failure: true
-- grok:
- field: dst
+ field: event.original
  patterns:
- - "^%{IPV4:dst}$"
- - "^%{IPV6:dst}$"
- if: ctx?.dst != null
-- convert:
- type: ip
- field: dst
- target_field: destination.ip
- ignore_failure: true
-- rename:
- field: protocol
- target_field: network.protocol
-- convert:
- field: sport
- target_field: source.port
- type: long
- if: ctx?.sport != "0"
- ignore_failure: true
-- convert:
- field: dport
- target_field: destination.port
- type: long
- if: ctx?.dport != "0"
- ignore_failure: true
+ - "flows( %{NOTSPACE:cisco_meraki.flows.op})? src=%{IP:source.ip:ip} dst=%{IP:destination.ip:ip}( mac=%{MAC:source.mac})? protocol=%{NOTSPACE:network.protocol}( sport=%{NONNEGINT:source.port:long})?( dport=%{NONNEGINT:destination.port:long})?"
 - gsub:
- field: mac
- target_field: source.mac
- pattern: '[-:.]'
+ field: source.mac
+ pattern: '[:.]'
  replacement: '-'
- if: ctx._temp.token.startsWith("src=") == false
+ ignore_missing: true
 - set:
  field: cisco_meraki.event_subtype
  value: "ip_session_initiated"
- if: ctx._temp.token.startsWith("src=") == true
+ if: ctx.cisco_meraki?.flows?.op == null
 - set:
  field: cisco_meraki.event_subtype
  value: "flow_allowed"
- if: ctx._temp.token.startsWith("src=") == false && ctx?.cisco_meraki?.flows?.op == 'allow'
+ if: ctx.cisco_meraki?.flows?.op == 'allow'
 - set:
  field: cisco_meraki.event_subtype
  value: "flow_denied"
- if: ctx._temp.token.startsWith("src=") == false && ctx?.cisco_meraki?.flows?.op == 'deny'
+ if: ctx.cisco_meraki?.flows?.op == 'deny'

packages/cisco_meraki/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: cisco_meraki
 title: Cisco Meraki
-version: 1.2.2
+version: 1.2.3
 license: basic
 description: Collect logs from Cisco Meraki with Elastic Agent.
 type: integration