[zeek] Remove redundant event.ingested from zeek pipeline (#2503)

event.ingested is added by Fleet's final_pipeline so this was redundant. Also refresh the readme.

Author: andrewkroh

packages/zeek/_dev/build/docs/README.md

@@ -1,15 +1,22 @@
 # Zeek Integration
 
-This is an integration for Zeek, which used to be called Bro. It
-parses logs that are in the [Zeek JSON
-format](https://www.zeek.org/manual/release/logs/index.html).
+This is an integration for [Zeek](https://www.zeek.org/), which was formerly
+named Bro. Zeek is a passive, open-source network traffic analyzer. This
+integrations ingests the logs Zeek produces about the network traffic that it
+analyzes.
+
+Zeek logs must be output in JSON format. This is normally done by appending the 
+[json-logs policy](https://docs.zeek.org/en/lts/scripts/policy/tuning/json-logs.zeek.html)
+to your `local.zeek` file. Add this line to your `local.zeek`.
+
+`@load policy/tuning/json-logs.zeek`
 
 ## Compatibility
-This module has been developed against Zeek 2.6.1, but is expected to
-work with other versions of Zeek.
+This module has been developed against Zeek 2.6.1, but is expected to work with
+other versions of Zeek.
 
 Zeek requires a Unix-like platform, and it currently supports Linux,
-FreeBSD, and Mac OS X. Find out how to use Zeek [here](https://www.zeek.org/).
+FreeBSD, and Mac OS X.
 
 ## Logs
 ### capture_loss

packages/zeek/changelog.yml

@@ -1,9 +1,14 @@
 # newer versions go on top
+- version: "1.5.4"
+ changes:
+ - description: Remove redundant event.ingested from Zeek pipelines.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/2503
 - version: "1.5.3"
  changes:
  - description: Ignore URI parse failures in zeek.http data.
  type: bugfix
- link: https://github.com/elastic/integrations/pull/
+ link: https://github.com/elastic/integrations/pull/2501
 - version: "1.5.2"
  changes:
  - description: Regenerate test files using the new GeoIP database

packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
- event.ingested: ".*"
 fields:
  "@timestamp": "2020-04-28T11:07:58.223Z"
  tags:

packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json

@@ -15,7 +15,6 @@
  }
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.224983069Z",
  "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
  "type": "info",
  "created": "2020-04-28T11:07:58.223Z",
@@ -40,7 +39,6 @@
  }
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.224985489Z",
  "original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}",
  "type": "info",
  "created": "2020-04-28T11:07:58.223Z",
@@ -65,7 +63,6 @@
  }
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.224985934Z",
  "original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}",
  "type": "info",
  "created": "2020-04-28T11:07:58.223Z",
@@ -90,7 +87,6 @@
  }
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.224986315Z",
  "original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}",
  "type": "info",
  "created": "2020-04-28T11:07:58.223Z",
@@ -115,7 +111,6 @@
  }
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.224986695Z",
  "original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}",
  "type": "info",
  "created": "2020-04-28T11:07:58.223Z",
@@ -148,7 +143,6 @@
  }
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.224987082Z",
  "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
  "type": "info",
  "created": "2020-04-28T11:07:58.223Z",

packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml

@@ -16,9 +16,7 @@ processors:
  - rename:
  target_field: zeek.capture_loss
  field: _temp_
- - set:
- field: event.ingested
- value: "{{_ingest.timestamp}}"
+
 # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
  - set:
  field: event.created

packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
- event.ingested: ".*"
 fields:
  "@timestamp": "2020-04-28T11:07:58.223Z"
  tags:

packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json

@@ -38,7 +38,6 @@
  },
  "event": {
  "duration": 76967000,
- "ingested": "2021-12-14T14:59:07.719052684Z",
  "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -120,7 +119,6 @@
  },
  "event": {
  "duration": 76967000,
- "ingested": "2021-12-14T14:59:07.719086167Z",
  "original": "{\"ts\":1547188416.857497,\"uid\":\"CAcJw21BbVedgFnYH4\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38340,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -219,7 +217,6 @@
  },
  "event": {
  "duration": 76967000,
- "ingested": "2021-12-14T14:59:07.719086748Z",
  "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"89.160.20.156\",\"id.orig_p\":38334,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -301,16 +298,15 @@
  "ip": "192.168.2.205"
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.719087140Z",
  "original": "{\"ts\":1551399000.57855,\"uid\":\"Cc6NJ3GRlfjE44I3h\",\"id.orig_h\":\"192.168.2.205\",\"id.orig_p\":3,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":3,\"proto\":\"icmp\",\"conn_state\":\"OTH\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"orig_pkts\":1,\"orig_ip_bytes\":107,\"resp_pkts\":0,\"resp_ip_bytes\":0,\"tunnel_parents\":[]}",
- "created": "2020-04-28T11:07:58.223Z",
- "kind": "event",
  "id": "Cc6NJ3GRlfjE44I3h",
  "category": "network",
  "type": [
  "connection",
  "info"
- ]
+ ],
+ "created": "2020-04-28T11:07:58.223Z",
+ "kind": "event"
  },
  "tags": [
  "preserve_original_event",
@@ -380,16 +376,15 @@
  "ip": "10.156.0.2"
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.719087516Z",
  "original": "{\"ts\":1617062400.404645,\"uid\":\"CCicIg43lOtCQOxXnb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":56190,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":443,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}",
- "created": "2020-04-28T11:07:58.223Z",
- "kind": "event",
  "id": "CCicIg43lOtCQOxXnb",
  "category": "network",
  "type": [
  "connection",
  "info"
- ]
+ ],
+ "created": "2020-04-28T11:07:58.223Z",
+ "kind": "event"
  },
  "tags": [
  "preserve_original_event",
@@ -460,7 +455,6 @@
  },
  "event": {
  "duration": 103708982,
- "ingested": "2021-12-14T14:59:07.719087917Z",
  "original": "{\"ts\":1617062100.419397,\"uid\":\"C52mXBCPJ4pPGkhr1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60810,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10370898246765137,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -540,7 +534,6 @@
  },
  "event": {
  "duration": 104128838,
- "ingested": "2021-12-14T14:59:07.719090564Z",
  "original": "{\"ts\":1617062100.419603,\"uid\":\"CTzCky2CyLT5JJvHck\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60804,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10412883758544922,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -620,7 +613,6 @@
  },
  "event": {
  "duration": 104333878,
- "ingested": "2021-12-14T14:59:07.719090943Z",
  "original": "{\"ts\":1617062100.419826,\"uid\":\"CIkS28PDxqQnN49m2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60802,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10433387756347656,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -682,7 +674,6 @@
  },
  "event": {
  "duration": 26802063,
- "ingested": "2021-12-14T14:59:07.719091294Z",
  "original": "{\"ts\":1617062390.563187,\"uid\":\"CezEGe4jeLNkayV976\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":38948,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.02680206298828125,\"orig_bytes\":0,\"resp_bytes\":241,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":269}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -745,7 +736,6 @@
  },
  "event": {
  "duration": 25056124,
- "ingested": "2021-12-14T14:59:07.719091659Z",
  "original": "{\"ts\":1617062390.563442,\"uid\":\"CKSr3w18mmW6t7bXC4\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":40080,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.025056123733520509,\"orig_bytes\":0,\"resp_bytes\":276,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":304}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -808,7 +798,6 @@
  },
  "event": {
  "duration": 3319979,
- "ingested": "2021-12-14T14:59:07.719092014Z",
  "original": "{\"ts\":1617062390.667048,\"uid\":\"CGUiHy4kLIF2ml95eg\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41407,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.003319978713989258,\"orig_bytes\":0,\"resp_bytes\":133,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":161}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -871,7 +860,6 @@
  },
  "event": {
  "duration": 1111984,
- "ingested": "2021-12-14T14:59:07.719092573Z",
  "original": "{\"ts\":1617062390.698943,\"uid\":\"CAOZZi4Qrio7gUVgVc\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":50487,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0011119842529296876,\"orig_bytes\":0,\"resp_bytes\":202,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":230}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -934,7 +922,6 @@
  },
  "event": {
  "duration": 908852,
- "ingested": "2021-12-14T14:59:07.719092936Z",
  "original": "{\"ts\":1617062390.699227,\"uid\":\"Chx5fs3xQ5ALB72i4e\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":49647,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0009088516235351563,\"orig_bytes\":0,\"resp_bytes\":145,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":173}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -996,16 +983,15 @@
  "ip": "10.156.0.2"
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.719093278Z",
  "original": "{\"ts\":1617062400.703865,\"uid\":\"C3pPjh1YRYcVDiZD3\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44944,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}",
- "created": "2020-04-28T11:07:58.223Z",
- "kind": "event",
  "id": "C3pPjh1YRYcVDiZD3",
  "category": "network",
  "type": [
  "connection",
  "info"
- ]
+ ],
+ "created": "2020-04-28T11:07:58.223Z",
+ "kind": "event"
  },
  "tags": [
  "preserve_original_event",
@@ -1057,16 +1043,15 @@
  "ip": "10.156.0.2"
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.719093624Z",
  "original": "{\"ts\":1617062400.703851,\"uid\":\"ChUxTmYLG37oO5qUb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44942,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}",
- "created": "2020-04-28T11:07:58.223Z",
- "kind": "event",
  "id": "ChUxTmYLG37oO5qUb",
  "category": "network",
  "type": [
  "connection",
  "info"
- ]
+ ],
+ "created": "2020-04-28T11:07:58.223Z",
+ "kind": "event"
  },
  "tags": [
  "preserve_original_event",
@@ -1118,16 +1103,15 @@
  "ip": "10.156.0.2"
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.719093964Z",
  "original": "{\"ts\":1617062400.704467,\"uid\":\"CpeAOT3B11CTXJgzw2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44946,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}",
- "created": "2020-04-28T11:07:58.223Z",
- "kind": "event",
  "id": "CpeAOT3B11CTXJgzw2",
  "category": "network",
  "type": [
  "connection",
  "info"
- ]
+ ],
+ "created": "2020-04-28T11:07:58.223Z",
+ "kind": "event"
  },
  "tags": [
  "preserve_original_event",
@@ -1236,7 +1220,6 @@
  },
  "event": {
  "duration": 76967000,
- "ingested": "2021-12-14T14:59:07.719094441Z",
  "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"89.160.20.156\",\"id.orig_p\":38334,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}",
  "created": "2020-04-28T11:07:58.223Z",
  "kind": "event",
@@ -1286,16 +1269,15 @@
  "ip": "10.0.2.15"
  },
  "event": {
- "ingested": "2021-12-14T14:59:07.719094822Z",
  "original": "{\"ts\":\"2021-06-09T20:55:13.160328Z\",\"uid\":\"C2KP1V3alRLoxl4JB9\",\"id.orig_h\":\"10.0.2.15\",\"id.orig_p\":46408,\"id.resp_h\":\"172.16.9.68\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}",
- "created": "2020-04-28T11:07:58.223Z",
- "kind": "event",
  "id": "C2KP1V3alRLoxl4JB9",
  "category": "network",
  "type": [
  "connection",
  "info"
- ]
+ ],
+ "created": "2020-04-28T11:07:58.223Z",
+ "kind": "event"
  },
  "tags": [
  "preserve_original_event",

packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml

@@ -17,9 +17,7 @@ processors:
  field: _temp_
  target_field: zeek.connection
  ignore_failure: true
- - set:
- field: event.ingested
- value: "{{_ingest.timestamp}}"
+
 # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
  - set:
  field: event.created

packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
- event.ingested: ".*"
 fields:
  "@timestamp": "2020-04-28T11:07:58.223Z"
  tags: