Skip to content

Commit 339158b

Browse files
efd6efd6
authored
network_traffic: add optional GeoIP enrichment (#4822)
Make the enrichment configurable to allow users with very high throughput and no need for geolocation to turn off enrichment, but set to on by default.
1 parent 261efc4 commit 339158b

File tree

96 files changed

+14837
-11622
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+14837
-11622
lines changed

‎packages/network_traffic/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.8.0"
3+
changes:
4+
- description: GeoIP enrich IP addresses.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/4822
27
- version: "1.7.1"
38
changes:
49
- description: Migrate the visualizations to by value in dashboards to minimize the saved object clutter and reduce time to load

‎packages/network_traffic/data_stream/amqp/agent/stream/amqp.yml.hbs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ ports:
55
- {{p}}
66
{{/each}}
77
{{/if}}
8+
fields_under_root: true
9+
fields:
10+
_conf:
11+
geoip_enrich: {{geoip_enrich}}
812
{{#if max_body_length}}
913
max_body_length: {{max_body_length}}
1014
{{/if}}

‎packages/network_traffic/data_stream/amqp/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ processors:
2121
- uppercase:
2222
field: host.mac
2323
ignore_missing: true
24+
25+
- pipeline:
26+
if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich
27+
name: '{{ IngestPipeline "geoip" }}'
28+
- remove:
29+
field: _conf
30+
ignore_missing: true
31+
2432
- script:
2533
description: Remove invalid "network_traffic" term added by packetbeat prior to v8.
2634
# This string-based comparison is valid while versions are below v10.x.

‎packages/network_traffic/data_stream/amqp/elasticsearch/ingest_pipeline/geoip.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
---
2+
description: GeoIP enrichment.
3+
processors:
4+
- geoip:
5+
field: source.ip
6+
target_field: source.geo
7+
ignore_missing: true
8+
- geoip:
9+
database_file: GeoLite2-ASN.mmdb
10+
field: source.ip
11+
target_field: source.as
12+
properties:
13+
- asn
14+
- organization_name
15+
ignore_missing: true
16+
- rename:
17+
field: source.as.asn
18+
target_field: source.as.number
19+
ignore_missing: true
20+
- rename:
21+
field: source.as.organization_name
22+
target_field: source.as.organization.name
23+
ignore_missing: true
24+
25+
- geoip:
26+
field: destination.ip
27+
target_field: destination.geo
28+
ignore_missing: true
29+
- geoip:
30+
database_file: GeoLite2-ASN.mmdb
31+
field: destination.ip
32+
target_field: destination.as
33+
properties:
34+
- asn
35+
- organization_name
36+
ignore_missing: true
37+
- rename:
38+
field: destination.as.asn
39+
target_field: destination.as.number
40+
ignore_missing: true
41+
- rename:
42+
field: destination.as.organization_name
43+
target_field: destination.as.organization.name
44+
ignore_missing: true
45+
46+
- geoip:
47+
field: server.ip
48+
target_field: server.geo
49+
ignore_missing: true
50+
- geoip:
51+
database_file: GeoLite2-ASN.mmdb
52+
field: server.ip
53+
target_field: server.as
54+
properties:
55+
- asn
56+
- organization_name
57+
ignore_missing: true
58+
- rename:
59+
field: server.as.asn
60+
target_field: server.as.number
61+
ignore_missing: true
62+
- rename:
63+
field: server.as.organization_name
64+
target_field: server.as.organization.name
65+
ignore_missing: true
66+
67+
- geoip:
68+
field: client.ip
69+
target_field: client.geo
70+
ignore_missing: true
71+
- geoip:
72+
database_file: GeoLite2-ASN.mmdb
73+
field: client.ip
74+
target_field: client.as
75+
properties:
76+
- asn
77+
- organization_name
78+
ignore_missing: true
79+
- rename:
80+
field: client.as.asn
81+
target_field: client.as.number
82+
ignore_missing: true
83+
- rename:
84+
field: client.as.organization_name
85+
target_field: client.as.organization.name
86+
ignore_missing: true
87+
88+
on_failure:
89+
- set:
90+
field: error.message
91+
value: "{{{ _ingest.on_failure_message }}}"

‎packages/network_traffic/data_stream/amqp/fields/ecs.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,59 @@
6666
name: source.ip
6767
- external: ecs
6868
name: source.port
69+
- external: ecs
70+
name: client.geo.city_name
71+
- external: ecs
72+
name: client.geo.continent_name
73+
- external: ecs
74+
name: client.geo.country_iso_code
75+
- external: ecs
76+
name: client.geo.country_name
77+
- external: ecs
78+
name: client.geo.location
79+
- external: ecs
80+
name: client.geo.region_iso_code
81+
- external: ecs
82+
name: client.geo.region_name
83+
- external: ecs
84+
name: destination.geo.city_name
85+
- external: ecs
86+
name: destination.geo.continent_name
87+
- external: ecs
88+
name: destination.geo.country_iso_code
89+
- external: ecs
90+
name: destination.geo.country_name
91+
- external: ecs
92+
name: destination.geo.location
93+
- external: ecs
94+
name: destination.geo.region_iso_code
95+
- external: ecs
96+
name: destination.geo.region_name
97+
- external: ecs
98+
name: server.geo.city_name
99+
- external: ecs
100+
name: server.geo.continent_name
101+
- external: ecs
102+
name: server.geo.country_iso_code
103+
- external: ecs
104+
name: server.geo.country_name
105+
- external: ecs
106+
name: server.geo.location
107+
- external: ecs
108+
name: server.geo.region_iso_code
109+
- external: ecs
110+
name: server.geo.region_name
111+
- external: ecs
112+
name: source.geo.city_name
113+
- external: ecs
114+
name: source.geo.continent_name
115+
- external: ecs
116+
name: source.geo.country_iso_code
117+
- external: ecs
118+
name: source.geo.country_name
119+
- external: ecs
120+
name: source.geo.location
121+
- external: ecs
122+
name: source.geo.region_iso_code
123+
- external: ecs
124+
name: source.geo.region_name

‎packages/network_traffic/data_stream/amqp/manifest.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ streams:
1414
required: true
1515
show_user: true
1616
default: [5672]
17+
- name: geoip_enrich
18+
required: true
19+
show_user: true
20+
title: GeoIP enrich IP addresses
21+
description: Perform GeoIP enrichment on IP addresses in events.
22+
type: bool
23+
multi: false
24+
default: true
1725
- name: monitor_processes
1826
type: bool
1927
title: Monitor Processes

‎packages/network_traffic/data_stream/amqp/sample_event.json‎

Lines changed: 65 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{
2-
"@timestamp": "2022-06-28T23:30:50.532Z",
2+
"@timestamp": "2022-12-13T23:20:31.501Z",
33
"agent": {
4-
"ephemeral_id": "a929a7a7-1dba-4961-8b38-b7ef8ed7f1e2",
5-
"id": "827ce6a9-85bd-4e07-9a7a-4896c17144cd",
4+
"ephemeral_id": "aa5ced14-2b74-4cf6-b122-067a35e57cd3",
5+
"id": "00a75d9d-728c-4ab5-acb6-9f78082797e7",
66
"name": "docker-fleet-agent",
77
"type": "packetbeat",
8-
"version": "8.2.3"
8+
"version": "8.4.1"
99
},
1010
"amqp": {
1111
"auto-delete": false,
@@ -19,7 +19,19 @@
1919
},
2020
"client": {
2121
"bytes": 25,
22-
"ip": "127.0.0.1",
22+
"geo": {
23+
"city_name": "Changchun",
24+
"continent_name": "Asia",
25+
"country_iso_code": "CN",
26+
"country_name": "China",
27+
"location": {
28+
"lat": 43.88,
29+
"lon": 125.3228
30+
},
31+
"region_iso_code": "CN-22",
32+
"region_name": "Jilin Sheng"
33+
},
34+
"ip": "175.16.199.1",
2335
"port": 34222
2436
},
2537
"data_stream": {
@@ -29,16 +41,28 @@
2941
},
3042
"destination": {
3143
"bytes": 26,
32-
"ip": "127.0.0.1",
44+
"geo": {
45+
"city_name": "Changchun",
46+
"continent_name": "Asia",
47+
"country_iso_code": "CN",
48+
"country_name": "China",
49+
"location": {
50+
"lat": 43.88,
51+
"lon": 125.3228
52+
},
53+
"region_iso_code": "CN-22",
54+
"region_name": "Jilin Sheng"
55+
},
56+
"ip": "175.16.199.1",
3357
"port": 5672
3458
},
3559
"ecs": {
3660
"version": "8.5.0"
3761
},
3862
"elastic_agent": {
39-
"id": "827ce6a9-85bd-4e07-9a7a-4896c17144cd",
63+
"id": "00a75d9d-728c-4ab5-acb6-9f78082797e7",
4064
"snapshot": false,
41-
"version": "8.2.3"
65+
"version": "8.4.1"
4266
},
4367
"event": {
4468
"action": "amqp.queue.declare",
@@ -47,11 +71,11 @@
4771
"network"
4872
],
4973
"dataset": "network_traffic.amqp",
50-
"duration": 1407200,
51-
"end": "2022-06-28T23:30:50.533Z",
52-
"ingested": "2022-06-28T23:30:54Z",
74+
"duration": 1201362,
75+
"end": "2022-12-13T23:20:31.502Z",
76+
"ingested": "2022-12-13T23:20:35Z",
5377
"kind": "event",
54-
"start": "2022-06-28T23:30:50.532Z",
78+
"start": "2022-12-13T23:20:31.501Z",
5579
"type": [
5680
"connection",
5781
"protocol"
@@ -61,11 +85,12 @@
6185
"architecture": "x86_64",
6286
"containerized": false,
6387
"hostname": "docker-fleet-agent",
88+
"id": "51511c1493f34922b559a964798246ec",
6489
"ip": [
65-
"192.168.48.7"
90+
"172.28.0.7"
6691
],
6792
"mac": [
68-
"02-42-C0-A8-30-07"
93+
"02-42-AC-1C-00-07"
6994
],
7095
"name": "docker-fleet-agent",
7196
"os": {
@@ -94,12 +119,36 @@
94119
},
95120
"server": {
96121
"bytes": 26,
97-
"ip": "127.0.0.1",
122+
"geo": {
123+
"city_name": "Changchun",
124+
"continent_name": "Asia",
125+
"country_iso_code": "CN",
126+
"country_name": "China",
127+
"location": {
128+
"lat": 43.88,
129+
"lon": 125.3228
130+
},
131+
"region_iso_code": "CN-22",
132+
"region_name": "Jilin Sheng"
133+
},
134+
"ip": "175.16.199.1",
98135
"port": 5672
99136
},
100137
"source": {
101138
"bytes": 25,
102-
"ip": "127.0.0.1",
139+
"geo": {
140+
"city_name": "Changchun",
141+
"continent_name": "Asia",
142+
"country_iso_code": "CN",
143+
"country_name": "China",
144+
"location": {
145+
"lat": 43.88,
146+
"lon": 125.3228
147+
},
148+
"region_iso_code": "CN-22",
149+
"region_name": "Jilin Sheng"
150+
},
151+
"ip": "175.16.199.1",
103152
"port": 34222
104153
},
105154
"status": "OK",

‎packages/network_traffic/data_stream/cassandra/agent/stream/cassandra.yml.hbs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ ports:
55
- {{p}}
66
{{/each}}
77
{{/if}}
8+
fields_under_root: true
9+
fields:
10+
_conf:
11+
geoip_enrich: {{geoip_enrich}}
812
{{#if send_request}}
913
send_request: {{send_request}}
1014
{{/if}}

‎packages/network_traffic/data_stream/cassandra/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ processors:
2121
- uppercase:
2222
field: host.mac
2323
ignore_missing: true
24+
25+
- pipeline:
26+
if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich
27+
name: '{{ IngestPipeline "geoip" }}'
28+
- remove:
29+
field: _conf
30+
ignore_missing: true
31+
2432
- script:
2533
description: Remove invalid "network_traffic" term added by packetbeat prior to v8.
2634
# This string-based comparison is valid while versions are below v10.x.

0 commit comments

Comments
 (0)