Skip to content

network_traffic: add optional GeoIP enrichment #4822

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 14, 2022
Merged

network_traffic: add optional GeoIP enrichment #4822

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f7ff135
network_traffic: add optional GeoIP enrichment
efd6 Dec 13, 2022
c9fed16
network_traffic: run elastic-package format
efd6 Dec 14, 2022
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/network_traffic/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.8.0"
changes:
- description: GeoIP enrich IP addresses.
type: enhancement
link: https://github.com/elastic/integrations/pull/4822
- version: "1.7.1"
changes:
- description: Migrate the visualizations to by value in dashboards to minimize the saved object clutter and reduce time to load
Expand Down
4 changes: 4 additions & 0 deletions packages/network_traffic/data_stream/amqp/agent/stream/amqp.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ ports:
- {{p}}
{{/each}}
{{/if}}
fields_under_root: true
fields:
_conf:
geoip_enrich: {{geoip_enrich}}
{{#if max_body_length}}
max_body_length: {{max_body_length}}
{{/if}}
Expand Down
8 changes: 8 additions & 0 deletions packages/network_traffic/data_stream/amqp/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ processors:
- uppercase:
field: host.mac
ignore_missing: true

- pipeline:
if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich
name: '{{ IngestPipeline "geoip" }}'
- remove:
field: _conf
ignore_missing: true

- script:
description: Remove invalid "network_traffic" term added by packetbeat prior to v8.
# This string-based comparison is valid while versions are below v10.x.
Expand Down
91 changes: 91 additions & 0 deletions packages/network_traffic/data_stream/amqp/elasticsearch/ingest_pipeline/geoip.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
---
description: GeoIP enrichment.
processors:
- geoip:
field: source.ip
target_field: source.geo
ignore_missing: true
- geoip:
database_file: GeoLite2-ASN.mmdb
field: source.ip
target_field: source.as
properties:
- asn
- organization_name
ignore_missing: true
- rename:
field: source.as.asn
target_field: source.as.number
ignore_missing: true
- rename:
field: source.as.organization_name
target_field: source.as.organization.name
ignore_missing: true

- geoip:
field: destination.ip
target_field: destination.geo
ignore_missing: true
- geoip:
database_file: GeoLite2-ASN.mmdb
field: destination.ip
target_field: destination.as
properties:
- asn
- organization_name
ignore_missing: true
- rename:
field: destination.as.asn
target_field: destination.as.number
ignore_missing: true
- rename:
field: destination.as.organization_name
target_field: destination.as.organization.name
ignore_missing: true

- geoip:
field: server.ip
target_field: server.geo
ignore_missing: true
- geoip:
database_file: GeoLite2-ASN.mmdb
field: server.ip
target_field: server.as
properties:
- asn
- organization_name
ignore_missing: true
- rename:
field: server.as.asn
target_field: server.as.number
ignore_missing: true
- rename:
field: server.as.organization_name
target_field: server.as.organization.name
ignore_missing: true

- geoip:
field: client.ip
target_field: client.geo
ignore_missing: true
- geoip:
database_file: GeoLite2-ASN.mmdb
field: client.ip
target_field: client.as
properties:
- asn
- organization_name
ignore_missing: true
- rename:
field: client.as.asn
target_field: client.as.number
ignore_missing: true
- rename:
field: client.as.organization_name
target_field: client.as.organization.name
ignore_missing: true

on_failure:
- set:
field: error.message
value: "{{{ _ingest.on_failure_message }}}"
56 changes: 56 additions & 0 deletions packages/network_traffic/data_stream/amqp/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,59 @@
name: source.ip
- external: ecs
name: source.port
- external: ecs
name: client.geo.city_name
- external: ecs
name: client.geo.continent_name
- external: ecs
name: client.geo.country_iso_code
- external: ecs
name: client.geo.country_name
- external: ecs
name: client.geo.location
- external: ecs
name: client.geo.region_iso_code
- external: ecs
name: client.geo.region_name
- external: ecs
name: destination.geo.city_name
- external: ecs
name: destination.geo.continent_name
- external: ecs
name: destination.geo.country_iso_code
- external: ecs
name: destination.geo.country_name
- external: ecs
name: destination.geo.location
- external: ecs
name: destination.geo.region_iso_code
- external: ecs
name: destination.geo.region_name
- external: ecs
name: server.geo.city_name
- external: ecs
name: server.geo.continent_name
- external: ecs
name: server.geo.country_iso_code
- external: ecs
name: server.geo.country_name
- external: ecs
name: server.geo.location
- external: ecs
name: server.geo.region_iso_code
- external: ecs
name: server.geo.region_name
- external: ecs
name: source.geo.city_name
- external: ecs
name: source.geo.continent_name
- external: ecs
name: source.geo.country_iso_code
- external: ecs
name: source.geo.country_name
- external: ecs
name: source.geo.location
- external: ecs
name: source.geo.region_iso_code
- external: ecs
name: source.geo.region_name
8 changes: 8 additions & 0 deletions packages/network_traffic/data_stream/amqp/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ streams:
required: true
show_user: true
default: [5672]
- name: geoip_enrich
required: true
show_user: true
title: GeoIP enrich IP addresses
description: Perform GeoIP enrichment on IP addresses in events.
type: bool
multi: false
default: true
- name: monitor_processes
type: bool
title: Monitor Processes
Expand Down
81 changes: 65 additions & 16 deletions packages/network_traffic/data_stream/amqp/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"@timestamp": "2022-06-28T23:30:50.532Z",
"@timestamp": "2022-12-13T23:20:31.501Z",
"agent": {
"ephemeral_id": "a929a7a7-1dba-4961-8b38-b7ef8ed7f1e2",
"id": "827ce6a9-85bd-4e07-9a7a-4896c17144cd",
"ephemeral_id": "aa5ced14-2b74-4cf6-b122-067a35e57cd3",
"id": "00a75d9d-728c-4ab5-acb6-9f78082797e7",
"name": "docker-fleet-agent",
"type": "packetbeat",
"version": "8.2.3"
"version": "8.4.1"
},
"amqp": {
"auto-delete": false,
Expand All @@ -19,7 +19,19 @@
},
"client": {
"bytes": 25,
"ip": "127.0.0.1",
"geo": {
"city_name": "Changchun",
"continent_name": "Asia",
"country_iso_code": "CN",
"country_name": "China",
"location": {
"lat": 43.88,
"lon": 125.3228
},
"region_iso_code": "CN-22",
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"port": 34222
},
"data_stream": {
Expand All @@ -29,16 +41,28 @@
},
"destination": {
"bytes": 26,
"ip": "127.0.0.1",
"geo": {
"city_name": "Changchun",
"continent_name": "Asia",
"country_iso_code": "CN",
"country_name": "China",
"location": {
"lat": 43.88,
"lon": 125.3228
},
"region_iso_code": "CN-22",
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"port": 5672
},
"ecs": {
"version": "8.5.0"
},
"elastic_agent": {
"id": "827ce6a9-85bd-4e07-9a7a-4896c17144cd",
"id": "00a75d9d-728c-4ab5-acb6-9f78082797e7",
"snapshot": false,
"version": "8.2.3"
"version": "8.4.1"
},
"event": {
"action": "amqp.queue.declare",
Expand All @@ -47,11 +71,11 @@
"network"
],
"dataset": "network_traffic.amqp",
"duration": 1407200,
"end": "2022-06-28T23:30:50.533Z",
"ingested": "2022-06-28T23:30:54Z",
"duration": 1201362,
"end": "2022-12-13T23:20:31.502Z",
"ingested": "2022-12-13T23:20:35Z",
"kind": "event",
"start": "2022-06-28T23:30:50.532Z",
"start": "2022-12-13T23:20:31.501Z",
"type": [
"connection",
"protocol"
Expand All @@ -61,11 +85,12 @@
"architecture": "x86_64",
"containerized": false,
"hostname": "docker-fleet-agent",
"id": "51511c1493f34922b559a964798246ec",
"ip": [
"192.168.48.7"
"172.28.0.7"
],
"mac": [
"02-42-C0-A8-30-07"
"02-42-AC-1C-00-07"
],
"name": "docker-fleet-agent",
"os": {
Expand Down Expand Up @@ -94,12 +119,36 @@
},
"server": {
"bytes": 26,
"ip": "127.0.0.1",
"geo": {
"city_name": "Changchun",
"continent_name": "Asia",
"country_iso_code": "CN",
"country_name": "China",
"location": {
"lat": 43.88,
"lon": 125.3228
},
"region_iso_code": "CN-22",
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"port": 5672
},
"source": {
"bytes": 25,
"ip": "127.0.0.1",
"geo": {
"city_name": "Changchun",
"continent_name": "Asia",
"country_iso_code": "CN",
"country_name": "China",
"location": {
"lat": 43.88,
"lon": 125.3228
},
"region_iso_code": "CN-22",
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"port": 34222
},
"status": "OK",
Expand Down
4 changes: 4 additions & 0 deletions packages/network_traffic/data_stream/cassandra/agent/stream/cassandra.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ ports:
- {{p}}
{{/each}}
{{/if}}
fields_under_root: true
fields:
_conf:
geoip_enrich: {{geoip_enrich}}
{{#if send_request}}
send_request: {{send_request}}
{{/if}}
Expand Down
8 changes: 8 additions & 0 deletions packages/network_traffic/data_stream/cassandra/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ processors:
- uppercase:
field: host.mac
ignore_missing: true

- pipeline:
if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich
name: '{{ IngestPipeline "geoip" }}'
- remove:
field: _conf
ignore_missing: true

- script:
description: Remove invalid "network_traffic" term added by packetbeat prior to v8.
# This string-based comparison is valid while versions are below v10.x.
Expand Down
Loading