Skip to content

Commit aa92497

Browse files
kaiyan-shengkaiyan-sheng
authored
Update AWS ELB ingest pipeline (#1212)
1 parent 954072f commit aa92497

File tree

6 files changed

+80
-7
lines changed

6 files changed

+80
-7
lines changed

‎packages/aws/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "0.6.2"
3+
changes:
4+
- description: Removed incorrect `http.request.referrer` field from elb logs
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/1212
27
- version: "0.6.1"
38
changes:
49
- description: Add support for CloudTrail Digest & Insight logs

‎packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@
1010
"port": "2817",
1111
"ip": "192.168.131.39"
1212
},
13+
"url": {
14+
"path": "/",
15+
"original": "http://www.example.com:80/",
16+
"scheme": "http",
17+
"port": 80,
18+
"domain": "www.example.com"
19+
},
1320
"tags": [
1421
"preserve_original_event"
1522
],
@@ -25,8 +32,7 @@
2532
"method": "get",
2633
"body": {
2734
"bytes": 34
28-
},
29-
"referrer": "http://www.example.com:80/"
35+
}
3036
},
3137
"version": "1.1",
3238
"response": {
@@ -37,7 +43,7 @@
3743
}
3844
},
3945
"event": {
40-
"ingested": "2021-06-04T09:43:16.047944100Z",
46+
"ingested": "2021-06-24T23:04:01.923552600Z",
4147
"original": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward,redirect\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"",
4248
"kind": "event",
4349
"start": "2018-07-02T22:22:48.364000Z",
@@ -86,7 +92,12 @@
8692
}
8793
},
8894
"user_agent": {
89-
"original": "curl/7.46.0"
95+
"name": "curl",
96+
"original": "curl/7.46.0",
97+
"device": {
98+
"name": "Other"
99+
},
100+
"version": "7.46.0"
90101
}
91102
}
92103
]

‎packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,8 @@ processors:
8585
(?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long})
8686
%{NUMBER:http.request.body.bytes:long}
8787
%{NUMBER:http.response.body.bytes:long}
88-
\"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\"
89-
\"%{DATA:user_agent.original}\"
88+
\"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.uri_orig}) (?:-|HTTP/%{NOTSPACE:http.version})\"
89+
\"%{DATA:_tmp.user_agent}\"
9090
%{ELBSSL}
9191
ELBTCPLOG: >-
9292
%{ELBCOMMON}
@@ -109,6 +109,17 @@ processors:
109109
if: ctx.http != null
110110
field: aws.elb.protocol
111111
value: http
112+
113+
- uri_parts:
114+
if: 'ctx?._tmp?.uri_orig != null'
115+
field: _tmp.uri_orig
116+
ignore_failure: true
117+
118+
- user_agent:
119+
if: 'ctx?._tmp?.user_agent != null'
120+
field: _tmp.user_agent
121+
ignore_missing: true
122+
112123
- set:
113124
if: ctx.http != null
114125
field: event.category

‎packages/aws/data_stream/elb_logs/fields/ecs.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,41 @@
1515
example: '["production", "env2"]'
1616
ignore_above: 1024
1717
type: keyword
18+
- name: url
19+
type: group
20+
fields:
21+
- name: domain
22+
level: extended
23+
type: keyword
24+
description: Domain of the url, such as "www.elastic.co".
25+
- name: original
26+
level: extended
27+
type: keyword
28+
description: Unmodified original url as seen in the event source.
29+
- name: path
30+
level: extended
31+
type: keyword
32+
description: Path of the request, such as "/search".
33+
- name: port
34+
level: extended
35+
type: long
36+
description: Port of the request, such as 443.
37+
- name: scheme
38+
level: extended
39+
type: keyword
40+
description: Scheme of the request, such as "https".
41+
- name: user_agent
42+
type: group
43+
fields:
44+
- name: device.name
45+
level: extended
46+
type: keyword
47+
description: Name of the device.
48+
- name: name
49+
level: extended
50+
type: keyword
51+
description: Name of the user agent.
52+
- name: version
53+
level: extended
54+
type: keyword
55+
description: Version of the user agent.

‎packages/aws/docs/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,15 @@ For network load balancer, please follow [enable access log for network load bal
432432
| source.port | Port of the source. | keyword |
433433
| tags | List of keywords used to tag each event. | keyword |
434434
| tracing.trace.id | Unique identifier of the trace. | keyword |
435+
| url.domain | Domain of the url, such as "www.elastic.co". | keyword |
436+
| url.original | Unmodified original url as seen in the event source. | keyword |
437+
| url.path | Path of the request, such as "/search". | keyword |
438+
| url.port | Port of the request, such as 443. | long |
439+
| url.scheme | Scheme of the request, such as "https". | keyword |
440+
| user_agent.device.name | Name of the device. | keyword |
441+
| user_agent.name | Name of the user agent. | keyword |
435442
| user_agent.original | Unparsed user_agent string. | keyword |
443+
| user_agent.version | Version of the user agent. | keyword |
436444

437445

438446
### s3access

‎packages/aws/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 1.0.0
22
name: aws
33
title: AWS
4-
version: 0.6.1
4+
version: 0.6.2
55
license: basic
66
description: AWS Integration
77
type: integration

0 commit comments

Comments
 (0)