Skip to content

Commit 35f9966

Browse files
efd6efd6
authored
cloudflare_logpush: use EdgeStartTimestamp event timestamp (#5599)
1 parent 50ea10d commit 35f9966

File tree

6 files changed

+32
-23
lines changed

6 files changed

+32
-23
lines changed

‎packages/cloudflare_logpush/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "0.6.0"
3+
changes:
4+
- description: Use `EdgeStartTimestamp` as `@timestamp` time.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/5599
27
- version: "0.5.1"
38
changes:
49
- description: Added categories and/or subcategories.

‎packages/cloudflare_logpush/data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"expected": [
33
{
4+
"@timestamp": "2022-05-25T13:25:26Z",
45
"cloudflare_logpush": {
56
"http_request": {
67
"bot": {
@@ -243,6 +244,7 @@
243244
}
244245
},
245246
{
247+
"@timestamp": "2022-05-25T13:25:26Z",
246248
"cloudflare_logpush": {
247249
"http_request": {
248250
"bot": {

‎packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ processors:
2222
field: event.type
2323
value: [info]
2424
- date:
25-
field: json.EdgeEndTimestamp
26-
if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != ''
25+
field: json.EdgeStartTimestamp
26+
if: ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp != ''
2727
formats:
2828
- ISO8601
2929
- uuuu-MM-dd'T'HH:mm:ssX
@@ -32,14 +32,18 @@ processors:
3232
- yyyy-MM-dd'T'HH:mm:ss.SSSZ
3333
- UNIX_MS
3434
timezone: UTC
35-
target_field: cloudflare_logpush.http_request.edge.end_time
35+
target_field: cloudflare_logpush.http_request.edge.start_time
3636
on_failure:
3737
- append:
3838
field: error.message
3939
value: '{{{_ingest.on_failure_message}}}'
40+
- set:
41+
if: ctx.json?.EdgeStartTimestamp != null
42+
field: '@timestamp'
43+
copy_from: json.EdgeStartTimestamp
4044
- date:
41-
field: json.EdgeStartTimestamp
42-
if: ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp != ''
45+
field: json.EdgeEndTimestamp
46+
if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != ''
4347
formats:
4448
- ISO8601
4549
- uuuu-MM-dd'T'HH:mm:ssX
@@ -48,7 +52,7 @@ processors:
4852
- yyyy-MM-dd'T'HH:mm:ss.SSSZ
4953
- UNIX_MS
5054
timezone: UTC
51-
target_field: cloudflare_logpush.http_request.edge.start_time
55+
target_field: cloudflare_logpush.http_request.edge.end_time
5256
on_failure:
5357
- append:
5458
field: error.message

‎packages/cloudflare_logpush/data_stream/http_request/sample_event.json‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
{
2-
"@timestamp": "2022-09-01T10:08:19.901Z",
2+
"@timestamp": "2022-05-25T13:25:26Z",
33
"agent": {
4-
"ephemeral_id": "799a05d5-4523-4df3-8588-0a26bce74843",
5-
"hostname": "docker-fleet-agent",
6-
"id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
4+
"ephemeral_id": "dfdb0a3e-5218-4b1e-8ce1-38ad94902bf6",
5+
"id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
76
"name": "docker-fleet-agent",
87
"type": "filebeat",
9-
"version": "7.17.0"
8+
"version": "8.6.1"
109
},
1110
"cloudflare_logpush": {
1211
"http_request": {
@@ -188,17 +187,17 @@
188187
"version": "8.6.0"
189188
},
190189
"elastic_agent": {
191-
"id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
190+
"id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
192191
"snapshot": false,
193-
"version": "7.17.0"
192+
"version": "8.6.1"
194193
},
195194
"event": {
196195
"agent_id_status": "verified",
197196
"category": [
198197
"network"
199198
],
200199
"dataset": "cloudflare_logpush.http_request",
201-
"ingested": "2022-09-01T10:08:20Z",
200+
"ingested": "2023-03-21T00:21:42Z",
202201
"kind": "event",
203202
"original": "{\"BotScore\":\"20\",\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":\"bing\",\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityLevel\":\"off\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAction\":\"unknown\",\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRuleID\":\"98d93d5\",\"WAFRuleMessage\":\"matchad variable message\",\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}",
204203
"type": [

‎packages/cloudflare_logpush/docs/README.md‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -735,14 +735,13 @@ An example event for `http_request` looks as following:
735735

736736
```json
737737
{
738-
"@timestamp": "2022-09-01T10:08:19.901Z",
738+
"@timestamp": "2022-05-25T13:25:26Z",
739739
"agent": {
740-
"ephemeral_id": "799a05d5-4523-4df3-8588-0a26bce74843",
741-
"hostname": "docker-fleet-agent",
742-
"id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
740+
"ephemeral_id": "dfdb0a3e-5218-4b1e-8ce1-38ad94902bf6",
741+
"id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
743742
"name": "docker-fleet-agent",
744743
"type": "filebeat",
745-
"version": "7.17.0"
744+
"version": "8.6.1"
746745
},
747746
"cloudflare_logpush": {
748747
"http_request": {
@@ -924,17 +923,17 @@ An example event for `http_request` looks as following:
924923
"version": "8.6.0"
925924
},
926925
"elastic_agent": {
927-
"id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
926+
"id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
928927
"snapshot": false,
929-
"version": "7.17.0"
928+
"version": "8.6.1"
930929
},
931930
"event": {
932931
"agent_id_status": "verified",
933932
"category": [
934933
"network"
935934
],
936935
"dataset": "cloudflare_logpush.http_request",
937-
"ingested": "2022-09-01T10:08:20Z",
936+
"ingested": "2023-03-21T00:21:42Z",
938937
"kind": "event",
939938
"original": "{\"BotScore\":\"20\",\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":\"bing\",\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityLevel\":\"off\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAction\":\"unknown\",\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRuleID\":\"98d93d5\",\"WAFRuleMessage\":\"matchad variable message\",\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}",
940939
"type": [

‎packages/cloudflare_logpush/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 1.0.0
22
name: cloudflare_logpush
33
title: Cloudflare Logpush
4-
version: "0.5.1"
4+
version: "0.6.0"
55
license: basic
66
description: Collect and parse logs from Cloudflare API with Elastic Agent.
77
type: integration

0 commit comments

Comments
 (0)