Skip to content

Commit 9782d4b

Browse files
efd6efd6
authored
cloudflare: fix handling of timestamps in unix (seconds) format (#7763)
Cloudflare provides most timestamps configurably as unixnano (default), unix, or RFC3339. Elasticsearch's date processor allows specification of unix, unix millis and via format string, RFC3339. Previously we were assuming either unixnano or RFC3999 since we could distinguish these. However, when a user had specified unix timestamps in their Cloudflare configuration we were treating the timestamp as unix millis, resulting in dates in the past. This change attempts to convert timestamps to unix millis, either downscaling unix nanos, or upscaling unix, if the timestamp is a long. The heuristic used to determine whether a timestamp is unix nanos is that it is greater than 1e18. A timestamp is unix if is less than 1e10. All others are treated as unix millis, though according to the Cloudflare docs these don't exist. Note that some timestamp-like fields exist that do not follow the rules here. They have an "ms" suffix and are milliseconds.
1 parent 7308338 commit 9782d4b

File tree

5 files changed

+254
-22
lines changed

5 files changed

+254
-22
lines changed

‎packages/cloudflare/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "2.14.1"
3+
changes:
4+
- description: Fix handling of unix timestamps.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/7763
27
- version: "2.14.0"
38
changes:
49
- description: Update package-spec to 2.10.0.

‎packages/cloudflare/data_stream/logpull/_dev/test/pipeline/test-http-json.log‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
{"CacheCacheStatus":"unknown","CacheResponseBytes":0,"CacheResponseStatus":0,"CacheTieredFill":false,"ClientASN":15169,"ClientCountry":"us","ClientDeviceType":"desktop","ClientIP":"89.160.20.156","ClientIPClass":"noRecord","ClientRequestBytes":2577,"ClientRequestHost":"cf-analytics.com","ClientRequestMethod":"POST","ClientRequestPath":"/wp-cron.php","ClientRequestProtocol":"HTTP/1.1","ClientRequestReferer":"https://cf-analytics.com/wp-cron.php?doing_wp_cron=1564759748.3962020874023437500000","ClientRequestURI":"/wp-cron.php?doing_wp_cron=1564759748.3962020874023437500000","ClientRequestUserAgent":"WordPress/5.2.2;https://cf-analytics.com","ClientSSLCipher":"ECDHE-ECDSA-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.2","ClientSrcPort":55028,"EdgeColoID":14,"EdgeEndTimestamp":"2019-08-02T15:29:08Z","EdgePathingOp":"ban","EdgePathingSrc":"filter_based_firewall","EdgePathingStatus":"captchaNew","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"","EdgeResponseBytes":2848,"EdgeResponseCompressionRatio":2.64,"EdgeResponseContentType":"text/html","EdgeResponseStatus":403,"EdgeServerIP":"","EdgeStartTimestamp":"2019-08-02T15:29:08Z","FirewallMatchesActions":["simulate","challenge"],"FirewallMatchesSources":["firewallRules","firewallRules"],"FirewallMatchesRuleIDs":["094b71fea25d4860a61fa0c6fbbd8d8b","e454fd4a0ce546b3a9a462536613692c"],"OriginIP":"","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"00","RayID":"500115ec386354d8","SecurityLevel":"med","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":155978002}
22
{"CacheCacheStatus":"hit","CacheResponseBytes":26888,"CacheResponseStatus":200,"CacheTieredFill":true,"ClientASN":1136,"ClientCountry":"nl","ClientDeviceType":"desktop","ClientIP":"89.160.20.156","ClientIPClass":"noRecord","ClientRequestBytes":5324,"ClientRequestHost":"eqlplayground.io","ClientRequestMethod":"GET","ClientRequestPath":"/40865/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.9.js","ClientRequestProtocol":"HTTP/1.1","ClientRequestReferer":"https://eqlplayground.io/s/eqldemo/app/security/timelines/default?sourcerer=(default:!(.siem-signals-eqldemo))&timerange=(global:(linkTo:!(),timerange:(from:%272021-03-03T19:55:15.519Z%27,fromStr:now-24h,kind:relative,to:%272021-03-04T19:55:15.519Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272020-03-04T19:55:28.684Z%27,fromStr:now-1y,kind:relative,to:%272021-03-04T19:55:28.692Z%27,toStr:now)))&timeline=(activeTab:eql,graphEventId:%27%27,id:%2769f93840-7d23-11eb-866c-79a0609409ba%27,isOpen:!t)","ClientRequestURI":"/40865/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.9.js","ClientRequestUserAgent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36","ClientSSLCipher":"NONE","ClientSSLProtocol":"none","ClientSrcPort":0,"ClientXRequestedWith":"","EdgeColoCode":"33.147.138.217","EdgeColoID":20,"EdgeEndTimestamp":1625752958875000000,"EdgePathingOp":"wl","EdgePathingSrc":"macro","EdgePathingStatus":"nr","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"eqlplayground.io","EdgeResponseBytes":24743,"EdgeResponseCompressionRatio":0,"EdgeResponseContentType":"application/javascript","EdgeResponseStatus":200,"EdgeServerIP":"89.160.20.156","EdgeStartTimestamp":1625752958812000000,"FirewallMatchesActions":[],"FirewallMatchesRuleIDs":[],"FirewallMatchesSources":[],"OriginIP":"","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"66b9d9f88b5b4c4f","RayID":"66b9d9f890ae4c4f","SecurityLevel":"off","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":true,"WorkerSubrequestCount":0,"ZoneID":393347122}
3-
{"CacheCacheStatus":"unknown","CacheResponseBytes":0,"CacheResponseStatus":0,"CacheTieredFill":false,"ClientASN":1136,"ClientCountry":"nl","ClientDeviceType":"desktop","ClientIP":"89.160.20.156","ClientIPClass":"noRecord","ClientRequestBytes":2520,"ClientRequestHost":"eqlplayground.io","ClientRequestMethod":"GET","ClientRequestPath":"/s/eqldemo/security/account","ClientRequestProtocol":"HTTP/2","ClientRequestReferer":"","ClientRequestURI":"/s/eqldemo/security/account","ClientRequestUserAgent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36","ClientSSLCipher":"AEAD-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":61593,"ClientXRequestedWith":"","EdgeColoCode":"AMS","EdgeColoID":20,"EdgeEndTimestamp":1625754264684000000,"EdgePathingOp":"ban","EdgePathingSrc":"filter_based_firewall","EdgePathingStatus":"nr","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"183.53.30.34","EdgeResponseBytes":2066,"EdgeResponseCompressionRatio":2.45,"EdgeResponseContentType":"text/html","EdgeResponseStatus":403,"EdgeServerIP":"","EdgeStartTimestamp":1625754264676000000,"FirewallMatchesActions":["block"],"FirewallMatchesRuleIDs":["391eb601201e4f2a81038910f2b63f6d"],"FirewallMatchesSources":["firewallRules"],"OriginIP":"89.160.20.156","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"00","RayID":"66b9f9da396e4c01","SecurityLevel":"unk","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":393347122}
3+
{"CacheCacheStatus":"unknown","CacheResponseBytes":0,"CacheResponseStatus":0,"CacheTieredFill":false,"ClientASN":1136,"ClientCountry":"nl","ClientDeviceType":"desktop","ClientIP":"89.160.20.156","ClientIPClass":"noRecord","ClientRequestBytes":2520,"ClientRequestHost":"eqlplayground.io","ClientRequestMethod":"GET","ClientRequestPath":"/s/eqldemo/security/account","ClientRequestProtocol":"HTTP/2","ClientRequestReferer":"","ClientRequestURI":"/s/eqldemo/security/account","ClientRequestUserAgent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36","ClientSSLCipher":"AEAD-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":61593,"ClientXRequestedWith":"","EdgeColoCode":"AMS","EdgeColoID":20,"EdgeEndTimestamp":1625754264684000000,"EdgePathingOp":"ban","EdgePathingSrc":"filter_based_firewall","EdgePathingStatus":"nr","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"183.53.30.34","EdgeResponseBytes":2066,"EdgeResponseCompressionRatio":2.45,"EdgeResponseContentType":"text/html","EdgeResponseStatus":403,"EdgeServerIP":"","EdgeStartTimestamp":1625754264676000000,"FirewallMatchesActions":["block"],"FirewallMatchesRuleIDs":["391eb601201e4f2a81038910f2b63f6d"],"FirewallMatchesSources":["firewallRules"],"OriginIP":"89.160.20.156","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"00","RayID":"66b9f9da396e4c01","SecurityLevel":"unk","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":393347122}
4+
{"CacheCacheStatus":"unknown","CacheResponseBytes":0,"CacheResponseStatus":0,"CacheTieredFill":false,"ClientASN":1136,"ClientCountry":"nl","ClientDeviceType":"desktop","ClientIP":"89.160.20.156","ClientIPClass":"noRecord","ClientRequestBytes":2520,"ClientRequestHost":"eqlplayground.io","ClientRequestMethod":"GET","ClientRequestPath":"/s/eqldemo/security/account","ClientRequestProtocol":"HTTP/2","ClientRequestReferer":"","ClientRequestURI":"/s/eqldemo/security/account","ClientRequestUserAgent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36","ClientSSLCipher":"AEAD-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":61593,"ClientXRequestedWith":"","EdgeColoCode":"AMS","EdgeColoID":20,"EdgeEndTimestamp":1625754264,"EdgePathingOp":"ban","EdgePathingSrc":"filter_based_firewall","EdgePathingStatus":"nr","EdgeRateLimitAction":"","EdgeRateLimitID":0,"EdgeRequestHost":"183.53.30.34","EdgeResponseBytes":2066,"EdgeResponseCompressionRatio":2.45,"EdgeResponseContentType":"text/html","EdgeResponseStatus":403,"EdgeServerIP":"","EdgeStartTimestamp":1625754264,"FirewallMatchesActions":["block"],"FirewallMatchesRuleIDs":["391eb601201e4f2a81038910f2b63f6d"],"FirewallMatchesSources":["firewallRules"],"OriginIP":"89.160.20.156","OriginResponseBytes":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseStatus":0,"OriginResponseTime":0,"OriginSSLProtocol":"unknown","ParentRayID":"00","RayID":"66b9f9da396e4c01","SecurityLevel":"unk","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":393347122}

‎packages/cloudflare/data_stream/logpull/_dev/test/pipeline/test-http-json.log-expected.json‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -592,6 +592,210 @@
592592
},
593593
"version": "91.0.4472.124"
594594
}
595+
},
596+
{
597+
"@timestamp": "2021-07-08T14:24:24.000Z",
598+
"client": {
599+
"address": "89.160.20.156",
600+
"as": {
601+
"number": 29518,
602+
"organization": {
603+
"name": "Bredband2 AB"
604+
}
605+
},
606+
"bytes": 2520,
607+
"geo": {
608+
"city_name": "Linköping",
609+
"continent_name": "Europe",
610+
"country_iso_code": "SE",
611+
"country_name": "Sweden",
612+
"location": {
613+
"lat": 58.4167,
614+
"lon": 15.6167
615+
},
616+
"region_iso_code": "SE-E",
617+
"region_name": "Östergötland County"
618+
},
619+
"ip": "89.160.20.156",
620+
"port": 61593
621+
},
622+
"cloudflare": {
623+
"cache": {
624+
"status": "unknown",
625+
"tiered_fill": false
626+
},
627+
"client": {
628+
"ip_class": "noRecord",
629+
"ssl": {
630+
"protocol": "TLSv1.3"
631+
}
632+
},
633+
"device_type": "desktop",
634+
"edge": {
635+
"colo": {
636+
"code": "AMS",
637+
"id": 20
638+
},
639+
"pathing": {
640+
"op": "ban",
641+
"src": "filter_based_firewall",
642+
"status": "nr"
643+
},
644+
"rate_limit": {
645+
"id": 0
646+
},
647+
"request": {
648+
"host": "183.53.30.34"
649+
},
650+
"response": {
651+
"bytes": 2066,
652+
"compression_ratio": 2.45,
653+
"content_type": "text/html",
654+
"status_code": 403
655+
}
656+
},
657+
"firewall": {
658+
"actions": [
659+
"block"
660+
],
661+
"rule_ids": [
662+
"391eb601201e4f2a81038910f2b63f6d"
663+
],
664+
"sources": [
665+
"firewallRules"
666+
]
667+
},
668+
"origin": {
669+
"response": {
670+
"bytes": 0,
671+
"status_code": 0,
672+
"time": 0
673+
},
674+
"ssl": {
675+
"protocol": "unknown"
676+
}
677+
},
678+
"parent": {
679+
"ray_id": "00"
680+
},
681+
"ray_id": "66b9f9da396e4c01",
682+
"security_level": "unk",
683+
"waf": {
684+
"action": "unknown",
685+
"flags": "0",
686+
"profile": "unknown"
687+
},
688+
"worker": {
689+
"cpu_time": 0,
690+
"status": "unknown",
691+
"subrequest": false,
692+
"subrequest_count": 0
693+
},
694+
"zone": {
695+
"id": 393347122
696+
}
697+
},
698+
"destination": {
699+
"address": "89.160.20.156",
700+
"bytes": 2066,
701+
"ip": "89.160.20.156"
702+
},
703+
"ecs": {
704+
"version": "8.9.0"
705+
},
706+
"event": {
707+
"action": [
708+
"block"
709+
],
710+
"category": [
711+
"network"
712+
],
713+
"duration": 0,
714+
"end": "2021-07-08T14:24:24.000Z",
715+
"kind": "event",
716+
"original": "{\"CacheCacheStatus\":\"unknown\",\"CacheResponseBytes\":0,\"CacheResponseStatus\":0,\"CacheTieredFill\":false,\"ClientASN\":1136,\"ClientCountry\":\"nl\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"89.160.20.156\",\"ClientIPClass\":\"noRecord\",\"ClientRequestBytes\":2520,\"ClientRequestHost\":\"eqlplayground.io\",\"ClientRequestMethod\":\"GET\",\"ClientRequestPath\":\"/s/eqldemo/security/account\",\"ClientRequestProtocol\":\"HTTP/2\",\"ClientRequestReferer\":\"\",\"ClientRequestURI\":\"/s/eqldemo/security/account\",\"ClientRequestUserAgent\":\"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36\",\"ClientSSLCipher\":\"AEAD-AES128-GCM-SHA256\",\"ClientSSLProtocol\":\"TLSv1.3\",\"ClientSrcPort\":61593,\"ClientXRequestedWith\":\"\",\"EdgeColoCode\":\"AMS\",\"EdgeColoID\":20,\"EdgeEndTimestamp\":1625754264,\"EdgePathingOp\":\"ban\",\"EdgePathingSrc\":\"filter_based_firewall\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"183.53.30.34\",\"EdgeResponseBytes\":2066,\"EdgeResponseCompressionRatio\":2.45,\"EdgeResponseContentType\":\"text/html\",\"EdgeResponseStatus\":403,\"EdgeServerIP\":\"\",\"EdgeStartTimestamp\":1625754264,\"FirewallMatchesActions\":[\"block\"],\"FirewallMatchesRuleIDs\":[\"391eb601201e4f2a81038910f2b63f6d\"],\"FirewallMatchesSources\":[\"firewallRules\"],\"OriginIP\":\"89.160.20.156\",\"OriginResponseBytes\":0,\"OriginResponseHTTPExpires\":\"\",\"OriginResponseHTTPLastModified\":\"\",\"OriginResponseStatus\":0,\"OriginResponseTime\":0,\"OriginSSLProtocol\":\"unknown\",\"ParentRayID\":\"00\",\"RayID\":\"66b9f9da396e4c01\",\"SecurityLevel\":\"unk\",\"WAFAction\":\"unknown\",\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"\",\"WAFProfile\":\"unknown\",\"WAFRuleID\":\"\",\"WAFRuleMessage\":\"\",\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":false,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122}",
717+
"start": "2021-07-08T14:24:24.000Z",
718+
"type": [
719+
"denied"
720+
]
721+
},
722+
"http": {
723+
"request": {
724+
"bytes": 2520,
725+
"method": "GET"
726+
},
727+
"response": {
728+
"bytes": 2066,
729+
"status_code": 403
730+
},
731+
"version": "2"
732+
},
733+
"network": {
734+
"bytes": 4586,
735+
"protocol": "http",
736+
"transport": "tcp"
737+
},
738+
"observer": {
739+
"type": "proxy",
740+
"vendor": "cloudflare"
741+
},
742+
"server": {
743+
"address": "89.160.20.156",
744+
"bytes": 2066,
745+
"ip": "89.160.20.156"
746+
},
747+
"source": {
748+
"address": "89.160.20.156",
749+
"as": {
750+
"number": 29518,
751+
"organization": {
752+
"name": "Bredband2 AB"
753+
}
754+
},
755+
"bytes": 2520,
756+
"geo": {
757+
"city_name": "Linköping",
758+
"continent_name": "Europe",
759+
"country_iso_code": "SE",
760+
"country_name": "Sweden",
761+
"location": {
762+
"lat": 58.4167,
763+
"lon": 15.6167
764+
},
765+
"region_iso_code": "SE-E",
766+
"region_name": "Östergötland County"
767+
},
768+
"ip": "89.160.20.156",
769+
"port": 61593
770+
},
771+
"tags": [
772+
"preserve_original_event"
773+
],
774+
"tls": {
775+
"cipher": "AEAD-AES128-GCM-SHA256",
776+
"version": "1.3",
777+
"version_protocol": "tls"
778+
},
779+
"url": {
780+
"domain": "eqlplayground.io",
781+
"full": "https://eqlplayground.io/s/eqldemo/security/account",
782+
"original": "/s/eqldemo/security/account",
783+
"path": "/s/eqldemo/security/account",
784+
"scheme": "https"
785+
},
786+
"user_agent": {
787+
"device": {
788+
"name": "Other"
789+
},
790+
"name": "Chrome",
791+
"original": "Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.124Safari/537.36",
792+
"os": {
793+
"full": "Windows $1",
794+
"name": "Windows",
795+
"version": "$1"
796+
},
797+
"version": "91.0.4472.124"
798+
}
595799
}
596800
]
597801
}

0 commit comments

Comments
 (0)