[akamai.siem] handle empty userRiskData strings

packages/akamai/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.23.1"
+ changes:
+ - description: Fix errors processing empty userRiskData.{risk,trust,general} values.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/9483
 - version: "2.23.0"
  changes:
  - description: Set sensitive values as secret and add missing mappings.

packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log

@@ -1,3 +1,4 @@
 {"format":"json","type":"akamai_siem","version":"1.0","attackData":{"clientIP":"89.160.20.156","configId":"14227","policyId":"qik1_26545","ruleActions":"YWxlcnQ%3d%3bYWxlcnQ%3d%3bZGVueQ%3d%3d","ruleData":"dGVsbmV0LmV4ZQ%3d%3d%3bdGVsbmV0LmV4ZQ%3d%3d%3bVmVjdG9yIFNjb3JlOiAxMCwgREVOWSB0aHJlc2hvbGQ6IDksIEFsZX ","ruleMessages":"U3lzdGVtIENvbW1hbmQgQWNjZXNz%3bU3lzdGVtIENvbW1hbmQgSW5qZWN0aW9u%3bQW5vbWFseSBTY29yZSBFeGNlZWRlZCBmb3 ","ruleSelectors":"QVJHUzpvcHRpb24%3d%3bQVJHUzpvcHRpb24%3d%3b","ruleTags":"T1dBU1BfQ1JTL1dFQl9BVFRBQ0svRklMRV9JTkpFQ1RJT04%3d%3bT1dBU1BfQ1JTL1dFQl9BVFRBQ0svQ09NTUFORF9JTkpFQ1R ","ruleVersions":"NA%3d%3d%3bNA%3d%3d%3bMQ%3d%3d","rules":"OTUwMDAy%3bOTUwMDA2%3bQ01ELUlOSkVDVElPTi1BTk9NQUxZ"},"geo":{"asn":"14618","city":"ASHBURN","continent":"288","country":"US","regionCode":"VA"},"httpMessage":{"bytes":"266","host":"www.hmapi.com","method":"GET","path":"/","port":"80","protocol":"HTTP/1.1","query":"option=com_jce%20telnet.exe","requestHeaders":"User-Agent%3a%20BOT%2f0.1%20(BOT%20for%20JCE)%0d%0aAccept%3a%20text%2fhtml,application%2fxhtml+xml","requestId":"1158db1758e37bfe67b7c09","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20150","start":"1491303422","status":"200"},"userRiskData":{"uuid":"964d54b7-0821-413a-a4d6-8131770ec8d5","status":"0","score":"75","risk":"udfp:1325gdg4g4343g/M|unp:74256/H","trust":"ugp:US","general":"duc_1h:10|duc_1d:30","allow":"0"},"clientData":{"appBundleId":"com.mydomain.myapp","appVersion":"1.23","sdkVersion":"4.7.1","telemetryType":"2"},"botData":{"botScore":"100","responseSegment":"3"}}
+{"format":"json","type":"akamai_siem","version":"1.0","attackData":{"clientIP":"89.160.20.156","configId":"14227","policyId":"qik1_26545","ruleActions":"YWxlcnQ%3d%3bYWxlcnQ%3d%3bZGVueQ%3d%3d","ruleData":"dGVsbmV0LmV4ZQ%3d%3d%3bdGVsbmV0LmV4ZQ%3d%3d%3bVmVjdG9yIFNjb3JlOiAxMCwgREVOWSB0aHJlc2hvbGQ6IDksIEFsZX ","ruleMessages":"U3lzdGVtIENvbW1hbmQgQWNjZXNz%3bU3lzdGVtIENvbW1hbmQgSW5qZWN0aW9u%3bQW5vbWFseSBTY29yZSBFeGNlZWRlZCBmb3 ","ruleSelectors":"QVJHUzpvcHRpb24%3d%3bQVJHUzpvcHRpb24%3d%3b","ruleTags":"T1dBU1BfQ1JTL1dFQl9BVFRBQ0svRklMRV9JTkpFQ1RJT04%3d%3bT1dBU1BfQ1JTL1dFQl9BVFRBQ0svQ09NTUFORF9JTkpFQ1R ","ruleVersions":"NA%3d%3d%3bNA%3d%3d%3bMQ%3d%3d","rules":"OTUwMDAy%3bOTUwMDA2%3bQ01ELUlOSkVDVElPTi1BTk9NQUxZ"},"geo":{"asn":"14618","city":"ASHBURN","continent":"288","country":"US","regionCode":"VA"},"httpMessage":{"bytes":"266","host":"www.hmapi.com","method":"GET","path":"/","port":"80","protocol":"HTTP/1.1","query":"option=com_jce%20telnet.exe","requestHeaders":"User-Agent%3a%20BOT%2f0.1%20(BOT%20for%20JCE)%0d%0aAccept%3a%20text%2fhtml,application%2fxhtml+xml","requestId":"1158db1758e37bfe67b7c09","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20150","start":"1491303422","status":"200"},"userRiskData":{"uuid":"964d54b7-0821-413a-a4d6-8131770ec8d5","status":"0","score":"75","risk":"","trust":"","general":"","allow":"0"},"clientData":{"appBundleId":"com.mydomain.myapp","appVersion":"1.23","sdkVersion":"4.7.1","telemetryType":"2"},"botData":{"botScore":"100","responseSegment":"3"}}
 {"format":"json","type":"akamai_siem","version":"1.0","attackData":{"clientIP":"89.160.20.156","configId":"6724","policyId":"scoe_5426","ruleActions":"QUxFUlQ;REVOWQ==","ruleData":"YWxlcnQo;Y3VybA==","ruleMessages":"Q3Jvc3Mtc2l0ZSBTY3 JpcHRpbmcgKFhTUykgQXR0YWNr; UmVxdWVzdCBJbmRpY2F0ZXMgYW4 gYXV0b21hdGVkIHByb2 dyYW0gZXhwbG9yZWQgdGhlIHNpdGU=","ruleSelectors":"QVJHUzph;UkVRVUVTVF9IRU FERVJTOlVzZXItQWdlbnQ=","ruleTags":"V0VCX0FUVEFDSy9YU1M=;QV VUT01BVElPTi9NSVND","ruleVersions":";","rules":"OTUwMDA0;OTkwMDEx"},"geo":{"asn":"12271","city":"NEWYORK","continent":"NA","country":"US","regionCode":"NY"},"httpMessage":{"bytes":"34523","host":"www.example.com","method":"POST","path":"/examples/1/","port":"80","protocol":"http/2","query":"a%3D..%2F..%2F..%2Fetc%2Fpasswd","requestHeaders":"User-Agent%3a%20BOT%2f0.1%20(BOT%20for%20JCE)%0d%0aAccept%3a%20text%2fhtml,application%2fxhtml+xml","requestId":"2ab418ac8515f33","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml","start":"1470923133.026","status":"301","tls": "TLSv1.2"},"userRiskData":{"uuid":"964d54b7-0821-413a-a4d6-8131770ec8d5","status":"0","score":"75","risk":"udfp:1325gdg4g4343g/M|unp:74256/H","trust":"ugp:US","general":"duc_1h:10|duc_1d:30","allow":"0"},"clientData":{"appBundleId":"com.mydomain.myapp","appVersion":"1.23","sdkVersion":"4.7.1","telemetryType":"2"},"botData":{"botScore":"100","responseSegment":"3"}}
 {"total":10000,"offset":"71cca;3phZmEdPj6YEqml0rvbdWDZGW3mCiJIwjyhkJfsLFM2gVYPgE8-N_0CiLI9gwH0_4OJ87xDQ3b-gIsx_kEBdf7aaC_AvDpG9fMxypeaCma10FKrY9VKE","limit":10000}

packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log-expected.json

@@ -175,6 +175,170 @@
  "query": "option=com_jce telnet.exe"
  }
  },
+ {
+ "@timestamp": "2017-04-04T10:57:02.000Z",
+ "akamai": {
+ "siem": {
+ "bot": {
+ "response_segment": 3,
+ "score": 100
+ },
+ "client_data": {
+ "app_bundle_id": "com.mydomain.myapp",
+ "app_version": "1.23",
+ "sdk_version": "4.7.1",
+ "telemetry_type": 2
+ },
+ "config_id": "14227",
+ "policy_id": "qik1_26545",
+ "request": {
+ "headers": {
+ "Accept": "text/html,application/xhtml xml",
+ "User-Agent": "BOT/0.1 (BOT for JCE)"
+ }
+ },
+ "response": {
+ "headers": {
+ "Content-Length": "150",
+ "Content-Type": "text/html",
+ "Mime-Version": "1.0",
+ "Server": "AkamaiGHost"
+ }
+ },
+ "rule_actions": [
+ "alert",
+ "deny"
+ ],
+ "rule_tags": [
+ "owasp_crs/web_attack/file_injection",
+ "owasp_crs/web_attack/command_inject"
+ ],
+ "rules": [
+ {
+ "ruleActions": "alert",
+ "ruleData": "telnet.exe",
+ "ruleMessages": "System Command Access",
+ "ruleSelectors": "ARGS:option",
+ "ruleTags": "OWASP_CRS/WEB_ATTACK/FILE_INJECTION",
+ "ruleVersions": "4",
+ "rules": "950002"
+ },
+ {
+ "ruleActions": "alert",
+ "ruleData": "telnet.exe",
+ "ruleMessages": "System Command Injection",
+ "ruleSelectors": "ARGS:option",
+ "ruleTags": "OWASP_CRS/WEB_ATTACK/COMMAND_INJECT",
+ "ruleVersions": "4",
+ "rules": "950006"
+ },
+ {
+ "ruleActions": "deny",
+ "ruleData": "Vector Score: 10, DENY threshold: 9, Ale",
+ "ruleMessages": "Anomaly Score Exceeded fo",
+ "ruleVersions": "1",
+ "rules": "CMD-INJECTION-ANOMALY"
+ }
+ ],
+ "user_risk": {
+ "allow": 0,
+ "score": 75,
+ "status": 0,
+ "uuid": "964d54b7-0821-413a-a4d6-8131770ec8d5"
+ }
+ }
+ },
+ "client": {
+ "address": "89.160.20.156",
+ "as": {
+ "number": 29518,
+ "organization": {
+ "name": "Bredband2 AB"
+ }
+ },
+ "geo": {
+ "city_name": "Linköping",
+ "continent_name": "Europe",
+ "country_iso_code": "SE",
+ "country_name": "Sweden",
+ "location": {
+ "lat": 58.4167,
+ "lon": 15.6167
+ },
+ "region_iso_code": "SE-E",
+ "region_name": "Östergötland County"
+ },
+ "ip": "89.160.20.156"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "category": [
+ "network"
+ ],
+ "id": "1158db1758e37bfe67b7c09",
+ "kind": "event",
+ "original": "{\"format\":\"json\",\"type\":\"akamai_siem\",\"version\":\"1.0\",\"attackData\":{\"clientIP\":\"89.160.20.156\",\"configId\":\"14227\",\"policyId\":\"qik1_26545\",\"ruleActions\":\"YWxlcnQ%3d%3bYWxlcnQ%3d%3bZGVueQ%3d%3d\",\"ruleData\":\"dGVsbmV0LmV4ZQ%3d%3d%3bdGVsbmV0LmV4ZQ%3d%3d%3bVmVjdG9yIFNjb3JlOiAxMCwgREVOWSB0aHJlc2hvbGQ6IDksIEFsZX \",\"ruleMessages\":\"U3lzdGVtIENvbW1hbmQgQWNjZXNz%3bU3lzdGVtIENvbW1hbmQgSW5qZWN0aW9u%3bQW5vbWFseSBTY29yZSBFeGNlZWRlZCBmb3 \",\"ruleSelectors\":\"QVJHUzpvcHRpb24%3d%3bQVJHUzpvcHRpb24%3d%3b\",\"ruleTags\":\"T1dBU1BfQ1JTL1dFQl9BVFRBQ0svRklMRV9JTkpFQ1RJT04%3d%3bT1dBU1BfQ1JTL1dFQl9BVFRBQ0svQ09NTUFORF9JTkpFQ1R \",\"ruleVersions\":\"NA%3d%3d%3bNA%3d%3d%3bMQ%3d%3d\",\"rules\":\"OTUwMDAy%3bOTUwMDA2%3bQ01ELUlOSkVDVElPTi1BTk9NQUxZ\"},\"geo\":{\"asn\":\"14618\",\"city\":\"ASHBURN\",\"continent\":\"288\",\"country\":\"US\",\"regionCode\":\"VA\"},\"httpMessage\":{\"bytes\":\"266\",\"host\":\"www.hmapi.com\",\"method\":\"GET\",\"path\":\"/\",\"port\":\"80\",\"protocol\":\"HTTP/1.1\",\"query\":\"option=com_jce%20telnet.exe\",\"requestHeaders\":\"User-Agent%3a%20BOT%2f0.1%20(BOT%20for%20JCE)%0d%0aAccept%3a%20text%2fhtml,application%2fxhtml+xml\",\"requestId\":\"1158db1758e37bfe67b7c09\",\"responseHeaders\":\"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20150\",\"start\":\"1491303422\",\"status\":\"200\"},\"userRiskData\":{\"uuid\":\"964d54b7-0821-413a-a4d6-8131770ec8d5\",\"status\":\"0\",\"score\":\"75\",\"risk\":\"\",\"trust\":\"\",\"general\":\"\",\"allow\":\"0\"},\"clientData\":{\"appBundleId\":\"com.mydomain.myapp\",\"appVersion\":\"1.23\",\"sdkVersion\":\"4.7.1\",\"telemetryType\":\"2\"},\"botData\":{\"botScore\":\"100\",\"responseSegment\":\"3\"}}",
+ "start": "2017-04-04T10:57:02.000Z"
+ },
+ "http": {
+ "request": {
+ "id": "1158db1758e37bfe67b7c09",
+ "method": "GET"
+ },
+ "response": {
+ "bytes": 266,
+ "status_code": 200
+ },
+ "version": "1.1"
+ },
+ "network": {
+ "protocol": "http",
+ "transport": "tcp"
+ },
+ "observer": {
+ "type": "proxy",
+ "vendor": "akamai"
+ },
+ "related": {
+ "ip": [
+ "89.160.20.156"
+ ]
+ },
+ "source": {
+ "address": "89.160.20.156",
+ "as": {
+ "number": 29518,
+ "organization": {
+ "name": "Bredband2 AB"
+ }
+ },
+ "geo": {
+ "city_name": "Linköping",
+ "continent_name": "Europe",
+ "country_iso_code": "SE",
+ "country_name": "Sweden",
+ "location": {
+ "lat": 58.4167,
+ "lon": 15.6167
+ },
+ "region_iso_code": "SE-E",
+ "region_name": "Östergötland County"
+ },
+ "ip": "89.160.20.156"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "url": {
+ "domain": "www.hmapi.com",
+ "full": "www.hmapi.com/?option=com_jce%20telnet.exe",
+ "path": "/",
+ "port": 80,
+ "query": "option=com_jce telnet.exe"
+ }
+ },
  {
  "@timestamp": "2016-08-11T13:45:33.026Z",
  "akamai": {

packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml

@@ -67,10 +67,12 @@ processors:
  target_field: url.domain
  ignore_missing: true
  - urldecode:
+ tag: urldecode_httpMessage_path
  field: json.httpMessage.path
  target_field: url.path
  ignore_missing: true
  - urldecode:
+ tag: urldecode_httpMessage_query
  field: json.httpMessage.query
  target_field: url.query
  ignore_missing: true
@@ -83,6 +85,7 @@ processors:
  type: long
  ignore_missing: true
  - urldecode:
+ tag: urldecode_httpMessage_responseHeaders
  field: json.httpMessage.responseHeaders
  target_field: _tmp.response.headers
  ignore_missing: true
@@ -93,6 +96,7 @@ processors:
  value_split: ': '
  ignore_missing: true
  - urldecode:
+ tag: urldecode_httpMessage_requestHeaders
  field: json.httpMessage.requestHeaders
  target_field: _tmp.request.headers
  ignore_missing: true
@@ -194,6 +198,7 @@ processors:
  ignore_missing: true
  ## Attack Data
  - urldecode:
+ tag: urldecode_attackData_ruleActions
  field: json.attackData.ruleActions
  target_field: json.attackData.ruleActions
  ignore_missing: true
@@ -203,6 +208,7 @@ processors:
  separator: ';'
  preserve_trailing: true
  - urldecode:
+ tag: urldecode_attackData_ruleData
  field: json.attackData.ruleData
  target_field: json.attackData.ruleData
  ignore_missing: true
@@ -212,6 +218,7 @@ processors:
  separator: ';'
  preserve_trailing: true
  - urldecode:
+ tag: urldecode_attackData_ruleMessages
  field: json.attackData.ruleMessages
  target_field: json.attackData.ruleMessages
  ignore_missing: true
@@ -221,6 +228,7 @@ processors:
  separator: ';'
  preserve_trailing: true
  - urldecode:
+ tag: urldecode_attackData_ruleSelectors
  field: json.attackData.ruleSelectors
  target_field: json.attackData.ruleSelectors
  ignore_missing: true
@@ -230,6 +238,7 @@ processors:
  separator: ';'
  preserve_trailing: true
  - urldecode:
+ tag: urldecode_attackData_ruleTags
  field: json.attackData.ruleTags
  target_field: json.attackData.ruleTags
  ignore_missing: true
@@ -239,6 +248,7 @@ processors:
  separator: ';'
  preserve_trailing: true
  - urldecode:
+ tag: urldecode_attackData_ruleVersions
  field: json.attackData.ruleVersions
  target_field: json.attackData.ruleVersions
  ignore_missing: true
@@ -248,6 +258,7 @@ processors:
  separator: ';'
  preserve_trailing: true
  - urldecode:
+ tag: urldecode_attackData_rules
  field: json.attackData.rules
  target_field: json.attackData.rules
  ignore_missing: true
@@ -259,6 +270,7 @@ processors:
  - script:
  lang: painless
  description: Base64 Decode the json.attackData.rule* fields
+ tag: script_base64_decode_attackData_rule
  source: |
  ArrayList items = new ArrayList(["rules", "ruleActions", "ruleData", "ruleMessages", "ruleTags", "ruleSelectors", "ruleVersions"]);
  ArrayList rules_array = new ArrayList();
@@ -386,18 +398,24 @@ processors:
  type: long
  ignore_missing: true
  - kv:
+ if: ctx.json?.userRiskData?.risk != ""
+ tag: kv_userRiskData_risk
  field: json.userRiskData.risk
  target_field: akamai.siem.user_risk.risk
  field_split: '\|'
  value_split: ':'
  ignore_missing: true
  - kv:
+ if: ctx.json?.userRiskData?.trust != ""
+ tag: kv_userRiskData_trust
  field: json.userRiskData.trust
  target_field: akamai.siem.user_risk.trust
  field_split: '\|'
  value_split: ':'
  ignore_missing: true
  - kv:
+ if: ctx.json?.userRiskData?.general != ""
+ tag: kv_userRiskData_general
  field: json.userRiskData.general
  target_field: akamai.siem.user_risk.general
  field_split: '\|'
@@ -458,4 +476,7 @@ on_failure:
  value: pipeline_error
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{ _ingest.on_failure_processor_type }}'
+ {{#_ingest.on_failure_processor_tag}}with tag '{{ _ingest.on_failure_processor_tag }}'
+ {{/_ingest.on_failure_processor_tag}}failed with message '{{ _ingest.on_failure_message }}'

packages/akamai/manifest.yml

@@ -1,6 +1,6 @@
 name: akamai
 title: Akamai
-version: "2.23.0"
+version: "2.23.1"
 description: Collect logs from Akamai with Elastic Agent.
 type: integration
 format_version: "3.0.2"