m365_defender,microsoft_defender_endpoint: drop empty event sets

Author: efd6

packages/m365_defender/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.1"
+ changes:
+ - description: Drop empty event sets in log data stream.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/5164
 - version: "1.5.0"
  changes:
  - description: Update package to ECS 8.6.0.

packages/m365_defender/data_stream/log/_dev/test/pipeline/test-m365-defender-empty-ndjson.log

@@ -1 +1,2 @@
 {"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}
+{"value":[]}

packages/m365_defender/data_stream/log/_dev/test/pipeline/test-m365-defender-empty-ndjson.log-expected.json

@@ -1,5 +1,6 @@
 {
  "expected": [
+ null,
  null
  ]
 }

packages/m365_defender/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -13,7 +13,7 @@ processors:
  target_field: json
  - drop:
  description: Drops duplicated events without alerts
- if: ctx.json?.status == 'Redirected'
+ if: ctx.json?.status == 'Redirected' || (ctx.json?.value != null && ctx.json.value.isEmpty())
  - remove:
  field:
  - json.alerts

packages/m365_defender/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: m365_defender
 title: Microsoft M365 Defender
-version: "1.5.0"
+version: "1.5.1"
 description: Collect logs from Microsoft M365 Defender with Elastic Agent.
 categories:
  - "network"

packages/microsoft_defender_endpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.8.1"
+ changes:
+ - description: Drop empty event sets.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/5164
 - version: "2.8.0"
  changes:
  - description: Adding support for Oauth2 scopes that is required for some users

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log

@@ -2,3 +2,4 @@
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"543bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"123543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"Process","sha1":"b6d237154f2e528f0b503b58b025862d66b02b73","sha256":"a92056d772260b39a876d01552496b2f8b4610a0b1e084952fe1176784e2ce77","fileName":"notepad.exe","filePath":"C:\\Windows\\System32","processId":4104,"processCommandLine":"\"notepad.exe\"","processCreationTime":"2020-06-30T09:45:38.9784654Z","parentProcessId":6012,"parentProcessCreationTime":"2020-06-30T09:04:51.487396Z","ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"53425a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"43521344-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"User","sha1":null,"sha256":null,"fileName":null,"filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":"administrator1","domainName":"TestServer4","userSid":"S-1-5-21-46152456-1367606905-4031241297-500","aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291063515066999_-2102938302","incidentId":12,"investigationId":9,"assignedTo":"Automation","severity":"Informational","status":"Resolved","classification":null,"determination":null,"investigationState":"Benign","detectionSource":"WindowsDefenderAv","category":"Malware","threatFamilyName":null,"title":"'Mountsi' malware was detected","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","alertCreationTime":"2020-06-30T09:32:31.4579225Z","firstEventTime":"2020-06-30T09:31:22.5729558Z","lastEventTime":"2020-06-30T09:46:15.0876676Z","lastUpdateTime":"2020-06-30T11:13:12.9Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","machineId":"t4563234bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"1234543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":null,"comments":[],"evidence":{"entityType":"File","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
+{"value":[]}

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log-expected.json

@@ -350,6 +350,7 @@
  "name": "Malware"
  }
  }
- }
+ },
+ null
  ]
 }

packages/microsoft_defender_endpoint/data_stream/log/agent/stream/httpjson.yml.hbs

@@ -30,6 +30,7 @@ request.transforms:
  default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05.9999999Z"]]'
 response.split:
  target: body.value
+ ignore_empty_value: true
  split:
  target: body.evidence
  keep_parent: true

packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -11,6 +11,8 @@ processors:
  - json:
  field: event.original
  target_field: json
+ - drop:
+ if: ctx.json?.value != null && ctx.json.value.isEmpty()
  - remove:
  field:
  - json.comments