Microsoft M365 Defender Responce.Split issue #5147
Description
Hi Team,
In the Microsoft M365 Defender integration we have found an issue where the empty valued events are getting thru to elastic without getting dropped and making the index flooded with. when we investigate we found that in Elastic version 8.6.0 there has been a change in
response.split parameter where it keeps the parent document if the split target is empty. in order to drop the empty valued events we need to set the ignore_empty_value to true
where it did not required such in 8.5.x or less versions
Integration version: 1.5.0
Response from the API
{"@odata.context":"https://api.security.microsoft.com/api/$metadata#Incidents","value":[]} example output:
{ "_index": ".ds-logs-m365_defender.log-xyz-2023.01.24-000001", "_id": "VPy164UBrQd1IU94z6fJ", "_version": 1, "_score": 0, "_source": { "cloud": { "availability_zone": "ap-southeast-2a", "image": { "id": "ami-073cb2577fd69dc18" }, "instance": { "id": "i-02ba13ddb5de36695" }, "provider": "azure", "machine": { "type": "r5.xlarge" }, "service": { "name": "EC2" }, "region": "ap-southeast-2", "account": { "id": "237400951785" } }, "input": { "type": "httpjson" }, "observer": { "product": "365 Defender", "vendor": "Microsoft" }, "agent": { "name": "elastic-agent-xyz-1", "id": "0d93a8bc-ee0c-43ac-9e9c-11e3ebeb00d5", "ephemeral_id": "0fae8b22-5544-449d-905f-a4b55f563f64", "type": "filebeat", "version": "8.6.0" }, "@timestamp": "2023-01-26T01:33:58.102Z", "ecs": { "version": "8.6.0" }, "data_stream": { "namespace": "xyz", "type": "logs", "dataset": "m365_defender.log" }, "elastic_agent": { "id": "0d93a8bc-ee0c-43ac-9e9c-11e3ebeb00d5", "version": "8.6.0", "snapshot": false }, "event": { "agent_id_status": "verified", "ingested": "2023-01-26T01:33:59Z", "original": "{\"@odata.context\":\"https://api.security.microsoft.com/api/$metadata#Incidents\",\"value\":[]}", "created": "2023-01-26T01:33:58.102Z", "kind": "alert", "timezone": "UTC", "category": [ "host" ], "dataset": "m365_defender.log" }, "m365_defender": { "@odata.context": "https://api.security.microsoft.com/api/$metadata#Incidents", "value": [] }, "tags": [ "preserve_original_event", "m365_defender", "forwarded" ] }, "fields": { "event.category": [ "host" ], "elastic_agent.version": [ "8.6.0" ], "cloud.availability_zone": [ "ap-southeast-2a" ], "observer.vendor": [ "Microsoft" ], "cloud.instance.id": [ "i-02ba13ddb5de36695" ], "agent.type": [ "filebeat" ], "event.module": [ "m365_defender" ], "observer.product": [ "365 Defender" ], "agent.name": [ "elastic-agent-xyz-1" ], "elastic_agent.snapshot": [ false ], "event.agent_id_status": [ "verified" ], "event.kind": [ "alert" ], "event.timezone": [ "UTC" ], "m365_defender.@odata.context": [ "https://api.security.microsoft.com/api/$metadata#Incidents" ], "event.original": [ "{\"@odata.context\":\"https://api.security.microsoft.com/api/$metadata#Incidents\",\"value\":[]}" ], "cloud.region": [ "ap-southeast-2" ], "elastic_agent.id": [ "0d93a8bc-ee0c-43ac-9e9c-11e3ebeb00d5" ], "data_stream.namespace": [ "xyz" ], "input.type": [ "httpjson" ], "data_stream.type": [ "logs" ], "tags": [ "preserve_original_event", "m365_defender", "forwarded" ], "cloud.image.id": [ "ami-073cb2577fd69dc18" ], "cloud.machine.type": [ "r5.xlarge" ], "event.ingested": [ "2023-01-26T01:33:59Z" ], "cloud.provider": [ "azure" ], "@timestamp": [ "2023-01-26T01:33:58.102Z" ], "cloud.service.name": [ "EC2" ], "agent.id": [ "0d93a8bc-ee0c-43ac-9e9c-11e3ebeb00d5" ], "cloud.account.id": [ "237400951785" ], "ecs.version": [ "8.6.0" ], "data_stream.dataset": [ "m365_defender.log" ], "event.created": [ "2023-01-26T01:33:58.102Z" ], "agent.ephemeral_id": [ "0fae8b22-5544-449d-905f-a4b55f563f64" ], "agent.version": [ "8.6.0" ], "event.dataset": [ "m365_defender.log" ] } } The fixing needs to be done on the below files in the integration
packages/m365_defender/data_stream/log/agent/stream/httpjson.yml.hbs