Upgrade ECS to 8.0.0

Author: r00tu53r

packages/microsoft_defender_endpoint/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
  ecs:
- reference: git@1.12
+ reference: git@8.0

packages/microsoft_defender_endpoint/_dev/build/docs/README.md

@@ -38,8 +38,6 @@ These values are:
 | id | event.id |
 | lastEventTime | event.end |
 | machineId | cloud.instance.id |
-| relatedUser.userName | host.user.name |
-| relatedUser.domainName | host.user.domain |
 | title | message |
 | severity | event.severity |
 

packages/microsoft_defender_endpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.0.0"
+ changes:
+ - description: Update to ECS 8.0
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/2422
 - version: "1.1.0"
  changes:
  - description: Add 8.0.0 version constraint

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log-expected.json

@@ -41,7 +41,7 @@
  "path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5"
  },
  "ecs": {
- "version": "1.12.0"
+ "version": "8.0.0"
  },
  "related": {
  "hosts": [
@@ -69,7 +69,7 @@
  "end"
  ],
  "duration": 0,
- "ingested": "2021-06-09T11:57:25.803509100Z",
+ "ingested": "2022-01-02T01:28:49.601370210Z",
  "provider": "defender_endpoint",
  "action": "Malware",
  "end": "2020-06-30T10:07:44.333733Z",
@@ -132,7 +132,7 @@
  }
  },
  "ecs": {
- "version": "1.12.0"
+ "version": "8.0.0"
  },
  "related": {
  "user": [
@@ -148,10 +148,6 @@
  },
  "host": {
  "name": "testserver4",
- "user": {
- "name": "administrator1",
- "domain": "TestServer4"
- },
  "hostname": "testserver4"
  },
  "threat": {
@@ -172,7 +168,7 @@
  "start"
  ],
  "duration": 2442699369800,
- "ingested": "2021-06-09T11:57:25.803531Z",
+ "ingested": "2022-01-02T01:28:49.601372227Z",
  "provider": "defender_endpoint",
  "action": "DefenseEvasion",
  "end": "2020-06-30T09:45:39.5484377Z",
@@ -224,7 +220,7 @@
  "vendor": "Microsoft"
  },
  "ecs": {
- "version": "1.12.0"
+ "version": "8.0.0"
  },
  "related": {
  "user": [
@@ -236,11 +232,6 @@
  },
  "host": {
  "name": "testserver4",
- "user": {
- "name": "administrator1",
- "domain": "TestServer4",
- "id": "S-1-5-21-46152456-1367606905-4031241297-500"
- },
  "hostname": "testserver4"
  },
  "threat": {
@@ -262,7 +253,7 @@
  "start"
  ],
  "duration": 2442699369800,
- "ingested": "2021-06-09T11:57:25.803537900Z",
+ "ingested": "2022-01-02T01:28:49.601373311Z",
  "provider": "defender_endpoint",
  "action": "DefenseEvasion",
  "end": "2020-06-30T09:45:39.5484377Z",
@@ -321,7 +312,7 @@
  }
  },
  "ecs": {
- "version": "1.12.0"
+ "version": "8.0.0"
  },
  "related": {
  "hosts": [
@@ -353,7 +344,7 @@
  "end"
  ],
  "duration": 892514711800,
- "ingested": "2021-06-09T11:57:25.803543400Z",
+ "ingested": "2022-01-02T01:28:49.601374271Z",
  "provider": "defender_endpoint",
  "action": "Malware",
  "end": "2020-06-30T09:46:15.0876676Z",

packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
  value: '{{_ingest.timestamp}}'
  - set:
  field: ecs.version
- value: '1.12.0'
+ value: '8.0.0'
  - rename:
  field: message
  target_field: event.original
@@ -272,23 +272,6 @@ processors:
  target_field: user.id
  ignore_missing: true
 
-##############################
-## ECS host.user Mapping ##
-## Deprecated since ECS 1.8 ##
-##############################
- - set:
- field: host.user.name
- value: '{{user.name}}'
- ignore_empty_value: true
- - set:
- field: host.user.domain
- value: '{{user.domain}}'
- ignore_empty_value: true
- - set:
- field: host.user.id
- value: '{{user.id}}'
- ignore_empty_value: true
-
 #########################
 ## ECS Related Mapping ##
 #########################

packages/microsoft_defender_endpoint/data_stream/log/sample_event.json

@@ -1,73 +1,110 @@
 {
- "rule": {
- "description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection."
- },
- "message": "An active 'Exeselrun' malware was detected",
- "microsoft": {
- "defender_endpoint": {
- "investigationId": "9",
- "evidence": {
- "entityType": "File"
- },
- "resolvedTime": "2020-06-30T11:13:12.2680434Z",
- "investigationState": "Benign",
- "incidentId": "12",
- "assignedTo": "elastic@elasticuser.com",
- "lastUpdateTime": "2020-07-03T15:15:39.13Z",
- "status": "Resolved"
- }
+ "@timestamp": "2022-01-02T01:30:05.670Z",
+ "agent": {
+ "ephemeral_id": "9cc31363-7ffb-4763-9bec-cef372647d15",
+ "id": "b1d83907-ff3e-464a-b79a-cf843f6f0bba",
+ "name": "docker-fleet-agent",
+ "type": "filebeat",
+ "version": "8.0.0-beta1"
  },
  "cloud": {
- "provider": "azure",
  "account": {
- "id": "123543-d66c-4c7e-9e30-40034eb7c6f3"
+ "id": "a839b112-1253-6432-9bf6-94542403f21c"
  },
  "instance": {
- "id": "c5a964f417c11f6277d5bf9489f0d"
+ "id": "111e6dd8c833c8a052ea231ec1b19adaf497b625"
+ },
+ "provider": "azure"
+ },
+ "data_stream": {
+ "dataset": "microsoft_defender_endpoint.log",
+ "namespace": "ep",
+ "type": "logs"
+ },
+ "ecs": {
+ "version": "8.0.0"
+ },
+ "elastic_agent": {
+ "id": "b1d83907-ff3e-464a-b79a-cf843f6f0bba",
+ "snapshot": false,
+ "version": "8.0.0-beta1"
+ },
+ "event": {
+ "action": "Execution",
+ "agent_id_status": "verified",
+ "category": [
+ "host"
+ ],
+ "created": "2021-01-26T20:33:57.7220239Z",
+ "dataset": "microsoft_defender_endpoint.log",
+ "duration": 101466100,
+ "end": "2021-01-26T20:31:33.0577322Z",
+ "id": "da637472900382838869_1364969609",
+ "ingested": "2022-01-02T01:30:06Z",
+ "kind": "alert",
+ "provider": "defender_endpoint",
+ "severity": 2,
+ "start": "2021-01-26T20:31:32.9562661Z",
+ "timezone": "UTC",
+ "type": [
+ "user",
+ "creation",
+ "start"
+ ]
+ },
+ "host": {
+ "hostname": "temp123.middleeast.corp.microsoft.com",
+ "name": "temp123.middleeast.corp.microsoft.com"
+ },
+ "input": {
+ "type": "httpjson"
+ },
+ "message": "Low-reputation arbitrary code executed by signed executable",
+ "microsoft": {
+ "defender_endpoint": {
+ "evidence": {
+ "aadUserId": "11118379-2a59-1111-ac3c-a51eb4a3c627",
+ "accountName": "name",
+ "domainName": "DOMAIN",
+ "entityType": "User",
+ "userPrincipalName": "temp123@microsoft.com"
+ },
+ "incidentId": "1126093",
+ "investigationState": "Queued",
+ "lastUpdateTime": "2021-01-26T20:33:59.2Z",
+ "rbacGroupName": "A",
+ "status": "New"
  }
  },
  "observer": {
- "name": "WindowsDefenderAv",
- "product": "Defender ATP",
+ "name": "WindowsDefenderAtp",
+ "product": "Defender for Endpoint",
  "vendor": "Microsoft"
  },
- "file": {
- "name": "SB.xsl",
- "path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5"
- },
  "related": {
  "hosts": [
- "testserver4"
+ "temp123.middleeast.corp.microsoft.com"
+ ],
+ "user": [
+ "temp123"
  ]
  },
- "host": {
- "name": "testserver4",
- "hostname": "testserver4"
+ "rule": {
+ "description": "Binaries signed by Microsoft can be used to run low-reputation arbitrary code. This technique hides the execution of malicious code within a trusted process. As a result, the trusted process might exhibit suspicious behaviors, such as opening a listening port or connecting to a command-and-control (C\u0026C) server."
  },
+ "tags": [
+ "microsoft-defender-endpoint",
+ "forwarded"
+ ],
  "threat": {
+ "framework": "MITRE ATT\u0026CK",
  "technique": {
- "name": "Malware"
- },
- "framework": "MITRE ATT\u0026CK"
+ "name": "Execution"
+ }
  },
- "event": {
- "severity": 2,
- "kind": "alert",
- "timezone": "UTC",
- "created": "2020-06-30T10:09:01.1569718Z",
- "start": "2020-06-30T10:07:44.333733Z",
- "type": [
- "end"
- ],
- "duration": 0,
- "ingested": "2021-02-18T13:34:35.126958300Z",
- "provider": "defender_endpoint",
- "action": "Malware",
- "end": "2020-06-30T10:07:44.333733Z",
- "id": "da637291085411733957_-1043898914",
- "category": [
- "host",
- "malware"
- ]
+ "user": {
+ "domain": "DOMAIN",
+ "id": "S-1-5-21-11111607-1111760036-109187956-75141",
+ "name": "temp123"
  }
 }