[Abuse CH] Add Threat Fox Datastream (#3962)

Add Threat Fox Datastream to Abuse CH Integration.

Author: legoguy1000

packages/ti_abusech/_dev/build/docs/README.md

@@ -20,4 +20,8 @@ The AbuseCH malware data_stream retrieves threat intelligence indicators from th
 
 The AbuseCH malwarebazaar data_stream retrieves threat intelligence indicators from the MalwareBazaar API endpoint `https://mb-api.abuse.ch/api/v1/`.
 
-{{fields "malwarebazaar"}}
+{{fields "malwarebazaar"}}
+
+The AbuseCH threatfox data_stream retrieves threat intelligence indicators from the Threat Fox API endpoint `https://threatfox-api.abuse.ch/api/v1/`.
+
+{{fields "threatfox"}}

packages/ti_abusech/_dev/deploy/docker/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "2.3"
 services:
  abusech:
- image: docker.elastic.co/observability/stream:v0.6.1
+ image: docker.elastic.co/observability/stream:v0.7.0
  ports:
  - 8080
  volumes:
@@ -12,3 +12,15 @@ services:
  - http-server
  - --addr=:8080
  - --config=/files/config.yml
+ abusech-threatfox:
+ image: docker.elastic.co/observability/stream:v0.7.0
+ ports:
+ - 8081
+ volumes:
+ - ./files:/files:ro
+ environment:
+ PORT: 8081
+ command:
+ - http-server
+ - --addr=:8081
+ - --config=/files/config-threatfox.yml

packages/ti_abusech/_dev/deploy/docker/files/config-threatfox.yml

@@ -0,0 +1,17 @@
+rules:
+ - path: /api/v1/
+ methods: ["POST"]
+ request_headers:
+ Content-Type: "application/json"
+ body:
+ query: "get_iocs"
+ responses:
+ - status_code: 200
+ body: |-
+ {
+ "query_status": "ok",
+ "data": [
+ {"id":"841537","ioc":"wizzy.hopto.org","threat_type":"botnet_cc","threat_type_desc":"Indicator that identifies a botnet command&control server (C&C)","ioc_type":"domain","ioc_type_desc":"Domain that is used for botnet Command&control (C&C)","malware":"win.asyncrat","malware_printable":"AsyncRAT","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.asyncrat","confidence_level":100,"first_seen":"2022-08-05 19:43:08 UTC","last_seen":null,"reference":"https://tria.ge/220805-w57pxsgae2","reporter":"AndreGironda","tags":["asyncrat"]},
+ {"id":"839586","ioc":"872ff530d50579ae6bdc7cb4d658324b1d0e7a3e","threat_type":"payload","threat_type_desc":"Indicator that identifies a malware sample (payload)","ioc_type":"sha1_hash","ioc_type_desc":"SHA1 hash of a malware sample (payload)","malware":"win.vidar","malware_printable":"Vidar","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.vidar","confidence_level":75,"first_seen":"2022-07-25 22:27:09 UTC","last_seen":null,"reference":"","reporter":"crep1x","tags":["Vidar"]}
+ ]
+ }

packages/ti_abusech/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.0"
+ changes:
+ - description: Add Threat Fox datastream
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/3962
 - version: "1.6.0"
  changes:
  - description: Update package to ECS 8.4.0

packages/ti_abusech/data_stream/malware/sample_event.json

@@ -1,34 +1,34 @@
 {
- "@timestamp": "2022-04-11T08:43:51.252Z",
+ "@timestamp": "2022-08-06T00:06:27.079Z",
  "abusech": {
  "malware": {}
  },
  "agent": {
- "ephemeral_id": "3c096aaa-3fd9-4560-87fe-375b99890402",
- "id": "0cd371ed-8f03-437b-909d-8daccf9843fc",
+ "ephemeral_id": "1760e1ca-6974-4a32-80c6-0e7e58a6d573",
+ "id": "87d4d8f8-b034-42ba-a5bb-33ff670e619e",
  "name": "docker-fleet-agent",
  "type": "filebeat",
- "version": "8.0.0"
+ "version": "8.3.0"
  },
  "data_stream": {
  "dataset": "ti_abusech.malware",
  "namespace": "ep",
  "type": "logs"
  },
  "ecs": {
- "version": "8.3.0"
+ "version": "8.4.0"
  },
  "elastic_agent": {
- "id": "0cd371ed-8f03-437b-909d-8daccf9843fc",
+ "id": "87d4d8f8-b034-42ba-a5bb-33ff670e619e",
  "snapshot": false,
- "version": "8.0.0"
+ "version": "8.3.0"
  },
  "event": {
  "agent_id_status": "verified",
  "category": "threat",
- "created": "2022-04-11T08:43:51.252Z",
+ "created": "2022-08-06T00:06:27.079Z",
  "dataset": "ti_abusech.malware",
- "ingested": "2022-04-11T08:43:52Z",
+ "ingested": "2022-08-06T00:06:30Z",
  "kind": "enrichment",
  "original": "{\"file_size\":\"1563\",\"file_type\":\"unknown\",\"firstseen\":\"2021-10-05 04:17:02\",\"imphash\":null,\"md5_hash\":\"9cd5a4f0231a47823c4adba7c8ef370f\",\"sha256_hash\":\"7c0852d514df7faf8fdbfa4f358cc235dd1b1a2d843cc65495d03b502e4099f2\",\"signature\":null,\"ssdeep\":\"48:yazkS7neW+mfe4CJjNXcq5Co4Fr1PpsHn:yrmGNt5mbP2n\",\"tlsh\":\"T109314C5E7822CA70B91AD69300C22D8C2F53EAF229E6686C3BDD4C86FA1344208CF1\",\"urlhaus_download\":\"https://urlhaus-api.abuse.ch/v1/download/7c0852d514df7faf8fdbfa4f358cc235dd1b1a2d843cc65495d03b502e4099f2/\",\"virustotal\":null}",
  "type": "indicator"

packages/ti_abusech/data_stream/malwarebazaar/sample_event.json

@@ -1,5 +1,5 @@
 {
- "@timestamp": "2022-04-11T08:44:21.828Z",
+ "@timestamp": "2022-08-06T00:08:33.562Z",
  "abusech": {
  "malwarebazaar": {
  "anonymous": 0,
@@ -15,31 +15,31 @@
  }
  },
  "agent": {
- "ephemeral_id": "15657330-8e8b-49be-b82d-529320d9c53c",
- "id": "0cd371ed-8f03-437b-909d-8daccf9843fc",
+ "ephemeral_id": "7d65c47e-ccda-4f97-9896-6118ffb92a61",
+ "id": "87d4d8f8-b034-42ba-a5bb-33ff670e619e",
  "name": "docker-fleet-agent",
  "type": "filebeat",
- "version": "8.0.0"
+ "version": "8.3.0"
  },
  "data_stream": {
  "dataset": "ti_abusech.malwarebazaar",
  "namespace": "ep",
  "type": "logs"
  },
  "ecs": {
- "version": "8.3.0"
+ "version": "8.4.0"
  },
  "elastic_agent": {
- "id": "0cd371ed-8f03-437b-909d-8daccf9843fc",
+ "id": "87d4d8f8-b034-42ba-a5bb-33ff670e619e",
  "snapshot": false,
- "version": "8.0.0"
+ "version": "8.3.0"
  },
  "event": {
  "agent_id_status": "verified",
  "category": "threat",
- "created": "2022-04-11T08:44:21.828Z",
+ "created": "2022-08-06T00:08:33.562Z",
  "dataset": "ti_abusech.malwarebazaar",
- "ingested": "2022-04-11T08:44:22Z",
+ "ingested": "2022-08-06T00:08:36Z",
  "kind": "enrichment",
  "original": "{\"anonymous\":0,\"code_sign\":[],\"dhash_icon\":null,\"file_name\":\"7a6c03013a2f2ab8b9e8e7e5d226ea89e75da72c1519e.exe\",\"file_size\":432640,\"file_type\":\"exe\",\"file_type_mime\":\"application/x-dosexec\",\"first_seen\":\"2021-10-05 14:02:45\",\"imphash\":\"f34d5f2d4577ed6d9ceec516c1f5a744\",\"intelligence\":{\"clamav\":null,\"downloads\":\"11\",\"mail\":null,\"uploads\":\"1\"},\"last_seen\":null,\"md5_hash\":\"1fc1c2997c8f55ac10496b88e23f5320\",\"origin_country\":\"FR\",\"reporter\":\"abuse_ch\",\"sha1_hash\":\"42c7153680d7402e56fe022d1024aab49a9901a0\",\"sha256_hash\":\"7a6c03013a2f2ab8b9e8e7e5d226ea89e75da72c1519e78fd28b2253ea755c28\",\"sha3_384_hash\":\"d63e73b68973bc73ab559549aeee2141a48b8a3724aabc0d81fb14603c163a098a5a10be9f6d33b888602906c0d89955\",\"signature\":\"RedLineStealer\",\"ssdeep\":\"12288:jhhl1Eo+iEXvpb1C7drqAd1uUaJvzXGyO2F5V3bS1jsTacr:7lL\",\"tags\":[\"exe\",\"RedLineStealer\"],\"telfhash\":null,\"tlsh\":\"T13794242864BFC05994E3EEA12DDCA8FBD99A55E3640C743301B4633B8B52B84DE4F479\"}",
  "type": "indicator"

packages/ti_abusech/data_stream/threatfox/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,3 @@
+fields:
+ tags:
+ - preserve_original_event