[Ti_AbuseCH] - Added support for optional "Auth Key" request header across all data streams

packages/ti_abusech/_dev/build/docs/README.md

@@ -7,6 +7,12 @@ This integration is for [AbuseCH](https://urlhaus.abuse.ch/) logs. It includes t
 - `malwarebazaar` dataset: Supports indicators from the MalwareBazaar from AbuseCH.
 - `threatfox` dataset: Supports indicators from AbuseCH Threat Fox API.
 
+## Note:
+
+From February 2025, AbuseCH recommends using an optional `Auth Key` (API Key) in the requests to avoid rate limiting issues.
+More details on this topic can be found [here](https://abuse.ch/blog/community-first/). In version 2.8.0, we have added an optional `Auth Key`
+configuration option that can be used to avoid rate limiting.
+
 ## Agentless Enabled Integration
 
 Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).

packages/ti_abusech/_dev/deploy/docker/files/config-threatfox.yml

@@ -1,4 +1,23 @@
 rules:
+ - path: /api/v1/
+ methods: ["POST"]
+ request_headers:
+ Content-Type: "application/json"
+ Auth-Key: "test_auth_key"
+ body:
+ query: "get_iocs"
+ days: /[1-7]+/
+ responses:
+ - status_code: 200
+ body: |-
+ {
+ "query_status": "ok",
+ "data": [
+ {"id":"841537","ioc":"wizzy.hopto.org","threat_type":"botnet_cc","threat_type_desc":"Indicator that identifies a botnet command&control server (C&C)","ioc_type":"domain","ioc_type_desc":"Domain that is used for botnet Command&control (C&C)","malware":"win.asyncrat","malware_printable":"AsyncRAT","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.asyncrat","confidence_level":100,"first_seen":"2022-08-05 19:43:08 UTC","last_seen":null,"reference":"https://tria.ge/220805-w57pxsgae2","reporter":"AndreGironda","tags":["asyncrat"]},
+ {"id":"839586","ioc":"872ff530d50579ae6bdc7cb4d658324b1d0e7a3e","threat_type":"payload","threat_type_desc":"Indicator that identifies a malware sample (payload)","ioc_type":"sha1_hash","ioc_type_desc":"SHA1 hash of a malware sample (payload)","malware":"win.vidar","malware_printable":"Vidar","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.vidar","confidence_level":75,"first_seen":"2022-07-25 22:27:09 UTC","last_seen":null,"reference":"","reporter":"crep1x","tags":["Vidar"]},
+ {"id":"839587","ioc":"a3b5c6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1s2t3","threat_type":"payload","threat_type_desc":"Indicator that identifies a malware sample (payload)","ioc_type":"sha1_hash","ioc_type_desc":"SHA1 hash of a malware sample (payload)","malware":"win.redline","malware_printable":"RedLine","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.redline","confidence_level":80,"first_seen":"2025-03-15 10:15:00 UTC","last_seen":null,"reference":"","reporter":"cyberhunter","tags":["RedLine"]}
+ ]
+ }
  - path: /api/v1/
  methods: ["POST"]
  request_headers:

packages/ti_abusech/_dev/deploy/docker/files/config.yml

@@ -1,4 +1,55 @@
 rules:
+ - path: /v1/payloads/recent/
+ methods: ["GET"]
+ request_headers:
+ Content-Type: "application/json"
+ Auth-Key: "test_auth_key"
+ responses:
+ - status_code: 200
+ body: |-
+ {
+ "query_status": "ok",
+ "payloads": [{
+ "md5_hash": "9cd5a4f0231a47823c4adba7c8ef370f",
+ "sha256_hash": "7c0852d514df7faf8fdbfa4f358cc235dd1b1a2d843cc65495d03b502e4099f2",
+ "file_type": "unknown",
+ "file_size": "1563",
+ "signature": null,
+ "firstseen": "2021-10-05 04:17:02",
+ "urlhaus_download": "https://urlhaus-api.abuse.ch/v1/download/7c0852d514df7faf8fdbfa4f358cc235dd1b1a2d843cc65495d03b502e4099f2/",
+ "virustotal": null,
+ "imphash": null,
+ "ssdeep": "48:yazkS7neW+mfe4CJjNXcq5Co4Fr1PpsHn:yrmGNt5mbP2n",
+ "tlsh": "T109314C5E7822CA70B91AD69300C22D8C2F53EAF229E6686C3BDD4C86FA1344208CF1"
+ },
+ {
+ "md5_hash": "31da38bee299f10d40b307828239af98",
+ "sha256_hash": "e1a5ef457aca7c27768917c8dfd9a0647799be2bd5185f61d889830e2ac4125d",
+ "file_type": "unknown",
+ "file_size": "1565",
+ "signature": null,
+ "firstseen": "2021-10-05 04:16:59",
+ "urlhaus_download": "https://urlhaus-api.abuse.ch/v1/download/e1a5ef457aca7c27768917c8dfd9a0647799be2bd5185f61d889830e2ac4125d/",
+ "virustotal": null,
+ "imphash": null,
+ "ssdeep": "24:SpbS988bp6qdsehrRASqw/VVyz9YhmqgcqiWHPsul0eGqa3gm4pwas6rx30ixN05:Spu16qS8v9hZqixul1GtvoRlkKy9O5k",
+ "tlsh": "T165310A8254FCC87563EBCA5B7793F31B6219740961E3227A63B790CAA3458985C4F8"
+ },
+ {
+ "md5_hash": "31da38bee299f10d40b307828239af98",
+ "sha256_hash": "e1a5ef457aca7c27768917c8dfd9a0647799be2bd5185f61d889830e2ac4125d",
+ "file_type": "unknown",
+ "file_size": "1565",
+ "signature": null,
+ "firstseen": "2021-10-05 04:16:59",
+ "urlhaus_download": "https://urlhaus-api.abuse.ch/v1/download/e1a5ef457aca7c27768917c8dfd9a0647799be2bd5185f61d889830e2ac4125d/",
+ "virustotal": null,
+ "imphash": null,
+ "ssdeep": "24:SpbS988bp6qdsehrRASqw/VVyz9YhmqgcqiWHPsul0eGqa3gm4pwas6rx30ixN05:Spu16qS8v9hZqixul1GtvoRlkKy9O5k",
+ "tlsh": "T165310A8254FCC87563EBCA5B7793F31B6219740961E3227A63B790CAA3458985C4F8"
+ }
+ ]
+ }
  - path: /v1/payloads/recent/
  methods: ["GET"]
  request_headers:
@@ -84,6 +135,15 @@ rules:
  }
  ]
  }
+ - path: /downloads/json
+ methods: ["GET"]
+ request_headers:
+ Content-Type: ["application/zip"]
+ Auth-Key: "test_auth_key"
+ responses:
+ - status_code: 200
+ body: |
+ {{file "/files/urlhaus_full.json.zip"}}
  - path: /downloads/json
  methods: ["GET"]
  request_headers:
@@ -92,6 +152,121 @@ rules:
  - status_code: 200
  body: |
  {{file "/files/urlhaus_full.json.zip"}}
+ - path: /api/v1/
+ methods: ["POST"]
+ request_headers:
+ Content-Type: "application/x-www-form-urlencoded"
+ Auth-Key: "test_auth_key"
+ query_params:
+ query: "get_recent"
+ selector: "time"
+ responses:
+ - status_code: 200
+ body: |-
+ {
+ "query_status": "ok",
+ "data": [{
+ "sha256_hash": "7a6c03013a2f2ab8b9e8e7e5d226ea89e75da72c1519e78fd28b2253ea755c28",
+ "sha3_384_hash": "d63e73b68973bc73ab559549aeee2141a48b8a3724aabc0d81fb14603c163a098a5a10be9f6d33b888602906c0d89955",
+ "sha1_hash": "42c7153680d7402e56fe022d1024aab49a9901a0",
+ "md5_hash": "1fc1c2997c8f55ac10496b88e23f5320",
+ "first_seen": "2021-10-05 14:02:45",
+ "last_seen": null,
+ "file_name": "7a6c03013a2f2ab8b9e8e7e5d226ea89e75da72c1519e.exe",
+ "file_size": 432640,
+ "file_type_mime": "application/x-dosexec",
+ "file_type": "exe",
+ "reporter": "abuse_ch",
+ "origin_country": "FR",
+ "anonymous": 0,
+ "signature": "RedLineStealer",
+ "imphash": "f34d5f2d4577ed6d9ceec516c1f5a744",
+ "tlsh": "T13794242864BFC05994E3EEA12DDCA8FBD99A55E3640C743301B4633B8B52B84DE4F479",
+ "telfhash": null,
+ "ssdeep": "12288:jhhl1Eo+iEXvpb1C7drqAd1uUaJvzXGyO2F5V3bS1jsTacr:7lL",
+ "dhash_icon": null,
+ "tags": [
+ "exe",
+ "RedLineStealer"
+ ],
+ "code_sign": [],
+ "intelligence": {
+ "clamav": null,
+ "downloads": "11",
+ "uploads": "1",
+ "mail": null
+ }
+ },
+ {
+ "sha256_hash": "670e59f677706e51b84984f42f8f89229e294e4c482dfa4fc72964def42a3626",
+ "sha3_384_hash": "60b388f9e2d0e24328af41421cc71f6cf097a432b7393e122c1f2103a4be0c1f1ab8b3e0a61a4393ac4eb27716e4bab2",
+ "sha1_hash": "47da737c89777cd1d08c5427d08ec3e77ef0da2c",
+ "md5_hash": "657edf01cd9100a52abd7a9d8e585a28",
+ "first_seen": "2021-10-05 14:00:50",
+ "last_seen": null,
+ "file_name": "657edf01cd9100a52abd7a9d8e585a28",
+ "file_size": 414720,
+ "file_type_mime": "application/x-dosexec",
+ "file_type": "exe",
+ "reporter": "zbetcheckin",
+ "origin_country": "FR",
+ "anonymous": 0,
+ "signature": "Loki",
+ "imphash": "f34d5f2d4577ed6d9ceec516c1f5a744",
+ "tlsh": "T1C7949D69322F5917CE288EF5185EA1C183F90C3B155AE7E85DCAF1FA65C3FB01A81493",
+ "telfhash": null,
+ "ssdeep": "6144:FQq8qc0WpMge4UPvaMmdtq1a/tZFJYhVFyWpEGDNi0x7ynZPppgjQ+3HwO/A:FQq8qc0WfqaFq8tfJWVFL1Ri0x453o4",
+ "dhash_icon": null,
+ "tags": [
+ "32",
+ "exe",
+ "Loki",
+ "trojan"
+ ],
+ "code_sign": [],
+ "intelligence": {
+ "clamav": null,
+ "downloads": "11",
+ "uploads": "1",
+ "mail": null
+ }
+ },
+ {
+ "sha256_hash": "a1b2c3d4e5f67890123456789abcdef0123456789abcdef0123456789abcdef",
+ "sha3_384_hash": "b1c2d3e4f5a67890123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+ "sha1_hash": "c1d2e3f4a567890123456789abcdef012345678",
+ "md5_hash": "d1e2f3a4567890123456789abcdef01",
+ "first_seen": "2023-06-15 10:30:45",
+ "last_seen": null,
+ "file_name": "dummy_malware.exe",
+ "file_size": 512345,
+ "file_type_mime": "application/x-dosexec",
+ "file_type": "exe",
+ "reporter": "dummy_researcher",
+ "origin_country": "US",
+ "anonymous": 0,
+ "signature": "FakeSignature",
+ "imphash": "e1f2a3b4567890abcdef1234567890ab",
+ "tlsh": "F1E2D3C4B5A67890ABCDEF1234567890ABCDEF1234567890ABCDEF1234567890ABCDEF12",
+ "telfhash": null,
+ "ssdeep": "6144:ABCDEF0123456789abcdefABCDEF0123456789abcdefABCDEF0123456789abcdef:A1B2C3D4E5F6G7H8",
+ "dhash_icon": null,
+ "tags": [
+ "64",
+ "exe",
+ "FakeSignature",
+ "ransomware"
+ ],
+ "code_sign": [],
+ "intelligence": {
+ "clamav": null,
+ "downloads": "25",
+ "uploads": "3",
+ "mail": null
+ }
+ }
+ ]
+ }
  - path: /api/v1/
  methods: ["POST"]
  request_headers:

packages/ti_abusech/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.8.0"
+ changes:
+ - description: Add support for 'Auth-Key' in request header for all data streams.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/13261
 - version: "2.7.0"
  changes:
  - description: Enable Agentless deployment.

packages/ti_abusech/data_stream/malware/_dev/test/system/test-auth-config.yml

@@ -0,0 +1,14 @@
+input: cel
+service: abusech
+vars:
+ logging:
+ level: debug
+ enable_request_tracer: true
+ auth_key: test_auth_key
+data_stream:
+ vars:
+ url: http://{{Hostname}}:{{Port}}/v1/payloads/recent/
+ preserve_original_event: true
+ ioc_expiration_duration: 5d
+assert:
+ hit_count: 3

packages/ti_abusech/data_stream/malware/agent/stream/cel.yml.hbs

@@ -14,12 +14,22 @@ resource.ssl: {{ssl}}
 resource.timeout: {{http_client_timeout}}
 {{/if}}
 resource.url: {{url}}
+{{#if auth_key}}
+state:
+ auth_key: {{auth_key}}
+{{/if}}
 redact:
- fields: ~
+ fields: 
+ - auth_key
 program: |
+ state.with(
  request("GET", state.url).with({
  "Header":{
  "Content-Type": ["application/json"],
+ ?"Auth-Key": has(state.auth_key) ? 
+ optional.of([state.auth_key]) 
+ :
+ optional.none(),
  }
  }).as(req, req.do_request().as(resp, 
  bytes(resp.Body).decode_json().as(body, {
@@ -29,6 +39,7 @@ program: |
  "url": state.url
  })
  ))
+ )
 
 {{#if ioc_expiration_duration}}
 fields_under_root: true

packages/ti_abusech/data_stream/malwarebazaar/_dev/test/system/test-auth-config.yml

@@ -0,0 +1,12 @@
+input: cel
+service: abusech
+data_stream:
+ vars:
+ url: http://{{Hostname}}:{{Port}}/api/v1/
+ preserve_original_event: true
+ ioc_expiration_duration: 5d
+vars:
+ auth_key: test_auth_key
+ enable_request_tracer: true
+assert:
+ hit_count: 3