[Fortinet FortiMail] Replace RSA2ELK with Syslog integration

packages/fortinet_fortimail/_dev/build/build.yml

@@ -1,3 +1,4 @@
 dependencies:
  ecs:
- reference: git@8.6
+ reference: git@v8.6.0
+ import_mappings: true

packages/fortinet_fortimail/_dev/build/docs/README.md

@@ -1,14 +1,54 @@
-# Fortinet FortiMail Integration
+# Fortinet FortiMail
 
-This integration is for Fortinet FortiMail logs sent in the syslog format.
+## Overview
 
-## Compatibility
+The [Fortinet FortiMail](https://www.fortinet.com/products/email-security) integration allows users to monitor History, System, Mail, Antispam, Antivirus, and Encryption events. FortiMail delivers advanced multi-layered protection against the full spectrum of email-borne threats. Powered by FortiGuard Labs threat intelligence and integrated into the Fortinet Security Fabric, FortiMail helps your organization prevent, detect, and respond to email-based threats including spam, phishing, malware, zero-day threats, impersonation, and Business Email Compromise (BEC) attacks.
 
-This integration has been tested against FortiOS version 6.0.x and 6.2.x. Versions above this are expected to work but have not been tested.
+Use the Fortinet FortiMail integration to collect and parse data from the Syslog. Then visualize that data in Kibana.
+
+## Data streams
+
+The Fortinet FortiMail integration collects one type of data stream: log.
+
+**Log** helps users to keep a record of email activity and traffic including system-related events, such as system restarts and HA activity, virus detections, spam filtering results, POP3, SMTP, IMAP, and webmail events. See more details [About FortiMail logging](https://docs.fortinet.com/document/fortimail/7.2.2/administration-guide/435158/about-fortimail-logging)
+
+This integration targets the six types of events as mentioned below:
+
+- **History** records all email traffic going through the FortiMail unit.
+
+- **System** records system management activities, including changes to the system configuration as well as administrator and user login and logouts.
+
+- **Mail** records mail activities.
+
+- **Antispam** records spam detection events.
+
+- **Antivirus** records virus intrusion events.
+
+- **Encryption** records detection of IBE-related events.
+
+## Requirements
+
+Elasticsearch is needed to store and search data, and Kibana is needed for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your hardware.
+
+This module has been tested against **Fortinet FortiMail version 7.2.2**.
+
+**Note:** The User must have to **Enable CSV format** option.
+
+## Setup
+
+### To collect data from Fortinet FortiMail Syslog server, follow the below steps:
+
+- [Configure Syslog server](https://docs.fortinet.com/document/fortimail/7.2.2/administration-guide/332364/configuring-logging#logging_2063907032_1949484)
+
+![Fortinet FortiMail Syslog Server](../img/fortinet-fortimail-configure-syslog-server.png)
+
+## Logs Reference
 
 ### Log
 
-The `log` dataset collects Fortinet FortiMail logs.
+This is the `Log` dataset.
+
+#### Example
 
 {{event "log"}}
 

packages/fortinet_fortimail/_dev/deploy/docker/docker-compose.yml

@@ -7,12 +7,17 @@ services:
  - ${SERVICE_LOGS_DIR}:/var/log
  command: /bin/sh -c "cp /sample_logs/* /var/log/"
  fortinet-fortimail-tcp:
- image: docker.elastic.co/observability/stream:v0.7.0
+ image: docker.elastic.co/observability/stream:v0.10.0
  volumes:
  - ./sample_logs:/sample_logs:ro
- command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=tcp /sample_logs/fortinet-fortimail.log
+ command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9024 -p=tcp /sample_logs/fortinet-fortimail.log
+ fortinet-fortimail-tls:
+ image: docker.elastic.co/observability/stream:v0.10.0
+ volumes:
+ - ./sample_logs:/sample_logs:ro
+ command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9024 -p=tls --insecure /sample_logs/fortinet-fortimail.log
  fortinet-fortimail-udp:
- image: docker.elastic.co/observability/stream:v0.7.0
+ image: docker.elastic.co/observability/stream:v0.10.0
  volumes:
  - ./sample_logs:/sample_logs:ro
- command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9515 -p=udp /sample_logs/fortinet-fortimail.log
+ command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9024 -p=udp /sample_logs/fortinet-fortimail.log

packages/fortinet_fortimail/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.0.0"
+ changes:
+ - description: Replace RSA2ELK with Syslog integration.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/5437
 - version: "1.3.1"
  changes:
  - description: Added categories and/or subcategories.

packages/fortinet_fortimail/data_stream/log/_dev/test/pipeline/test-antispam.log

@@ -0,0 +1 @@
+<190>date=2023-01-30,time=16:09:16.825,device_id=FEVM02TM23000064,log_id=0300003065,type=spam,subtype=default,pri=information,session_id="q6OL7fsQ018870-q6OL7fsR018870",client_name="",client_ip="192.168.100.1",dst_ip="10.50.2.225",from="user1@example.com",to="user2@example.com",subject="Test1516",msg="mailfilterd: Starting"

packages/fortinet_fortimail/data_stream/log/_dev/test/pipeline/test-antispam.log-expected.json

@@ -0,0 +1,88 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2023-01-30T16:09:16.825Z",
+ "destination": {
+ "ip": "10.50.2.225"
+ },
+ "ecs": {
+ "version": "8.6.0"
+ },
+ "email": {
+ "from": {
+ "address": [
+ "user1@example.com"
+ ]
+ },
+ "subject": "Test1516",
+ "to": {
+ "address": [
+ "user2@example.com"
+ ]
+ }
+ },
+ "event": {
+ "code": "0300003065",
+ "kind": "event",
+ "original": "\u003c190\u003edate=2023-01-30,time=16:09:16.825,device_id=FEVM02TM23000064,log_id=0300003065,type=spam,subtype=default,pri=information,session_id=\"q6OL7fsQ018870-q6OL7fsR018870\",client_name=\"\",client_ip=\"192.168.100.1\",dst_ip=\"10.50.2.225\",from=\"user1@example.com\",to=\"user2@example.com\",subject=\"Test1516\",msg=\"mailfilterd: Starting\""
+ },
+ "fortinet_fortimail": {
+ "log": {
+ "client": {
+ "ip": "192.168.100.1"
+ },
+ "date": "2023-01-30",
+ "destination_ip": "10.50.2.225",
+ "device_id": "FEVM02TM23000064",
+ "from": "user1@example.com",
+ "id": "0300003065",
+ "message": "mailfilterd: Starting",
+ "priority": "information",
+ "priority_number": 190,
+ "session_id": "q6OL7fsQ018870-q6OL7fsR018870",
+ "sub_type": "default",
+ "subject": "Test1516",
+ "time": "16:09:16.825",
+ "to": "user2@example.com",
+ "type": "spam"
+ }
+ },
+ "log": {
+ "level": "information",
+ "syslog": {
+ "facility": {
+ "code": 23
+ },
+ "priority": 190,
+ "severity": {
+ "code": 6
+ }
+ }
+ },
+ "message": "mailfilterd: Starting",
+ "observer": {
+ "product": "FortiMail",
+ "serial_number": "FEVM02TM23000064",
+ "type": "firewall",
+ "vendor": "Fortinet"
+ },
+ "related": {
+ "ip": [
+ "192.168.100.1",
+ "10.50.2.225"
+ ],
+ "user": [
+ "user1@example.com",
+ "user2@example.com"
+ ]
+ },
+ "source": {
+ "ip": "192.168.100.1"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ }
+ ]
+}

packages/fortinet_fortimail/data_stream/log/_dev/test/pipeline/test-antivirus.log

@@ -0,0 +1 @@
+<190>date=2023-01-30,time=16:09:15.246,device_id=FEVM02TM23000064,log_id=0400003064,type=virus,subtype=infected,pri=information,from="syntax@www.ca",to="user2@1.ca",src=67.43.156.28,session_id="q6OL7fsQ018870-q6OL7fsR018870",msg="The file inline16-69.dat is infected with EICAR_TEST_FILE."

packages/fortinet_fortimail/data_stream/log/_dev/test/pipeline/test-antivirus.log-expected.json

@@ -0,0 +1,90 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2023-01-30T16:09:15.246Z",
+ "ecs": {
+ "version": "8.6.0"
+ },
+ "email": {
+ "from": {
+ "address": [
+ "syntax@www.ca"
+ ]
+ },
+ "to": {
+ "address": [
+ "user2@1.ca"
+ ]
+ }
+ },
+ "event": {
+ "code": "0400003064",
+ "kind": "event",
+ "original": "\u003c190\u003edate=2023-01-30,time=16:09:15.246,device_id=FEVM02TM23000064,log_id=0400003064,type=virus,subtype=infected,pri=information,from=\"syntax@www.ca\",to=\"user2@1.ca\",src=67.43.156.28,session_id=\"q6OL7fsQ018870-q6OL7fsR018870\",msg=\"The file inline16-69.dat is infected with EICAR_TEST_FILE.\""
+ },
+ "fortinet_fortimail": {
+ "log": {
+ "date": "2023-01-30",
+ "device_id": "FEVM02TM23000064",
+ "from": "syntax@www.ca",
+ "id": "0400003064",
+ "message": "The file inline16-69.dat is infected with EICAR_TEST_FILE.",
+ "priority": "information",
+ "priority_number": 190,
+ "session_id": "q6OL7fsQ018870-q6OL7fsR018870",
+ "source": {
+ "ip": "67.43.156.28"
+ },
+ "sub_type": "infected",
+ "time": "16:09:15.246",
+ "to": "user2@1.ca",
+ "type": "virus"
+ }
+ },
+ "log": {
+ "level": "information",
+ "syslog": {
+ "facility": {
+ "code": 23
+ },
+ "priority": 190,
+ "severity": {
+ "code": 6
+ }
+ }
+ },
+ "message": "The file inline16-69.dat is infected with EICAR_TEST_FILE.",
+ "observer": {
+ "product": "FortiMail",
+ "serial_number": "FEVM02TM23000064",
+ "type": "firewall",
+ "vendor": "Fortinet"
+ },
+ "related": {
+ "ip": [
+ "67.43.156.28"
+ ],
+ "user": [
+ "syntax@www.ca",
+ "user2@1.ca"
+ ]
+ },
+ "source": {
+ "geo": {
+ "continent_name": "Asia",
+ "country_iso_code": "BT",
+ "country_name": "Bhutan",
+ "location": {
+ "lat": 27.5,
+ "lon": 90.5
+ }
+ },
+ "ip": "67.43.156.28"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ }
+ ]
+}

packages/fortinet_fortimail/data_stream/log/_dev/test/pipeline/test-common-config.yml

@@ -1,3 +1,4 @@
 fields:
  tags:
  - preserve_original_event
+ - preserve_duplicate_custom_fields

packages/fortinet_fortimail/data_stream/log/_dev/test/pipeline/test-encryption.log

@@ -0,0 +1,2 @@
+<190>date=2023-01-30,time=16:09:15.246,device_id=FEVM02TM23000064,log_id=0400003064,type=encrypt,pri=information,session_id="q6OL7fsQ018870-q6OL7fsR018870",msg="Starting cryptod"
+<142>date=2023-01-30,time=16:09:15.246,device_id=FEVM02TM23000064,log_id=0400003064,type=encrypt,pri=information,session_id="q6OL7fsQ018870-q6OL7fsR018870",msg="User user1@1.ca read secure message, id:'q79EiV8S007017-q79EiV8T0070170001474', sent from: 'user2@2.ca', subject: 'ppt file'"