[Sophos] update sophos ECS version and adding event.original options

packages/sophos/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.3.0"
+ changes:
+ - description: update to ECS 1.10.0 and adding event.original options
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/1102
 - version: "0.2.1"
  changes:
  - description: update to ECS 1.9.0

packages/sophos/data_stream/utm/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,5 @@
+dynamic_fields:
+ event.ingested: ".*"
+fields:
+ tags:
+ - preserve_original_event

packages/sophos/data_stream/utm/agent/stream/stream.yml.hbs

@@ -4,8 +4,11 @@ paths:
 {{/each}}
 exclude_files: [".gz$"]
 tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -18,6 +21,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
  lang: javascript
  params:
@@ -5065,7 +5071,3 @@ processors:
  target_subdomain_field: url.subdomain
  target_etld_field: url.top_level_domain
 - add_locale: ~
-- add_fields:
- target: ''
- fields:
- ecs.version: 1.9.0

packages/sophos/data_stream/utm/agent/stream/tcp.yml.hbs

@@ -1,8 +1,11 @@
 tcp:
 host: "{{tcp_host}}:{{tcp_port}}"
 tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -15,6 +18,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
  lang: javascript
  params:
@@ -5062,7 +5068,3 @@ processors:
  target_subdomain_field: url.subdomain
  target_etld_field: url.top_level_domain
 - add_locale: ~
-- add_fields:
- target: ''
- fields:
- ecs.version: 1.9.0

packages/sophos/data_stream/utm/agent/stream/udp.yml.hbs

@@ -1,8 +1,11 @@
 udp:
 host: "{{udp_host}}:{{udp_port}}"
 tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -15,6 +18,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
  lang: javascript
  params:
@@ -5061,8 +5067,4 @@ processors:
  target_field: url.registered_domain
  target_subdomain_field: url.subdomain
  target_etld_field: url.top_level_domain
-- add_locale: ~
-- add_fields:
- target: ''
- fields:
- ecs.version: 1.9.0
+- add_locale: ~

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml

@@ -4,61 +4,69 @@ description: Pipeline for Astaro Security Gateway
 processors:
  # ECS event.ingested
  - set:
- field: event.ingested
- value: '{{_ingest.timestamp}}'
+ field: event.ingested
+ value: '{{_ingest.timestamp}}'
+ - set:
+ field: ecs.version
+ value: "1.10.0"
  # User agent
  - user_agent:
-  field: user_agent.original
-  ignore_missing: true
+ field: user_agent.original
+ ignore_missing: true
  # IP Geolocation Lookup
  - geoip:
-  field: source.ip
-  target_field: source.geo
-  ignore_missing: true
+ field: source.ip
+ target_field: source.geo
+ ignore_missing: true
  - geoip:
-  field: destination.ip
-  target_field: destination.geo
-  ignore_missing: true
+ field: destination.ip
+ target_field: destination.geo
+ ignore_missing: true
 
  # IP Autonomous System (AS) Lookup
  - geoip:
-  database_file: GeoLite2-ASN.mmdb
-  field: source.ip
-  target_field: source.as
-  properties:
-  - asn
-  - organization_name
-  ignore_missing: true
+ database_file: GeoLite2-ASN.mmdb
+ field: source.ip
+ target_field: source.as
+ properties:
+ - asn
+ - organization_name
+ ignore_missing: true
  - geoip:
-  database_file: GeoLite2-ASN.mmdb
-  field: destination.ip
-  target_field: destination.as
-  properties:
-  - asn
-  - organization_name
-  ignore_missing: true
+ database_file: GeoLite2-ASN.mmdb
+ field: destination.ip
+ target_field: destination.as
+ properties:
+ - asn
+ - organization_name
+ ignore_missing: true
  - rename:
-  field: source.as.asn
-  target_field: source.as.number
-  ignore_missing: true
+ field: source.as.asn
+ target_field: source.as.number
+ ignore_missing: true
  - rename:
-  field: source.as.organization_name
-  target_field: source.as.organization.name
-  ignore_missing: true
+ field: source.as.organization_name
+ target_field: source.as.organization.name
+ ignore_missing: true
  - rename:
-  field: destination.as.asn
-  target_field: destination.as.number
-  ignore_missing: true
+ field: destination.as.asn
+ target_field: destination.as.number
+ ignore_missing: true
  - rename:
-  field: destination.as.organization_name
-  target_field: destination.as.organization.name
-  ignore_missing: true
+ field: destination.as.organization_name
+ target_field: destination.as.organization.name
+ ignore_missing: true
  - append:
- field: related.hosts
- value: '{{host.name}}'
- allow_duplicates: false
- if: ctx.host?.name != null && ctx.host?.name != ''
+ field: related.hosts
+ value: '{{host.name}}'
+ allow_duplicates: false
+ if: ctx.host?.name != null && ctx.host?.name != ''
+ - remove:
+ field: event.original
+ if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+ ignore_failure: true
+ ignore_missing: true
 on_failure:
  - append:
-  field: error.message
-  value: "{{ _ingest.on_failure_message }}"
+ field: error.message
+ value: "{{ _ingest.on_failure_message }}"

packages/sophos/data_stream/utm/manifest.yml

@@ -54,6 +54,23 @@ streams:
  required: false
  show_user: false
  default: false
+ - name: preserve_original_event
+ required: true
+ show_user: true
+ title: Preserve original event
+ description: Preserves a raw copy of the original event, added to the field `event.original`
+ type: bool
+ multi: false
+ default: false
+ - name: processors
+ type: yaml
+ title: Processors
+ multi: false
+ required: false
+ show_user: false
+ description: >
+ Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
  - input: tcp
  title: Sophos UTM logs
  description: Collect Sophos UTM logs
@@ -106,6 +123,23 @@ streams:
  required: false
  show_user: false
  default: false
+ - name: preserve_original_event
+ required: true
+ show_user: true
+ title: Preserve original event
+ description: Preserves a raw copy of the original event, added to the field `event.original`
+ type: bool
+ multi: false
+ default: false
+ - name: processors
+ type: yaml
+ title: Processors
+ multi: false
+ required: false
+ show_user: false
+ description: >
+ Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
  - input: logfile
  enabled: false
  title: Sophos UTM logs
@@ -152,3 +186,19 @@ streams:
  required: false
  show_user: false
  default: false
+ - name: preserve_original_event
+ required: true
+ show_user: true
+ title: Preserve original event
+ description: Preserves a raw copy of the original event, added to the field `event.original`
+ type: bool
+ multi: false
+ default: false
+ - name: processors
+ type: yaml
+ title: Processors
+ multi: false
+ required: false
+ show_user: false
+ description: >-
+ Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.

packages/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-xg.log

@@ -48,7 +48,7 @@
 <30>device="SFW" date=2017-03-16 time=12:56:01 timezone="IST" device_name="XG125w" device_id=S1601E1F9FCB7EE log_id=066811618014 log_type="Event" log_component="RED" log_subtype="System" priority=Information red_id=A350196C47072B0 status="Connected" eventtime="2017-03-16 12:56:01 IST" duration=164000 branch_name=Gaurav Patel recv_bytes=0 sent_bytes=0 message="A350196C47072B0/Gaurav Patel is now re-connected after 164000 ms"
 <30>device="SFW" date=2017-03-16 time=12:53:27 timezone="IST" device_name="XG125w" device_id=S1601E1F9FCB7EE log_id=066811618015 log_type="Event" log_component="RED" log_subtype="System" priority=Information red_id=A350196C47072B0 status="Disconnected" eventtime="2017-03-16 12:53:27 IST" duration=0 branch_name=Gaurav Patel recv_bytes=31488 sent_bytes=22368 message="A350196C47072B0/Gaurav Patel is now disconnected"
 <30>device="SFW" date=2017-03-16 time=12:46:26 timezone="IST" device_name="XG125w" device_id=S1601E1F9FCB7EE log_id=066811618016 log_type="Event" log_component="RED" log_subtype="System" priority=Information red_id=A350196C47072B0 status="Interim" eventtime="2017-03-16 12:46:26 IST" duration=0 branch_name=NY recv_bytes=0 sent_bytes=0 message="A350196C47072B0/NY transfered bytes TX: 0 RX: 0"
-<30>device="SFW" date=2018-06-06 time=11:12:10 timezone="IST" device_name="SG430" device_id=S4000806149EE49 log_id=063711517815 log_type="Event" log_component="DDNS" log_subtype="System" status="Success" priority=Notice host=test1. customtest.dyndns.org updatedip=10.198.232.86 reason="" message="DDNS update for host test1.customtest.dyndns.org was Successful. Updated with IP 10.198.232.86."
+<30>device="SFW" date=2018-06-06 time=11:12:10 timezone="IST" device_name="SG430" device_id=S4000806149EE49 log_id=063711517815 log_type="Event" log_component="DDNS" log_subtype="System" status="Success" priority=Notice host=test1.customtest.dyndns.org updatedip=10.198.232.86 reason="" message="DDNS update for host test1.customtest.dyndns.org was Successful. Updated with IP 10.198.232.86."
 <30>device="SFW" date=2020-05-18 time=14:38:37 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=010101600001 log_type="Firewall" log_component="Firewall Rule" log_subtype="Allowed" status="Allow" priority=Information duration=11 fw_rule_id=21 policy_type=1 user_name="" user_gp="" iap=0 ips_policy_id=0 appfilter_policy_id=0 application="HTTP" application_risk=1 application_technology="Browser Based" application_category="General Internet" in_interface="Port1" out_interface="Port2" src_mac=00:00:00:00:00:00 src_ip=172.17.34.15 src_country_code=R1 dst_ip=91.228.167.86 dst_country_code=SVK protocol="TCP" src_port=62841 dst_port=80 sent_pkts=6 recv_pkts=5 sent_bytes=459 recv_bytes=606 tran_src_ip=213.167.51.66 tran_src_port=0 tran_dst_ip="" tran_dst_port=0 srczonetype="LAN" srczone="LAN" dstzonetype="WAN" dstzone="WAN" dir_disp="" connevent="Stop" connid="1617925280" vconnid="" hb_health="No Heartbeat" message="" appresolvedby="Signature" app_is_cloud=0
 <30>device="SFW" date=2020-05-18 time=14:38:38 timezone="CEST" device_name="XG230" device_id=1234567890123457 log_id=010101600001 log_type="Firewall" log_component="Firewall Rule" log_subtype="Allowed" status="Allow" priority=Information duration=0 fw_rule_id=67 policy_type=1 user_name="" user_gp="" iap=0 ips_policy_id=15 appfilter_policy_id=0 application="DNS" application_risk=1 application_technology="Network Protocol" application_category="Infrastructure" in_interface="Port3.400" out_interface="Port2" src_mac=00:00:00:00:00:00 src_ip=172.16.66.155 src_country_code=R1 dst_ip=91.228.165.117 dst_country_code=SVK protocol="UDP" src_port=49144 dst_port=53 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=185.8.209.194 tran_src_port=0 tran_dst_ip="" tran_dst_port=0 srczonetype="DMZ" srczone="DMZ" dstzonetype="WAN" dstzone="WAN" dir_disp="" connevent="Start" connid="3360392048" vconnid="" hb_health="No Heartbeat" message="" appresolvedby="Signature" app_is_cloud=0
 <30>device="SFW" date=2020-05-18 time=14:38:39 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=010102600002 log_type="Firewall" log_component="Firewall Rule" log_subtype="Denied" status="Deny" priority=Information duration=0 fw_rule_id=29 policy_type=1 user_name="" user_gp="" iap=2 ips_policy_id=0 appfilter_policy_id=0 application="" application_risk=0 application_technology="" application_category="" in_interface="Port1" out_interface="Port2" src_mac=24:01:c7:07:2b:a2 src_ip=172.17.35.113 src_country_code="" dst_ip=172.20.4.52 dst_country_code="" protocol="TCP" src_port=53287 dst_port=4980 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip="" tran_src_port=0 tran_dst_ip="" tran_dst_port=0 srczonetype="" srczone="" dstzonetype="" dstzone="" dir_disp="" connid="" vconnid="" hb_health="No Heartbeat" message="" appresolvedby="Signature" app_is_cloud=0

packages/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-xg.log-config.yml

@@ -1,2 +1,10 @@
 dynamic_fields:
  event.ingested: ".*"
+fields:
+ tags:
+ - preserve_original_event
+ _conf:
+ default: defaulttest.local
+ mappings:
+ - serial_number: "1234567890123456"
+ hostname: "testhost.local"