[PANW] add audit log parsing

packages/panw/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.25.0"
+ changes:
+ - description: Add audit log parsing
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/9663
 - version: "3.24.4"
  changes:
  - description: Make / in url optional

packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-audit-sample.log

@@ -0,0 +1,2 @@
+Apr 11 20:06:15 192.168.0.1 01111111111,2024/04/11 20:06:15,audit,2561,gui-op,suser,"<debug><dataplane><packet-diag><show><setting/></show></packet-diag></dataplane></debug>",success
+Apr 18 18:35:20 10.1.1.1 003001000000,2024/04/18 18:35:20,audit,2561,gui-op,Mustang,"<show><config-locks><vsys>all</vsys></config-locks></show>",success

packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-audit-sample.log-config.yml

@@ -0,0 +1,5 @@
+fields:
+ tags:
+ - preserve_original_event
+ _conf:
+ tz_offset: "-04:00"

packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-audit-sample.log-expected.json

@@ -0,0 +1,100 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2024-04-11T20:06:15.000-04:00",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "category": [
+ "configuration"
+ ],
+ "created": "2024-04-11T16:06:15.000-04:00",
+ "kind": "event",
+ "original": "Apr 11 20:06:15 192.168.0.1 01111111111,2024/04/11 20:06:15,audit,2561,gui-op,suser,\"<debug><dataplane><packet-diag><show><setting/></show></packet-diag></dataplane></debug>\",success",
+ "outcome": "success",
+ "timezone": "-04:00"
+ },
+ "message": "2561,gui-op,suser,\"<debug><dataplane><packet-diag><show><setting/></show></packet-diag></dataplane></debug>\",success",
+ "network": {
+ "type": "ipv4"
+ },
+ "observer": {
+ "product": "PAN-OS",
+ "serial_number": "01111111111",
+ "type": "firewall",
+ "vendor": "Palo Alto Networks"
+ },
+ "panw": {
+ "panos": {
+ "cmd": "<debug><dataplane><packet-diag><show><setting/></show></packet-diag></dataplane></debug>",
+ "cmd_source": "gui-op",
+ "config_version": "2561",
+ "type": "AUDIT"
+ }
+ },
+ "related": {
+ "ip": [
+ "192.168.0.1"
+ ]
+ },
+ "source": {
+ "ip": "192.168.0.1"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "suser"
+ }
+ },
+ {
+ "@timestamp": "2024-04-18T18:35:20.000-04:00",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "category": [
+ "configuration"
+ ],
+ "created": "2024-04-18T14:35:20.000-04:00",
+ "kind": "event",
+ "original": "Apr 18 18:35:20 10.1.1.1 003001000000,2024/04/18 18:35:20,audit,2561,gui-op,Mustang,\"<show><config-locks><vsys>all</vsys></config-locks></show>\",success",
+ "outcome": "success",
+ "timezone": "-04:00"
+ },
+ "message": "2561,gui-op,Mustang,\"<show><config-locks><vsys>all</vsys></config-locks></show>\",success",
+ "network": {
+ "type": "ipv4"
+ },
+ "observer": {
+ "product": "PAN-OS",
+ "serial_number": "003001000000",
+ "type": "firewall",
+ "vendor": "Palo Alto Networks"
+ },
+ "panw": {
+ "panos": {
+ "cmd": "<show><config-locks><vsys>all</vsys></config-locks></show>",
+ "cmd_source": "gui-op",
+ "config_version": "2561",
+ "type": "AUDIT"
+ }
+ },
+ "related": {
+ "ip": [
+ "10.1.1.1"
+ ]
+ },
+ "source": {
+ "ip": "10.1.1.1"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "Mustang"
+ }
+ }
+ ]
+}

packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/audit.yml

@@ -0,0 +1,53 @@
+---
+description: Pipeline for PanOS Audit Logs.
+# API Docs: https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/audit-log-fields
+processors:
+ - csv:
+ field: message
+ ignore_failure: true
+ target_fields:
+ - panw.panos.config_version
+ - panw.panos.cmd_source
+ - user.name
+ - panw.panos.cmd
+ - event.outcome
+ - uppercase:
+ field: panw.panos.type
+ ignore_missing: true
+# Set event.kind & event.category field.
+ - set:
+ field: event.kind
+ value: event
+ - append:
+ field: event.category
+ value:
+ - configuration
+
+# Set custom fields to ECS fields
+ - date:
+ if: ctx._temp_?.syslog_time != null
+ field: _temp_.syslog_time
+ target_field: event.created
+ formats:
+ - MMM d HH:mm:ss
+ - MMM dd HH:mm:ss
+ - set:
+ field: observer.hostname
+ copy_from: panw.panos.device_name
+ ignore_failure: true
+ - set:
+ field: source.ip
+ copy_from: panw.panos.source.ip
+ ignore_failure: true
+
+on_failure:
+ - set:
+ field: event.kind
+ value: pipeline_error
+ - append:
+ field: error.message
+ value: >-
+ error in Audit pipeline:
+ error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}}
+ with tag [{{{_ingest.on_failure_processor_tag}}}]{{{/_ingest.on_failure_processor_tag}}}
+ {{{_ingest.on_failure_message}}}

packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/default.yml

@@ -31,6 +31,7 @@ processors:
  field: _temp_.message
  patterns:
  - "^%{DATA},%{TIMESTAMP:event.created},%{FIELD:observer.serial_number},%{FIELD:panw.panos.type},(?:%{FIELD:panw.panos.sub_type})?,%{FIELD:_temp_.config_version},%{TIMESTAMP:_temp_.generated_time},%{GREEDYDATA:message}$"
+ - "^%{SYSLOGTIMESTAMP:_temp_.syslog_time} %{IP:source.ip} %{NOTSPACE:observer.serial_number},%{PANW_DATE:_temp_.generated_time},%{FIELD:panw.panos.type},%{GREEDYDATA:message}$"
  pattern_definitions:
  TIMESTAMP: "%{PANW_DATE}|%{TIMESTAMP_ISO8601}"
  PANW_DATE: "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}"
@@ -92,6 +93,10 @@ processors:
  - pipeline:
  if: ctx.panw?.panos?.type == 'START' || ctx.panw.panos.type == 'END'
  name: '{{ IngestPipeline "tunnel_inspection" }}'
+## AUDIT
+ - pipeline:
+ if: ctx.panw?.panos?.type == 'AUDIT' || ctx.panw?.panos?.type == 'audit'
+ name: '{{ IngestPipeline "audit" }}'
 
  - set:
  field: panw.panos.observer.serial_number

packages/panw/data_stream/panos/fields/fields.yml

@@ -147,7 +147,10 @@
  description: Unique 32 character ID for a file scanned by the DLP cloud service sent by a firewall.
  - name: cmd
  type: keyword
- description: Command performed by the Admin; values are add, clone, commit, delete, edit, move, rename, set.
+ description: Command performed by the Admin; values are add, clone, commit, delete, edit, move, rename, set, or any command generated by cli, gui, gui-opt, gnmi, or rest;
+ - name: cmd_source
+ type: keyword
+ description: 'Source of the command that generated the audit log. Value are: cli, gui, gui-op, gnmi, rest.'
  - name: comment
  type: keyword
  description: The audit comment entered in a policy rule configuration change.
@@ -319,6 +322,9 @@
  - name: id
  type: keyword
  description: A string showing the name of the event.
+ - name: outcome
+ type: keyword
+ description: A string showing the outcome of the event.
  - name: reason
  type: keyword
  description: A string that shows the reason for the quarantine.
@@ -942,7 +948,7 @@
  description: Type of tunnel, such as GRE or IPSec or SSLVPN.
  - name: type
  type: keyword
- description: Specifies the type of log; values are HIP-MATCH, CONFIG, GLOBALPROTECT, THREAT, TRAFFIC, USERID, AUTHENTICATION, CORRELATION, DECRYPTION, GTP, IPTAG, SCTP, SYSTEM.
+ description: Specifies the type of log; values are HIP-MATCH, CONFIG, GLOBALPROTECT, THREAT, TRAFFIC, USERID, AUTHENTICATION, CORRELATION, DECRYPTION, GTP, IPTAG, SCTP, SYSTEM, AUDIT.
  - name: ugflags
  type: keyword
  description: 'Displays whether the user group that was found during user group mapping. Supported values are: User Group Found—Indicates whether the user could be mapped to a group.Duplicate User—Indicates whether duplicate users were found in a user group. Displays N/A if no user group is found.'

packages/panw/docs/README.md

@@ -402,7 +402,8 @@ An example event for `panos` looks as following:
 | panw.panos.client_type | Type of client to used by administrator or complete authentication. | keyword |
 | panw.panos.client_ver | The client's GlobalProtect app version. | keyword |
 | panw.panos.cloud_report.id | Unique 32 character ID for a file scanned by the DLP cloud service sent by a firewall. | keyword |
-| panw.panos.cmd | Command performed by the Admin; values are add, clone, commit, delete, edit, move, rename, set. | keyword |
+| panw.panos.cmd | Command performed by the Admin; values are add, clone, commit, delete, edit, move, rename, set, or any command generated by cli, gui, gui-opt, gnmi, or rest; | keyword |
+| panw.panos.cmd_source | Source of the command that generated the audit log. Value are: cli, gui, gui-op, gnmi, rest. | keyword |
 | panw.panos.comment | The audit comment entered in a policy rule configuration change. | keyword |
 | panw.panos.config_version | The software version. | keyword |
 | panw.panos.connect_method | A string showing the how the GlobalProtect app connects to Gateway, (for example, on-demand or user-logon. | keyword |
@@ -450,6 +451,7 @@ An example event for `panos` looks as following:
 | panw.panos.error_code | An integer associated with any errors that occurred. | integer |
 | panw.panos.error_message | A string showing that error that has occurred in any event. | keyword |
 | panw.panos.event.id | A string showing the name of the event. | keyword |
+| panw.panos.event.outcome | A string showing the outcome of the event. | keyword |
 | panw.panos.event.reason | A string that shows the reason for the quarantine. | keyword |
 | panw.panos.event.result | Result of the authentication attempt. | keyword |
 | panw.panos.event.status | The status (success or failure) of the event. | keyword |
@@ -625,7 +627,7 @@ An example event for `panos` looks as following:
 | panw.panos.tunnel_fragment | Number of packets the firewall dropped because of fragmentation errors. | long |
 | panw.panos.tunnel_inspection_rule | Name of the tunnel inspection rule matching the cleartext tunnel traffic. | keyword |
 | panw.panos.tunnel_type | Type of tunnel, such as GRE or IPSec or SSLVPN. | keyword |
-| panw.panos.type | Specifies the type of log; values are HIP-MATCH, CONFIG, GLOBALPROTECT, THREAT, TRAFFIC, USERID, AUTHENTICATION, CORRELATION, DECRYPTION, GTP, IPTAG, SCTP, SYSTEM. | keyword |
+| panw.panos.type | Specifies the type of log; values are HIP-MATCH, CONFIG, GLOBALPROTECT, THREAT, TRAFFIC, USERID, AUTHENTICATION, CORRELATION, DECRYPTION, GTP, IPTAG, SCTP, SYSTEM, AUDIT. | keyword |
 | panw.panos.ugflags | Displays whether the user group that was found during user group mapping. Supported values are: User Group Found—Indicates whether the user could be mapped to a group.Duplicate User—Indicates whether duplicate users were found in a user group. Displays N/A if no user group is found. | keyword |
 | panw.panos.unknown_protocol | Number of packets the firewall dropped because the packet contains an unknown protocol, as enabled in the Tunnel Inspection policy rule (Drop packet if unknown protocol inside tunnel). | long |
 | panw.panos.url.category | For URL Subtype, it is the URL Category; For WildFire subtype, it is the verdict on the file and is either ‘malware’, ‘phishing’, ‘grayware’, or ‘benign’; For other subtypes, the value is ‘any’. | keyword |

packages/panw/manifest.yml

@@ -1,6 +1,6 @@
 name: panw
 title: Palo Alto Next-Gen Firewall
-version: "3.24.4"
+version: "3.25.0"
 description: Collect logs from Palo Alto next-gen firewalls with Elastic Agent.
 type: integration
 format_version: "3.0.3"