elastic · marc-gr · May 14, 2021 · May 14, 2021
diff --git a/packages/auditd/changelog.yml b/packages/auditd/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.1.2"
+ changes:
+ - description: set version in the ingest pipeline and make event.original optional
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/989
 - version: "0.1.1"
  changes:
  - description: update to ECS 1.9.0

diff --git a/packages/auditd/data_stream/log/_dev/test/pipeline/test-auditd-raw.log-expected.json b/packages/auditd/data_stream/log/_dev/test/pipeline/test-auditd-raw.log-expected.json
diff --git a/packages/auditd/data_stream/log/_dev/test/pipeline/test-auditd-useradd.log-expected.json b/packages/auditd/data_stream/log/_dev/test/pipeline/test-auditd-useradd.log-expected.json
@@ -6,24 +6,26 @@
  "executable": "/usr/sbin/groupadd"
  },
  "@timestamp": "2021-01-17T17:12:33.686Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749045516Z",
- "original": "type=ADD_GROUP msg=audit(1610903553.686:584): pid=2940 uid=0 auid=1000 ses=14 msg='op=adding group to /etc/group id=1004 exe=\"/usr/sbin/groupadd\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "added-group-account-to"
  ],
+ "ingested": "2021-05-14T09:45:49.161706500Z",
  "category": [
  "iam"
  ],
  "type": [
  "group",
  "creation"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -57,24 +59,26 @@
  "executable": "/usr/sbin/groupadd"
  },
  "@timestamp": "2021-01-17T17:12:33.710Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749056502Z",
- "original": "type=ADD_GROUP msg=audit(1610903553.710:586): pid=2940 uid=0 auid=1000 ses=14 msg='op=adding group to /etc/gshadow id=1004 exe=\"/usr/sbin/groupadd\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "added-group-account-to"
  ],
+ "ingested": "2021-05-14T09:45:49.161722100Z",
  "category": [
  "iam"
  ],
  "type": [
  "group",
  "creation"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -108,24 +112,26 @@
  "executable": "/usr/sbin/groupadd"
  },
  "@timestamp": "2021-01-17T17:12:33.710Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749058427Z",
- "original": "type=ADD_GROUP msg=audit(1610903553.710:587): pid=2940 uid=0 auid=1000 ses=14 msg='op= id=1004 exe=\"/usr/sbin/groupadd\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "added-group-account-to"
  ],
+ "ingested": "2021-05-14T09:45:49.161732100Z",
  "category": [
  "iam"
  ],
  "type": [
  "group",
  "creation"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -158,24 +164,26 @@
  "executable": "/usr/sbin/useradd"
  },
  "@timestamp": "2021-01-17T17:12:33.730Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749060058Z",
- "original": "type=ADD_USER msg=audit(1610903553.730:591): pid=2945 uid=0 auid=1000 ses=14 msg='op=adding user id=1004 exe=\"/usr/sbin/useradd\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "added-user-account"
  ],
+ "ingested": "2021-05-14T09:45:49.161741600Z",
  "category": [
  "iam"
  ],
  "type": [
  "user",
  "creation"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -209,23 +217,25 @@
  "executable": "/sbin/pam_tally2"
  },
  "@timestamp": "2021-01-17T17:12:33.814Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749061593Z",
- "original": "type=USER_ACCT msg=audit(1610903553.814:593): pid=2948 uid=0 auid=1000 ses=14 msg='pam_tally2 uid=1004 reset=0 exe=\"/sbin/pam_tally2\" hostname=localhost addr=127.0.0.1 terminal=/dev/pts/2 res=success'",
- "kind": "event",
  "action": [
  "was-authorized"
  ],
+ "ingested": "2021-05-14T09:45:49.161751Z",
  "category": [
  "authentication"
  ],
  "type": [
  "info"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -255,24 +265,26 @@
  "executable": "/usr/bin/passwd"
  },
  "@timestamp": "2021-01-17T17:12:38.174Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749063122Z",
- "original": "type=USER_CHAUTHTOK msg=audit(1610903558.174:594): pid=2953 uid=0 auid=1000 ses=14 msg='op=PAM:chauthtok acct=\"charlie\" exe=\"/usr/bin/passwd\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "changed-password"
  ],
+ "ingested": "2021-05-14T09:45:49.161760300Z",
  "category": [
  "iam"
  ],
  "type": [
  "user",
  "change"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -306,23 +318,25 @@
  "executable": "/usr/bin/chfn"
  },
  "@timestamp": "2021-01-17T17:12:38.178Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749064635Z",
- "original": "type=USER_AUTH msg=audit(1610903558.178:595): pid=2954 uid=0 auid=1000 ses=14 msg='op=PAM:authentication acct=\"root\" exe=\"/usr/bin/chfn\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "authenticated"
  ],
+ "ingested": "2021-05-14T09:45:49.161769900Z",
  "category": [
  "authentication"
  ],
  "type": [
  "info"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {
@@ -353,23 +367,25 @@
  "executable": "/usr/bin/chfn"
  },
  "@timestamp": "2021-01-17T17:12:38.178Z",
+ "ecs": {
+ "version": "1.9.0"
+ },
  "source": {
  "address": "127.0.0.1",
  "ip": "127.0.0.1"
  },
  "event": {
- "ingested": "2021-04-23T12:52:54.749066130Z",
- "original": "type=USER_ACCT msg=audit(1610903558.178:596): pid=2954 uid=0 auid=1000 ses=14 msg='op=PAM:accounting acct=\"root\" exe=\"/usr/bin/chfn\" hostname=ubuntu-bionic addr=127.0.0.1 terminal=pts/2 res=success'",
- "kind": "event",
  "action": [
  "was-authorized"
  ],
+ "ingested": "2021-05-14T09:45:49.161777800Z",
  "category": [
  "authentication"
  ],
  "type": [
  "info"
  ],
+ "kind": "event",
  "outcome": "success"
  },
  "auditd": {

diff --git a/packages/auditd/data_stream/log/agent/stream/log.yml.hbs b/packages/auditd/data_stream/log/agent/stream/log.yml.hbs
@@ -3,8 +3,7 @@ paths:
  - {{path}}
 {{/each}}
 exclude_files: [".gz$"]
-processors:
- - add_fields:
- target: ''
- fields:
- ecs.version: 1.9.0
+tags:
+{{#if preserve_original_event}}
+- preserve_original_event
+{{/if}}
diff --git a/packages/auditd/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/auditd/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -4,8 +4,15 @@ processors:
 - set:
  field: event.ingested
  value: '{{_ingest.timestamp}}'
-- grok:
+- set:
+ field: ecs.version
+ value: 1.9.0
+- rename:
  field: message
+ target_field: event.original
+ ignore_failure: true
+- grok:
+ field: event.original
  pattern_definitions:
  AUDIT_TYPE: "type=%{NOTSPACE:auditd.log.record_type}"
  AUDIT_NODE: "node=%{IPORHOST:auditd.log.node} "
@@ -31,10 +38,6 @@ processors:
  value_split: "="
  target_field: auditd.log
  ignore_missing: true
-- rename:
- field: message
- target_field: event.original
- ignore_failure: true
 - date:
  field: auditd.log.epoch
  target_field: "@timestamp"
@@ -2159,6 +2162,11 @@ processors:
  - auditd.log.copy
  ignore_failure: true
  ignore_missing: true
+- remove:
+ field: event.original
+ if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+ ignore_failure: true
+ ignore_missing: true
 on_failure:
 - set:
  field: error.message

diff --git a/packages/auditd/data_stream/log/fields/base-fields.yml b/packages/auditd/data_stream/log/fields/base-fields.yml
@@ -10,3 +10,8 @@
 - name: '@timestamp'
  type: date
  description: Event timestamp.
+- name: tags
+ description: List of keywords used to tag each event.
+ example: '["production", "env2"]'
+ ignore_above: 1024
+ type: keyword
diff --git a/packages/auditd/data_stream/log/manifest.yml b/packages/auditd/data_stream/log/manifest.yml
@@ -12,6 +12,14 @@ streams:
  show_user: true
  default:
  - /var/log/audit/audit.log*
+ - name: preserve_original_event
+ required: true
+ show_user: true
+ title: Preserve original event
+ description: Preserves a raw copy of the original event, added to the field `event.original`
+ type: bool
+ multi: false
+ default: false
  template_path: log.yml.hbs
  title: Auditd logs
  description: Collect Auditd logs using log input
diff --git a/packages/auditd/docs/README.md b/packages/auditd/docs/README.md
@@ -198,13 +198,14 @@ An example event for `log` looks as following:
 | source.geo.region_iso_code | Region ISO code. | keyword |
 | source.geo.region_name | Region name. | keyword |
 | source.ip | IP address of the source (IPv4 or IPv6). | ip |
+| tags | List of keywords used to tag each event. | keyword |
 | user.audit.group.id | Unique identifier for the group on the system/platform. | keyword |
 | user.audit.group.name | Name of the group. | keyword |
 | user.audit.id | One or multiple unique identifiers of the user. | keyword |
 | user.audit.name | Short name or login of the user. | keyword |
 | user.effective.group.id | Unique identifier for the group on the system/platform. | keyword |
 | user.effective.group.name | Name of the group. | keyword |
-| user.effective.id | One or multiple unique identifiers of the user. | keyword |
+| user.effective.id | Unique identifier of the user. | keyword |
 | user.effective.name | Short name or login of the user. | keyword |
 | user.filesystem.group.id | Unique identifier for the group on the system/platform. | keyword |
 | user.filesystem.group.name | Name of the group. | keyword |

diff --git a/packages/auditd/manifest.yml b/packages/auditd/manifest.yml
@@ -1,6 +1,6 @@
 name: auditd
 title: Auditd
-version: 0.1.1
+version: 0.1.2
 release: experimental
 description: Auditd Integration
 type: integration