elastic · yug-rajani · Jul 18, 2022 · Jun 27, 2022 · Jun 27, 2022 · Jun 28, 2022
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -72,6 +72,7 @@
 /packages/hid_bravura_monitor @elastic/security-external-integrations
 /packages/http_endpoint @elastic/security-external-integrations
 /packages/httpjson @elastic/security-external-integrations
+/packages/ibmmq @elastic/obs-service-integrations
 /packages/iis @elastic/obs-service-integrations
 /packages/imperva @elastic/security-external-integrations
 /packages/infoblox @elastic/security-external-integrations

@@ -0,0 +1,3 @@
+dependencies:
+ ecs:
+ reference: git@v8.2.0
@@ -0,0 +1,22 @@
+# IBM MQ integration
+
+The IBM MQ Integration is used to fetch observability data from [IBM MQ web endpoints](https://www.ibm.com/docs/en/ibm-mq) and ingest it into Elasticsearch.
+
+## Compatibility
+
+This integration has been tested against `IBM MQ v9.1` and `IBM MQ v9.2`.
+
+## Requirements
+
+In order to ingest data from IBM MQ:
+- User should specify the path of IBM MQ Queue Manager Error logs. (default paths: `/var/mqm/errors/*.LOG` and `/var/mqm/qmgrs/*/errors/*.LOG`)
+
+## Logs
+
+### Queue Manager Error logs
+
+The `errorlog` data stream collects [Error logs of Queue Manager](https://www.site24x7.com/help/log-management/ibm-mq-error-logs.html) which include the description, action, explanation and code of the error.
+
+{{event "errorlog"}}
+
+{{fields "errorlog"}}
@@ -0,0 +1,18 @@
+version: '2.3'
+services:
+ ibmmq:
+ image: ibmcom/mq:${SERVICE_VERSION:-v9.2.4.0-r1-amd64}
+ environment:
+ - LICENSE=accept
+ - MQ_QMGR_NAME=QM1
+ - MQ_ENABLE_METRICS=true
+ user: root
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:9157/metrics"]
+ interval: 15s
+ timeout: 30s
+ retries: 40
+ ports:
+ - 9157
+ volumes:
+ - ${SERVICE_LOGS_DIR}:/var/mqm/errors:rw
@@ -0,0 +1,4 @@
+variants:
+ v9.2.4.0-r1-amd64:
+ SERVICE_VERSION: 9.2.4.0-r1-amd64
+default: v9.2.4.0-r1-amd64
@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: 0.1.0
+ changes:
+ - description: IBM MQ integration package with "errorlog" data stream
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/3590
@@ -0,0 +1,9 @@
+dynamic_fields:
+ event.ingested: ".*"
+fields:
+ tags:
+ - preserve_original_event
+multiline:
+ pattern: "^[\\-]{5}.*[\\-]{10,}$"
+ negate: true
+ match: after
@@ -0,0 +1,3 @@
+06/14/22 11:54:29 - Process(119.8) User(mqm) Program(amqzmur0) Host(00c6b2142371) Installation(Installation1) VRMF(9.2.4.0) QMgr(QM1) Time(2022-06-14T11:54:29.349Z) ArithInsert2(3) CommentInsert1(APP-SIGNAL) AMQ5037I: The queue manager task 'APP-SIGNAL' has started. EXPLANATION: The restartable utility task manager has started the APP-SIGNAL task. This task has now started 3 times. ACTION: None.
+06/13/22 06:53:29 - Process(60.1) User(mqm) Program(crtmqm) Host(2661e9dd5c0b) Installation(Installation1) VRMF(9.2.4.0) Time(2022-06-13T06:53:29.794Z) CommentInsert3(QM1) AMQ8001I: IBM MQ queue manager 'QM1' created. EXPLANATION: IBM MQ queue manager 'QM1' created. ACTION: None.
+06/15/22 06:10:59 - Process(597.4) User(mqm) Program(amqrmppa) Host(00c6b2142371) Installation(Installation1) VRMF(9.2.4.0) QMgr(QM1) Time(2022-06-15T06:10:59.254Z) RemoteHost(10.212.128.113) ArithInsert2(TCP/IP) CommentInsert1(10.212.128.113) CommentInsert3(????) AMQ9207E: The data received from host '10.212.128.113' on channel '????' is not valid. EXPLANATION: Incorrect data format received from host '10.212.128.113' over TCP/IP. It my be that an unknown host is attempting to send data. An FFST file might be generated containing the invalid data received. It will not be generated if this is the beginning of a conversation with the remote side, and the format is a simple known format (example: a GET request from an HTTP web browser). If you want to override this, to cause FFST files to be written for any bad data, including simple known formats, then set the environment variable AMQ_BAD_COMMS_DATA_FDCS=TRUE and restart the queue manager. The channel name is '????'; in some cases it cannot be determined and so is shown as '????'. ACTION: Tell the systems administrator.
@@ -0,0 +1,151 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2022-06-14T11:54:29.349Z",
+ "ecs": {
+ "version": "8.2.0"
+ },
+ "event": {
+ "ingested": "2022-06-29T08:24:38.696773196Z",
+ "kind": "event",
+ "module": "ibmmq",
+ "original": "06/14/22 11:54:29 - Process(119.8) User(mqm) Program(amqzmur0) Host(00c6b2142371) Installation(Installation1) VRMF(9.2.4.0) QMgr(QM1) Time(2022-06-14T11:54:29.349Z) ArithInsert2(3) CommentInsert1(APP-SIGNAL) AMQ5037I: The queue manager task 'APP-SIGNAL' has started. EXPLANATION: The restartable utility task manager has started the APP-SIGNAL task. This task has now started 3 times. ACTION: None.",
+ "type": "error"
+ },
+ "host": {
+ "hostname": "00c6b2142371"
+ },
+ "ibmmq": {
+ "errorlog": {
+ "error": {
+ "action": "None.",
+ "code": "AMQ5037I",
+ "description": "The queue manager task 'APP-SIGNAL' has started.",
+ "explanation": "The restartable utility task manager has started the APP-SIGNAL task. This task has now started 3 times."
+ },
+ "insert": {
+ "arith": [
+ "3"
+ ],
+ "comment": [
+ "APP-SIGNAL"
+ ]
+ },
+ "installation": "Installation1",
+ "queue_manager": "QM1"
+ }
+ },
+ "process": {
+ "pid": 119.8,
+ "title": "amqzmur0"
+ },
+ "service": {
+ "version": "9.2.4.0"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "mqm"
+ }
+ },
+ {
+ "@timestamp": "2022-06-13T06:53:29.794Z",
+ "ecs": {
+ "version": "8.2.0"
+ },
+ "event": {
+ "ingested": "2022-06-29T08:24:38.696781941Z",
+ "kind": "event",
+ "module": "ibmmq",
+ "original": "06/13/22 06:53:29 - Process(60.1) User(mqm) Program(crtmqm) Host(2661e9dd5c0b) Installation(Installation1) VRMF(9.2.4.0) Time(2022-06-13T06:53:29.794Z) CommentInsert3(QM1) AMQ8001I: IBM MQ queue manager 'QM1' created. EXPLANATION: IBM MQ queue manager 'QM1' created. ACTION: None.",
+ "type": "error"
+ },
+ "host": {
+ "hostname": "2661e9dd5c0b"
+ },
+ "ibmmq": {
+ "errorlog": {
+ "error": {
+ "action": "None.",
+ "code": "AMQ8001I",
+ "description": "IBM MQ queue manager 'QM1' created.",
+ "explanation": "IBM MQ queue manager 'QM1' created."
+ },
+ "insert": {
+ "comment": [
+ "QM1"
+ ]
+ },
+ "installation": "Installation1"
+ }
+ },
+ "process": {
+ "pid": 60.1,
+ "title": "crtmqm"
+ },
+ "service": {
+ "version": "9.2.4.0"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "mqm"
+ }
+ },
+ {
+ "@timestamp": "2022-06-15T06:10:59.254Z",
+ "destination": {
+ "address": "10.212.128.113"
+ },
+ "ecs": {
+ "version": "8.2.0"
+ },
+ "event": {
+ "ingested": "2022-06-29T08:24:38.696783965Z",
+ "kind": "event",
+ "module": "ibmmq",
+ "original": "06/15/22 06:10:59 - Process(597.4) User(mqm) Program(amqrmppa) Host(00c6b2142371) Installation(Installation1) VRMF(9.2.4.0) QMgr(QM1) Time(2022-06-15T06:10:59.254Z) RemoteHost(10.212.128.113) ArithInsert2(TCP/IP) CommentInsert1(10.212.128.113) CommentInsert3(????) AMQ9207E: The data received from host '10.212.128.113' on channel '????' is not valid. EXPLANATION: Incorrect data format received from host '10.212.128.113' over TCP/IP. It my be that an unknown host is attempting to send data. An FFST file might be generated containing the invalid data received. It will not be generated if this is the beginning of a conversation with the remote side, and the format is a simple known format (example: a GET request from an HTTP web browser). If you want to override this, to cause FFST files to be written for any bad data, including simple known formats, then set the environment variable AMQ_BAD_COMMS_DATA_FDCS=TRUE and restart the queue manager. The channel name is '????'; in some cases it cannot be determined and so is shown as '????'. ACTION: Tell the systems administrator.",
+ "type": "error"
+ },
+ "host": {
+ "hostname": "00c6b2142371"
+ },
+ "ibmmq": {
+ "errorlog": {
+ "error": {
+ "action": "a GET request from an HTTP web browser). If you want to override this, to cause FFST files to be written for any bad data, including simple known formats, then set the environment variable AMQ_BAD_COMMS_DATA_FDCS=TRUE and restart the queue manager. The channel name is '????'; in some cases it cannot be determined and so is shown as '????'. ACTION: Tell the systems administrator.",
+ "code": "AMQ9207E",
+ "description": "The data received from host '10.212.128.113' on channel '????' is not valid.",
+ "explanation": "Incorrect data format received from host '10.212.128.113' over TCP/IP. It my be that an unknown host is attempting to send data. An FFST file might be generated containing the invalid data received. It will not be generated if this is the beginning of a conversation with the remote side, and the format is a simple known format"
+ },
+ "insert": {
+ "arith": [
+ "TCP/IP"
+ ],
+ "comment": [
+ "10.212.128.113",
+ "????"
+ ]
+ },
+ "installation": "Installation1",
+ "queue_manager": "QM1"
+ }
+ },
+ "process": {
+ "pid": 597.4,
+ "title": "amqrmppa"
+ },
+ "service": {
+ "version": "9.2.4.0"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "mqm"
+ }
+ }
+ ]
+}
@@ -0,0 +1,6 @@
+service: ibmmq
+input: logfile
+data_stream:
+ vars:
+ paths:
+ - "{{SERVICE_LOGS_DIR}}/*.LOG"
@@ -0,0 +1,23 @@
+paths:
+{{#each paths as |path|}}
+ - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+multiline:
+ pattern: "^[\\-]{5}.*[\\-]{10,}$"
+ negate: true
+ match: after
+tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+ - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
@@ -0,0 +1,128 @@
+---
+description: Pipeline for parsing MQ Queue Manager Error logs.
+processors:
+- set:
+ field: event.ingested
+ value: "{{{_ingest.timestamp}}}"
+ ignore_failure: true
+- set:
+ field: ecs.version
+ value: 8.2.0
+ ignore_empty_value: true
+ ignore_failure: true
+- set:
+ field: event.module
+ value: ibmmq
+ ignore_empty_value: true
+ ignore_failure: true
+- set:
+ field: event.kind
+ value: event
+ ignore_empty_value: true
+ ignore_failure: true
+- set:
+ field: event.type
+ value: error
+ ignore_empty_value: true
+ ignore_failure: true
+- gsub:
+ field: message
+ pattern: "^[\\-]{5}[a-z0-9\\. :]*[\\-]{5,}"
+ replacement: ''
+- gsub:
+ field: message
+ pattern: "\n"
+ replacement: " "
+- gsub:
+ field: message
+ pattern: "[ ]{2,}"
+ replacement: " "
+- trim:
+ field: message
+- set:
+ copy_from: "@timestamp"
+ field: event.created
+ ignore_failure: true
+- grok:
+ field: message
+ patterns:
+ - "^%{DATA:log_timestamp} -"
+- grok:
+ field: message
+ patterns:
+ - 'Process\(%{DATA:process.pid}\) User\(%{WORD:user.name}\) Program\(%{DATA:process.title}\)
+ Host\(%{DATA:host.hostname}\) Installation\(%{WORD:ibmmq.errorlog.installation}\)
+ VRMF\(%{DATA:service.version}\)( QMgr\(%{DATA:ibmmq.errorlog.queue_manager}\))?( Time\(%{TIMESTAMP_ISO8601:log_timestamp}\))?(
+ RemoteHost\(%{DATA:destination.address}\))?( ArithInsert1\(%{DATA:ibmmq.errorlog.arithinsert1}\))?(
+ ArithInsert2\(%{DATA:ibmmq.errorlog.arithinsert2}\))?( CommentInsert1\(%{DATA:ibmmq.errorlog.commentinsert1}\))?(
+ CommentInsert2\(%{DATA:ibmmq.errorlog.commentinsert2}\))?( CommentInsert3\(%{DATA:ibmmq.errorlog.commentinsert3}\))?
+ (?=AMQ[0-9]{4})%{DATA:ibmmq.errorlog.error.code}((?<=AMQ[0-9]{4}[A-Z])%{DATA:log.level})?:
+ %{DATA:ibmmq.errorlog.error.description} [^\ ]+:( %{DATA:ibmmq.errorlog.error.explanation})?
+ [^\ ]+:( %{DATA:ibmmq.errorlog.error.action})?$'
+- convert:
+ field: process.pid
+ type: float
+ ignore_failure: true
+- date:
+ field: log_timestamp
+ target_field: "@timestamp"
+ formats:
+ - ISO8601
+ - MM/dd/yyyy hh:mm:ss a
+ - dd/MM/yyyy HH:mm:ss
+ - dd.MM.yyyy HH:mm:ss
+ ignore_failure: true
+- append:
+ field: ibmmq.errorlog.insert.comment
+ value:
+ - "{{ibmmq.errorlog.commentinsert1}}"
+ - "{{ibmmq.errorlog.commentinsert2}}"
+ - "{{ibmmq.errorlog.commentinsert3}}"
+ ignore_failure: true
+- append:
+ field: ibmmq.errorlog.insert.arith
+ value:
+ - "{{ibmmq.errorlog.arithinsert1}}"
+ - "{{ibmmq.errorlog.arithinsert2}}"
+ ignore_failure: true
+- rename:
+ field: message
+ target_field: event.original
+ ignore_missing: true
+ ignore_failure: true
+- remove:
+ field:
+ - log_timestamp
+ - ibmmq.errorlog.arithinsert1
+ - ibmmq.errorlog.arithinsert2
+ - ibmmq.errorlog.commentinsert1
+ - ibmmq.errorlog.commentinsert2
+ - ibmmq.errorlog.commentinsert3
+ ignore_missing: true
+ ignore_failure: true
+- remove:
+ field: event.original
+ if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+ ignore_missing: true
+ ignore_failure: true
+- script:
+ description: Drops null/empty values recursively.
+ lang: painless
+ source: |
+ boolean drop(Object o) {
+ if (o == null || o == "") {
+ return true;
+ } else if (o instanceof Map) {
+ ((Map) o).values().removeIf(v -> drop(v));
+ return (((Map) o).size() == 0);
+ } else if (o instanceof List) {
+ ((List) o).removeIf(v -> drop(v));
+ return (((List) o).length == 0);
+ }
+ return false;
+ }
+ drop(ctx);
+on_failure:
+ - set:
+ field: error.message
+ value: "{{{_ingest.on_failure_message}}}"