Skip to content

Commit 1a80dc6

Browse files
agithomasagithomas
authored
[AmazonMQ] Add firehose routing rules for ActiveMQ and RabbitMQ metrics (#13342)
* AmazonMQ Add firehose routing rules for ActiveMQ and RabbitMQ metrics
1 parent 274930b commit 1a80dc6

File tree

5 files changed

+388
-1
lines changed

5 files changed

+388
-1
lines changed

‎packages/awsfirehose/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.6.0"
3+
changes:
4+
- description: Add support for AmazonMQ metrics for both managed RabbitMQ and ActiveMQ.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/13342
27
- version: "1.5.2"
38
changes:
49
- description: Avoid using dynamic template for flattened fields

‎packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-mq-metrics.json‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
{
2+
"events": [
3+
{
4+
"@timestamp": "2025-03-28T09:13:00.000Z",
5+
"start_timestamp": "2025-03-28T09:12:00.000Z",
6+
"agent.type": "firehose",
7+
"cloud.provider": "aws",
8+
"cloud.account.id": "111111111111111111",
9+
"cloud.region": "ap-south-1",
10+
"aws.exporter.arn": "arn:aws:cloudwatch:ap-south-1:627286350134:metric-stream/AWSBedrockMetricStream",
11+
"aws.cloudwatch.namespace": "AWS/AmazonMQ",
12+
"aws.firehose.arn": "arn:aws:firehose:ap-south-1:627286350134:deliverystream/AWS-Bedrok-Firehose-Stream-AgiKThomas",
13+
"aws.firehose.request_id": "bf51208a-d776-4161-b2ce-87ff64a6b095",
14+
"aws.dimensions": {
15+
"Broker": "ObsIntegrations-RabbitMQ",
16+
"Queue": "rabbit-mq-message-queue",
17+
"VirtualHost": "/"
18+
},
19+
"aws.amazonmq.metrics.MessageCount": {
20+
"count": 1,
21+
"sum": 0,
22+
"avg": 0,
23+
"min": 0,
24+
"max": 0
25+
},
26+
"aws.amazonmq.metrics.ConsumerCount": {
27+
"count": 1,
28+
"sum": 0,
29+
"avg": 0,
30+
"min": 0,
31+
"max": 0
32+
},
33+
"aws.amazonmq.metrics.MessageUnacknowledgedCount": {
34+
"count": 1,
35+
"sum": 0,
36+
"avg": 0,
37+
"min": 0,
38+
"max": 0
39+
},
40+
"aws.amazonmq.metrics.MessageReadyCount": {
41+
"count": 1,
42+
"sum": 0,
43+
"avg": 0,
44+
"min": 0,
45+
"max": 0
46+
},
47+
"data_stream.type": "metrics",
48+
"data_stream.dataset": "aws.cloudwatch",
49+
"data_stream.namespace": "default"
50+
},
51+
{
52+
"@timestamp": "2025-03-28T09:13:00.000Z",
53+
"start_timestamp": "2025-03-28T09:12:00.000Z",
54+
"agent.type": "firehose",
55+
"cloud.provider": "aws",
56+
"cloud.account.id": "111111111111111111",
57+
"cloud.region": "ap-south-1",
58+
"aws.exporter.arn": "arn:aws:cloudwatch:ap-south-1:627286350134:metric-stream/AWSBedrockMetricStream",
59+
"aws.cloudwatch.namespace": "AWS/AmazonMQ",
60+
"aws.firehose.arn": "arn:aws:firehose:ap-south-1:627286350134:deliverystream/AWS-Bedrok-Firehose-Stream-AgiKThomas",
61+
"aws.firehose.request_id": "bf51208a-d776-4161-b2ce-87ff64a6b095",
62+
"aws.dimensions": {
63+
"Broker": "ObsIntegrations-ActiveMQ-1",
64+
"Queue": "testQueue"
65+
},
66+
"aws.amazonmq.metrics.ExpiredCount": {
67+
"count": 1,
68+
"sum": 0,
69+
"avg": 0,
70+
"min": 0,
71+
"max": 0
72+
},
73+
"aws.amazonmq.metrics.ProducerCount": {
74+
"count": 1,
75+
"sum": 0,
76+
"avg": 0,
77+
"min": 0,
78+
"max": 0
79+
},
80+
"aws.amazonmq.metrics.InFlightCount": {
81+
"count": 1,
82+
"sum": 0,
83+
"avg": 0,
84+
"min": 0,
85+
"max": 0
86+
},
87+
"aws.amazonmq.metrics.DequeueCount": {
88+
"count": 1,
89+
"sum": 0,
90+
"avg": 0,
91+
"min": 0,
92+
"max": 0
93+
},
94+
"aws.amazonmq.metrics.MemoryUsage": {
95+
"count": 1,
96+
"sum": 0,
97+
"avg": 0,
98+
"min": 0,
99+
"max": 0
100+
},
101+
"aws.amazonmq.metrics.EnqueueCount": {
102+
"count": 1,
103+
"sum": 0,
104+
"avg": 0,
105+
"min": 0,
106+
"max": 0
107+
},
108+
"aws.amazonmq.metrics.QueueSize": {
109+
"count": 1,
110+
"sum": 9,
111+
"avg": 9,
112+
"min": 9,
113+
"max": 9
114+
},
115+
"aws.amazonmq.metrics.DispatchCount": {
116+
"count": 1,
117+
"sum": 0,
118+
"avg": 0,
119+
"min": 0,
120+
"max": 0
121+
},
122+
"aws.amazonmq.metrics.ConsumerCount": {
123+
"count": 1,
124+
"sum": 0,
125+
"avg": 0,
126+
"min": 0,
127+
"max": 0
128+
},
129+
"aws.amazonmq.metrics.EnqueueTime": {
130+
"count": 2,
131+
"sum": 0,
132+
"avg": 0,
133+
"min": 0,
134+
"max": 0
135+
},
136+
"data_stream.type": "metrics",
137+
"data_stream.dataset": "aws.cloudwatch",
138+
"data_stream.namespace": "default"
139+
}
140+
141+
]
142+
}

‎packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-mq-metrics.json-expected.json‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
{
2+
"expected": [
3+
{
4+
"@timestamp": "2025-03-28T09:13:00.000Z",
5+
"agent": {
6+
"type": "firehose"
7+
},
8+
"aws": {
9+
"amazonmq": {
10+
"metrics": {
11+
"ConsumerCount": {
12+
"avg": 0,
13+
"count": 1,
14+
"max": 0,
15+
"min": 0,
16+
"sum": 0
17+
},
18+
"MessageCount": {
19+
"avg": 0,
20+
"count": 1,
21+
"max": 0,
22+
"min": 0,
23+
"sum": 0
24+
},
25+
"MessageReadyCount": {
26+
"avg": 0,
27+
"count": 1,
28+
"max": 0,
29+
"min": 0,
30+
"sum": 0
31+
},
32+
"MessageUnacknowledgedCount": {
33+
"avg": 0,
34+
"count": 1,
35+
"max": 0,
36+
"min": 0,
37+
"sum": 0
38+
}
39+
}
40+
},
41+
"cloudwatch": {
42+
"namespace": "AWS/AmazonMQ"
43+
},
44+
"dimensions": {
45+
"Broker": "ObsIntegrations-RabbitMQ",
46+
"Queue": "rabbit-mq-message-queue",
47+
"VirtualHost": "/"
48+
},
49+
"exporter": {
50+
"arn": "arn:aws:cloudwatch:ap-south-1:627286350134:metric-stream/AWSBedrockMetricStream"
51+
},
52+
"firehose": {
53+
"arn": "arn:aws:firehose:ap-south-1:627286350134:deliverystream/AWS-Bedrok-Firehose-Stream-AgiKThomas",
54+
"request_id": "bf51208a-d776-4161-b2ce-87ff64a6b095"
55+
},
56+
"metrics_names_fingerprint": "Oo6DWr7SrPalaNAWIjgrXXLyKIg="
57+
},
58+
"cloud": {
59+
"account": {
60+
"id": "111111111111111111"
61+
},
62+
"provider": "aws",
63+
"region": "ap-south-1"
64+
},
65+
"data_stream": {
66+
"dataset": "aws_mq.rabbitmq_metrics",
67+
"namespace": "default",
68+
"type": "metrics"
69+
},
70+
"ecs": {
71+
"version": "8.11.0"
72+
},
73+
"start_timestamp": "2025-03-28T09:12:00.000Z"
74+
},
75+
{
76+
"@timestamp": "2025-03-28T09:13:00.000Z",
77+
"agent": {
78+
"type": "firehose"
79+
},
80+
"aws": {
81+
"amazonmq": {
82+
"metrics": {
83+
"ConsumerCount": {
84+
"avg": 0,
85+
"count": 1,
86+
"max": 0,
87+
"min": 0,
88+
"sum": 0
89+
},
90+
"DequeueCount": {
91+
"avg": 0,
92+
"count": 1,
93+
"max": 0,
94+
"min": 0,
95+
"sum": 0
96+
},
97+
"DispatchCount": {
98+
"avg": 0,
99+
"count": 1,
100+
"max": 0,
101+
"min": 0,
102+
"sum": 0
103+
},
104+
"EnqueueCount": {
105+
"avg": 0,
106+
"count": 1,
107+
"max": 0,
108+
"min": 0,
109+
"sum": 0
110+
},
111+
"EnqueueTime": {
112+
"avg": 0,
113+
"count": 2,
114+
"max": 0,
115+
"min": 0,
116+
"sum": 0
117+
},
118+
"ExpiredCount": {
119+
"avg": 0,
120+
"count": 1,
121+
"max": 0,
122+
"min": 0,
123+
"sum": 0
124+
},
125+
"InFlightCount": {
126+
"avg": 0,
127+
"count": 1,
128+
"max": 0,
129+
"min": 0,
130+
"sum": 0
131+
},
132+
"MemoryUsage": {
133+
"avg": 0,
134+
"count": 1,
135+
"max": 0,
136+
"min": 0,
137+
"sum": 0
138+
},
139+
"ProducerCount": {
140+
"avg": 0,
141+
"count": 1,
142+
"max": 0,
143+
"min": 0,
144+
"sum": 0
145+
},
146+
"QueueSize": {
147+
"avg": 9,
148+
"count": 1,
149+
"max": 9,
150+
"min": 9,
151+
"sum": 9
152+
}
153+
}
154+
},
155+
"cloudwatch": {
156+
"namespace": "AWS/AmazonMQ"
157+
},
158+
"dimensions": {
159+
"Broker": "ObsIntegrations-ActiveMQ-1",
160+
"Queue": "testQueue"
161+
},
162+
"exporter": {
163+
"arn": "arn:aws:cloudwatch:ap-south-1:627286350134:metric-stream/AWSBedrockMetricStream"
164+
},
165+
"firehose": {
166+
"arn": "arn:aws:firehose:ap-south-1:627286350134:deliverystream/AWS-Bedrok-Firehose-Stream-AgiKThomas",
167+
"request_id": "bf51208a-d776-4161-b2ce-87ff64a6b095"
168+
},
169+
"metrics_names_fingerprint": "2rvRGJG912vF5iOdng4PjKe2+nc="
170+
},
171+
"cloud": {
172+
"account": {
173+
"id": "111111111111111111"
174+
},
175+
"provider": "aws",
176+
"region": "ap-south-1"
177+
},
178+
"data_stream": {
179+
"dataset": "aws_mq.activemq_metrics",
180+
"namespace": "default",
181+
"type": "metrics"
182+
},
183+
"ecs": {
184+
"version": "8.11.0"
185+
},
186+
"start_timestamp": "2025-03-28T09:12:00.000Z"
187+
}
188+
]
189+
}

‎packages/awsfirehose/data_stream/metrics/routing_rules.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,54 @@
115115
namespace:
116116
- "{{data_stream.namespace}}"
117117
- default
118+
- target_dataset: aws_mq.activemq_metrics
119+
if: ctx.aws?.cloudwatch?.namespace == "AWS/AmazonMQ" && (
120+
ctx.aws?.amazonmq?.metrics?.AmqpMaximumConnections != null ||
121+
ctx.aws?.amazonmq?.metrics?.MqttMaximumConnections != null ||
122+
ctx.aws?.amazonmq?.metrics?.OpenwireMaximumConnections != null ||
123+
ctx.aws?.amazonmq?.metrics?.StompMaximumConnections != null ||
124+
ctx.aws?.amazonmq?.metrics?.WsMaximumConnections != null ||
125+
ctx.aws?.amazonmq?.metrics?.CurrentConnectionsCount != null ||
126+
ctx.aws?.amazonmq?.metrics?.EstablishedConnectionsCount != null ||
127+
ctx.aws?.amazonmq?.metrics?.InactiveDurableTopicSubscribersCount != null ||
128+
ctx.aws?.amazonmq?.metrics?.JournalFilesForFastRecovery != null ||
129+
ctx.aws?.amazonmq?.metrics?.JournalFilesForFullRecovery != null ||
130+
ctx.aws?.amazonmq?.metrics?.NetworkConnectorConnectionCount != null ||
131+
ctx.aws?.amazonmq?.metrics?.NetworkIn != null ||
132+
ctx.aws?.amazonmq?.metrics?.NetworkOut != null ||
133+
ctx.aws?.amazonmq?.metrics?.OpenTransactionCount != null ||
134+
ctx.aws?.amazonmq?.metrics?.TotalConsumerCount != null ||
135+
ctx.aws?.amazonmq?.metrics?.TotalMessageCount != null ||
136+
ctx.aws?.amazonmq?.metrics?.TotalProducerCount != null ||
137+
ctx.aws?.amazonmq?.metrics?.VolumeReadOps != null ||
138+
ctx.aws?.amazonmq?.metrics?.VolumeWriteOps != null ||
139+
ctx.aws?.amazonmq?.metrics?.ReceiveCount != null ||
140+
ctx.aws?.amazonmq?.metrics?.ProducerCount != null ||
141+
ctx.aws?.amazonmq?.metrics?.QueueSize != null ||
142+
ctx.aws?.amazonmq?.metrics?.TotalEnqueueCount != null ||
143+
ctx.aws?.amazonmq?.metrics?.TotalDequeueCount != null)
144+
namespace:
145+
- "{{data_stream.namespace}}"
146+
- default
147+
- target_dataset: aws_mq.rabbitmq_metrics
148+
if: ctx.aws?.cloudwatch?.namespace == "AWS/AmazonMQ" && (
149+
ctx.aws?.amazonmq?.metrics?.ExchangeCount != null ||
150+
ctx.aws?.amazonmq?.metrics?.QueueCount != null ||
151+
ctx.aws?.amazonmq?.metrics?.ConnectionCount != null ||
152+
ctx.aws?.amazonmq?.metrics?.ChannelCount != null ||
153+
ctx.aws?.amazonmq?.metrics?.MessageCount != null ||
154+
ctx.aws?.amazonmq?.metrics?.MessageReadyCount != null ||
155+
ctx.aws?.amazonmq?.metrics?.MessageUnacknowledgedCount != null ||
156+
ctx.aws?.amazonmq?.metrics?.PublishRate != null ||
157+
ctx.aws?.amazonmq?.metrics?.ConfirmRate != null ||
158+
ctx.aws?.amazonmq?.metrics?.AckRate != null ||
159+
ctx.aws?.amazonmq?.metrics?.SystemCpuUtilization != null ||
160+
ctx.aws?.amazonmq?.metrics?.RabbitMQMemLimit != null ||
161+
ctx.aws?.amazonmq?.metrics?.RabbitMQMemUsed != null ||
162+
ctx.aws?.amazonmq?.metrics?.RabbitMQDiskFreeLimit != null ||
163+
ctx.aws?.amazonmq?.metrics?.RabbitMQFdUsed != null ||
164+
ctx.aws?.amazonmq?.metrics?.RabbitMQIOReadAverageTime != null ||
165+
ctx.aws?.amazonmq?.metrics?.RabbitMQIOWriteAverageTime != null)
166+
namespace:
167+
- "{{data_stream.namespace}}"
168+
- default

0 commit comments

Comments
 (0)