Skip to content

Commit 3a8bf98

Browse files
yug-rajaniyug-rajani
authored
[hadoop][cluster_metrics] Add cluster_metrics data stream for hadoop (#2953)
* Add cluster_metrics data stream for hadoop * Update changelog * Remove irrelevant data streams * Make changes as per review comments * Update SSL configuration * Update manifest.yml * Update CODEOWNERS entry and manifest.yml * Make requested changes * Update SSL configuration * Update Kibana version * Resolve nitpick in manifest.yml
1 parent 7022539 commit 3a8bf98

File tree

19 files changed

+970
-12
lines changed

19 files changed

+970
-12
lines changed

‎packages/hadoop/_dev/build/docs/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,10 @@ This data stream collects Application metrics.
1818

1919
{{fields "application"}}
2020

21+
## cluster
22+
23+
This data stream collects Cluster metrics.
24+
25+
{{event "cluster"}}
26+
27+
{{fields "cluster"}}

‎packages/hadoop/changelog.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# newer versions go on top
22
- version: "0.1.0"
33
changes:
4-
- description: Add hadoop integration with application data stream.
4+
- description: Add cluster metrics data stream for Hadoop.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/2953
7+
- description: Add Hadoop integration with application data stream.
58
type: enhancement
69
link: https://github.com/elastic/integrations/pull/2952

‎packages/hadoop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"expected": [
33
{
44
"ecs": {
5-
"version": "8.0.0"
5+
"version": "8.1.0"
66
},
77
"event": {
88
"category": "database",
@@ -35,7 +35,7 @@
3535
},
3636
{
3737
"ecs": {
38-
"version": "8.0.0"
38+
"version": "8.1.0"
3939
},
4040
"event": {
4141
"category": "database",

‎packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Application metrics
33
processors:
44
- set:
55
field: ecs.version
6-
value: '8.0.0'
6+
value: '8.1.0'
77
- set:
88
field: event.type
99
value: info

‎packages/hadoop/data_stream/application/sample_event.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,20 @@
55
"id": "9944acc9-e39f-40e0-a02a-7529cf504db1",
66
"name": "docker-fleet-agent",
77
"type": "filebeat",
8-
"version": "8.0.0"
8+
"version": "8.1.0"
99
},
1010
"data_stream": {
1111
"dataset": "hadoop.application",
1212
"namespace": "ep",
1313
"type": "logs"
1414
},
1515
"ecs": {
16-
"version": "8.0.0"
16+
"version": "8.1.0"
1717
},
1818
"elastic_agent": {
1919
"id": "9944acc9-e39f-40e0-a02a-7529cf504db1",
2020
"snapshot": false,
21-
"version": "8.0.0"
21+
"version": "8.1.0"
2222
},
2323
"event": {
2424
"agent_id_status": "verified",

‎packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
{
2+
"events": [
3+
{
4+
"@timestamp": "2016-10-25T12:49:34.000Z",
5+
"http": {
6+
"cluster": {
7+
"beans": [
8+
{
9+
"AMLaunchDelayAvgTime": 60,
10+
"modelerType": "ClusterMetrics",
11+
"NumShutdownNMs": 0,
12+
"NumActiveNMs": 1,
13+
"NumRebootedNMs": 0,
14+
"AMRegisterDelayNumOps": 13,
15+
"AMLaunchDelayNumOps": 17,
16+
"CapabilityMB": 8192,
17+
"NumDecommissioningNMs": 0,
18+
"tag.ClusterMetrics": "ResourceManager",
19+
"NumDecommissionedNMs": 0,
20+
"UtilizedVirtualCores": 1,
21+
"CapabilityVirtualCores": 8,
22+
"UtilizedMB": 6723,
23+
"AMContainerAllocationDelayNumOps": 17,
24+
"RmEventProcCPUMax": 808,
25+
"tag.Context": "yarn",
26+
"NumUnhealthyNMs": 0,
27+
"name": "Hadoop:service=ResourceManager,name=ClusterMetrics",
28+
"RmEventProcCPUAvg": 113,
29+
"AMRegisterDelayAvgTime": 22101,
30+
"tag.Hostname": "localhost.localdomain",
31+
"AMContainerAllocationDelayAvgTime": 909,
32+
"NumLostNMs": 0
33+
}
34+
]
35+
}
36+
}
37+
},
38+
{
39+
"@timestamp": "2016-10-25T12:49:34.000Z",
40+
"http": {
41+
"cluster": {
42+
"clusterMetrics": {
43+
"containersPending": 0,
44+
"appsCompleted": 13,
45+
"rmSchedulerBusyPercent": 0,
46+
"rebootedNodes": 0,
47+
"containersAllocated": 0,
48+
"appsSubmitted": 15,
49+
"totalNodes": 1,
50+
"shutdownNodes": 0,
51+
"decommissionedNodes": 0,
52+
"totalMB": 8192,
53+
"availableMB": 8192,
54+
"reservedVirtualCores": 0,
55+
"allocatedMB": 0,
56+
"totalReservedResourcesAcrossPartition": {
57+
"memory": 0,
58+
"resourceInformations": {
59+
"resourceInformation": [
60+
{
61+
"maximumAllocation": 9223372036854776000,
62+
"name": "memory-mb",
63+
"attributes": {},
64+
"units": "Mi",
65+
"minimumAllocation": 0,
66+
"value": 0,
67+
"resourceType": "COUNTABLE"
68+
},
69+
{
70+
"name": "vcores",
71+
"maximumAllocation": 9223372036854776000,
72+
"attributes": {},
73+
"units": "",
74+
"minimumAllocation": 0,
75+
"value": 0,
76+
"resourceType": "COUNTABLE"
77+
}
78+
]
79+
},
80+
"vCores": 0
81+
},
82+
"decommissioningNodes": 0,
83+
"reservedMB": 0,
84+
"unhealthyNodes": 0,
85+
"utilizedMBPercent": 86,
86+
"totalAllocatedContainersAcrossPartition": 0,
87+
"appsKilled": 0,
88+
"appsRunning": 0,
89+
"allocatedVirtualCores": 0,
90+
"utilizedVirtualCoresPercent": 12,
91+
"appsPending": 0,
92+
"activeNodes": 1,
93+
"crossPartitionMetricsAvailable": true,
94+
"lostNodes": 0,
95+
"totalVirtualCores": 8,
96+
"containersReserved": 0,
97+
"totalClusterResourcesAcrossPartition": {
98+
"memory": 8192,
99+
"resourceInformations": {
100+
"resourceInformation": [
101+
{
102+
"name": "memory-mb",
103+
"maximumAllocation": 9223372036854776000,
104+
"attributes": {},
105+
"minimumAllocation": 0,
106+
"units": "Mi",
107+
"value": 8192,
108+
"resourceType": "COUNTABLE"
109+
},
110+
{
111+
"maximumAllocation": 9223372036854776000,
112+
"name": "vcores",
113+
"attributes": {},
114+
"units": "",
115+
"minimumAllocation": 0,
116+
"value": 8,
117+
"resourceType": "COUNTABLE"
118+
}
119+
]
120+
},
121+
"vCores": 8
122+
},
123+
"pendingMB": 0,
124+
"availableVirtualCores": 8,
125+
"pendingVirtualCores": 0,
126+
"totalUsedResourcesAcrossPartition": {
127+
"memory": 0,
128+
"resourceInformations": {
129+
"resourceInformation": [
130+
{
131+
"maximumAllocation": 9223372036854776000,
132+
"name": "memory-mb",
133+
"attributes": {},
134+
"units": "Mi",
135+
"minimumAllocation": 0,
136+
"value": 0,
137+
"resourceType": "COUNTABLE"
138+
},
139+
{
140+
"name": "vcores",
141+
"maximumAllocation": 9223372036854776000,
142+
"attributes": {},
143+
"minimumAllocation": 0,
144+
"units": "",
145+
"value": 0,
146+
"resourceType": "COUNTABLE"
147+
}
148+
]
149+
},
150+
"vCores": 0
151+
},
152+
"appsFailed": 2
153+
}
154+
}
155+
}
156+
}
157+
]
158+
}

‎packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{
2+
"expected": [
3+
{
4+
"@timestamp": "2016-10-25T12:49:34.000Z",
5+
"ecs": {
6+
"version": "8.1.0"
7+
},
8+
"event": {
9+
"category": "database",
10+
"kind": "metric",
11+
"type": "info"
12+
},
13+
"hadoop": {
14+
"cluster": {
15+
"application_main": {
16+
"launch_delay_avg_time": 60,
17+
"launch_delay_num_ops": 17,
18+
"register_delay_avg_time": 22101,
19+
"register_delay_num_ops": 13
20+
},
21+
"node_managers": {
22+
"num_active": 1,
23+
"num_decommissioned": 0,
24+
"num_lost": 0,
25+
"num_rebooted": 0,
26+
"num_unhealthy": 0
27+
}
28+
}
29+
}
30+
},
31+
{
32+
"@timestamp": "2016-10-25T12:49:34.000Z",
33+
"ecs": {
34+
"version": "8.1.0"
35+
},
36+
"event": {
37+
"category": "database",
38+
"kind": "metric",
39+
"type": "info"
40+
},
41+
"hadoop": {
42+
"cluster": {
43+
"applications": {
44+
"completed": 13,
45+
"failed": 2,
46+
"killed": 0,
47+
"pending": 0,
48+
"running": 0,
49+
"submitted": 15
50+
},
51+
"containers": {
52+
"allocated": 0,
53+
"pending": 0,
54+
"reserved": 0
55+
},
56+
"memory": {
57+
"allocated": 0,
58+
"available": 8192,
59+
"reserved": 0,
60+
"total": 8192
61+
},
62+
"nodes": {
63+
"active": 1,
64+
"decommissioned": 0,
65+
"decommissioning": 0,
66+
"lost": 0,
67+
"rebooted": 0,
68+
"shutdown": 0,
69+
"total": 1,
70+
"unhealthy": 0
71+
},
72+
"virtual_cores": {
73+
"allocated": 0,
74+
"available": 8,
75+
"reserved": 0,
76+
"total": 8
77+
}
78+
}
79+
}
80+
}
81+
]
82+
}

‎packages/hadoop/data_stream/cluster/_dev/test/system/test-cluster-metrics-config.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
input: http/metrics
2+
data_stream:
3+
vars:
4+
hosts:
5+
- http://{{Hostname}}:{{Ports.[0]}}

‎packages/hadoop/data_stream/cluster/agent/stream/stream.yml.hbs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
metricsets: ["json"]
2+
namespace: "cluster"
3+
hosts:
4+
{{#each hosts}}
5+
- "{{this}}/jmx?qry=Hadoop:service=ResourceManager,name=ClusterMetrics"
6+
- "{{this}}/ws/v1/cluster/metrics"
7+
{{/each}}
8+
period: {{period}}
9+
path: "/"
10+
{{#if ssl}}
11+
{{ssl}}
12+
{{/if}}
13+
tags:
14+
{{#each tags as |tag i|}}
15+
- {{tag}}
16+
{{/each}}
17+
{{#contains "forwarded" tags}}
18+
publisher_pipeline.disable_host: true
19+
{{/contains}}
20+
{{#if processors}}
21+
processors:
22+
{{processors}}
23+
{{/if}}

‎packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
---
2+
description: Pipeline for parsing Hadoop Cluster metrics
3+
processors:
4+
- set:
5+
field: ecs.version
6+
value: '8.1.0'
7+
- set:
8+
field: event.type
9+
value: info
10+
- set:
11+
field: event.kind
12+
value: metric
13+
- set:
14+
field: event.category
15+
value: database
16+
- rename:
17+
field: http
18+
target_field: hadoop
19+
ignore_missing: true
20+
ignore_failure: true
21+
- pipeline:
22+
if: ctx.hadoop.cluster.containsKey('beans')
23+
name: '{{ IngestPipeline "pipeline-cluster" }}'
24+
- pipeline:
25+
if: ctx.hadoop.cluster.containsKey('clusterMetrics')
26+
name: '{{ IngestPipeline "pipeline-expanded-cluster" }}'
27+
- script:
28+
description: Drops null/empty values recursively
29+
lang: painless
30+
source: |
31+
boolean drop(Object o) {
32+
if (o == null || o == "") {
33+
return true;
34+
} else if (o instanceof Map) {
35+
((Map) o).values().removeIf(v -> drop(v));
36+
return (((Map) o).size() == 0);
37+
} else if (o instanceof List) {
38+
((List) o).removeIf(v -> drop(v));
39+
return (((List) o).length == 0);
40+
}
41+
return false;
42+
}
43+
drop(ctx);
44+
on_failure:
45+
- set:
46+
field: error.message
47+
value: "{{{_ingest.on_failure_message}}}"

0 commit comments

Comments
 (0)