elastic · yug-rajani · May 3, 2022 · Mar 31, 2022 · Mar 31, 2022 · Mar 31, 2022
@@ -18,3 +18,10 @@ This data stream collects Application metrics.
 
 {{fields "application"}}
 
+## cluster
+
+This data stream collects Cluster metrics.
+
+{{event "cluster"}}
+
+{{fields "cluster"}}
@@ -1,6 +1,9 @@
 # newer versions go on top
 - version: "0.1.0"
  changes:
- - description: Add hadoop integration with application data stream.
+ - description: Add cluster metrics data stream for Hadoop.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/2953
+ - description: Add Hadoop integration with application data stream.
  type: enhancement
  link: https://github.com/elastic/integrations/pull/2952
@@ -2,7 +2,7 @@
  "expected": [
  {
  "ecs": {
- "version": "8.0.0"
+ "version": "8.1.0"
  },
  "event": {
  "category": "database",
@@ -35,7 +35,7 @@
  },
  {
  "ecs": {
- "version": "8.0.0"
+ "version": "8.1.0"
  },
  "event": {
  "category": "database",

@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Application metrics
 processors:
  - set:
  field: ecs.version
- value: '8.0.0'
+ value: '8.1.0'
  - set:
  field: event.type
  value: info

@@ -5,20 +5,20 @@
  "id": "9944acc9-e39f-40e0-a02a-7529cf504db1",
  "name": "docker-fleet-agent",
  "type": "filebeat",
- "version": "8.0.0"
+ "version": "8.1.0"
  },
  "data_stream": {
  "dataset": "hadoop.application",
  "namespace": "ep",
  "type": "logs"
  },
  "ecs": {
- "version": "8.0.0"
+ "version": "8.1.0"
  },
  "elastic_agent": {
  "id": "9944acc9-e39f-40e0-a02a-7529cf504db1",
  "snapshot": false,
- "version": "8.0.0"
+ "version": "8.1.0"
  },
  "event": {
  "agent_id_status": "verified",

@@ -0,0 +1,158 @@
+{
+ "events": [
+ {
+ "@timestamp": "2016-10-25T12:49:34.000Z",
+ "http": {
+ "cluster": {
+ "beans": [
+ {
+ "AMLaunchDelayAvgTime": 60,
+ "modelerType": "ClusterMetrics",
+ "NumShutdownNMs": 0,
+ "NumActiveNMs": 1,
+ "NumRebootedNMs": 0,
+ "AMRegisterDelayNumOps": 13,
+ "AMLaunchDelayNumOps": 17,
+ "CapabilityMB": 8192,
+ "NumDecommissioningNMs": 0,
+ "tag.ClusterMetrics": "ResourceManager",
+ "NumDecommissionedNMs": 0,
+ "UtilizedVirtualCores": 1,
+ "CapabilityVirtualCores": 8,
+ "UtilizedMB": 6723,
+ "AMContainerAllocationDelayNumOps": 17,
+ "RmEventProcCPUMax": 808,
+ "tag.Context": "yarn",
+ "NumUnhealthyNMs": 0,
+ "name": "Hadoop:service=ResourceManager,name=ClusterMetrics",
+ "RmEventProcCPUAvg": 113,
+ "AMRegisterDelayAvgTime": 22101,
+ "tag.Hostname": "localhost.localdomain",
+ "AMContainerAllocationDelayAvgTime": 909,
+ "NumLostNMs": 0
+ }
+ ]
+ }
+ }
+ },
+ {
+ "@timestamp": "2016-10-25T12:49:34.000Z",
+ "http": {
+ "cluster": {
+ "clusterMetrics": {
+ "containersPending": 0,
+ "appsCompleted": 13,
+ "rmSchedulerBusyPercent": 0,
+ "rebootedNodes": 0,
+ "containersAllocated": 0,
+ "appsSubmitted": 15,
+ "totalNodes": 1,
+ "shutdownNodes": 0,
+ "decommissionedNodes": 0,
+ "totalMB": 8192,
+ "availableMB": 8192,
+ "reservedVirtualCores": 0,
+ "allocatedMB": 0,
+ "totalReservedResourcesAcrossPartition": {
+ "memory": 0,
+ "resourceInformations": {
+ "resourceInformation": [
+ {
+ "maximumAllocation": 9223372036854776000,
+ "name": "memory-mb",
+ "attributes": {},
+ "units": "Mi",
+ "minimumAllocation": 0,
+ "value": 0,
+ "resourceType": "COUNTABLE"
+ },
+ {
+ "name": "vcores",
+ "maximumAllocation": 9223372036854776000,
+ "attributes": {},
+ "units": "",
+ "minimumAllocation": 0,
+ "value": 0,
+ "resourceType": "COUNTABLE"
+ }
+ ]
+ },
+ "vCores": 0
+ },
+ "decommissioningNodes": 0,
+ "reservedMB": 0,
+ "unhealthyNodes": 0,
+ "utilizedMBPercent": 86,
+ "totalAllocatedContainersAcrossPartition": 0,
+ "appsKilled": 0,
+ "appsRunning": 0,
+ "allocatedVirtualCores": 0,
+ "utilizedVirtualCoresPercent": 12,
+ "appsPending": 0,
+ "activeNodes": 1,
+ "crossPartitionMetricsAvailable": true,
+ "lostNodes": 0,
+ "totalVirtualCores": 8,
+ "containersReserved": 0,
+ "totalClusterResourcesAcrossPartition": {
+ "memory": 8192,
+ "resourceInformations": {
+ "resourceInformation": [
+ {
+ "name": "memory-mb",
+ "maximumAllocation": 9223372036854776000,
+ "attributes": {},
+ "minimumAllocation": 0,
+ "units": "Mi",
+ "value": 8192,
+ "resourceType": "COUNTABLE"
+ },
+ {
+ "maximumAllocation": 9223372036854776000,
+ "name": "vcores",
+ "attributes": {},
+ "units": "",
+ "minimumAllocation": 0,
+ "value": 8,
+ "resourceType": "COUNTABLE"
+ }
+ ]
+ },
+ "vCores": 8
+ },
+ "pendingMB": 0,
+ "availableVirtualCores": 8,
+ "pendingVirtualCores": 0,
+ "totalUsedResourcesAcrossPartition": {
+ "memory": 0,
+ "resourceInformations": {
+ "resourceInformation": [
+ {
+ "maximumAllocation": 9223372036854776000,
+ "name": "memory-mb",
+ "attributes": {},
+ "units": "Mi",
+ "minimumAllocation": 0,
+ "value": 0,
+ "resourceType": "COUNTABLE"
+ },
+ {
+ "name": "vcores",
+ "maximumAllocation": 9223372036854776000,
+ "attributes": {},
+ "minimumAllocation": 0,
+ "units": "",
+ "value": 0,
+ "resourceType": "COUNTABLE"
+ }
+ ]
+ },
+ "vCores": 0
+ },
+ "appsFailed": 2
+ }
+ }
+ }
+ }
+ ]
+}
@@ -0,0 +1,82 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2016-10-25T12:49:34.000Z",
+ "ecs": {
+ "version": "8.1.0"
+ },
+ "event": {
+ "category": "database",
+ "kind": "metric",
+ "type": "info"
+ },
+ "hadoop": {
+ "cluster": {
+ "application_main": {
+ "launch_delay_avg_time": 60,
+ "launch_delay_num_ops": 17,
+ "register_delay_avg_time": 22101,
+ "register_delay_num_ops": 13
+ },
+ "node_managers": {
+ "num_active": 1,
+ "num_decommissioned": 0,
+ "num_lost": 0,
+ "num_rebooted": 0,
+ "num_unhealthy": 0
+ }
+ }
+ }
+ },
+ {
+ "@timestamp": "2016-10-25T12:49:34.000Z",
+ "ecs": {
+ "version": "8.1.0"
+ },
+ "event": {
+ "category": "database",
+ "kind": "metric",
+ "type": "info"
+ },
+ "hadoop": {
+ "cluster": {
+ "applications": {
+ "completed": 13,
+ "failed": 2,
+ "killed": 0,
+ "pending": 0,
+ "running": 0,
+ "submitted": 15
+ },
+ "containers": {
+ "allocated": 0,
+ "pending": 0,
+ "reserved": 0
+ },
+ "memory": {
+ "allocated": 0,
+ "available": 8192,
+ "reserved": 0,
+ "total": 8192
+ },
+ "nodes": {
+ "active": 1,
+ "decommissioned": 0,
+ "decommissioning": 0,
+ "lost": 0,
+ "rebooted": 0,
+ "shutdown": 0,
+ "total": 1,
+ "unhealthy": 0
+ },
+ "virtual_cores": {
+ "allocated": 0,
+ "available": 8,
+ "reserved": 0,
+ "total": 8
+ }
+ }
+ }
+ }
+ ]
+}
@@ -0,0 +1,5 @@
+input: http/metrics
+data_stream:
+ vars:
+ hosts:
+ - http://{{Hostname}}:{{Ports.[0]}}
@@ -0,0 +1,23 @@
+metricsets: ["json"]
+namespace: "cluster"
+hosts:
+{{#each hosts}}
+ - "{{this}}/jmx?qry=Hadoop:service=ResourceManager,name=ClusterMetrics"
+ - "{{this}}/ws/v1/cluster/metrics"
+{{/each}}
+period: {{period}}
+path: "/"
+{{#if ssl}}
+{{ssl}}
+{{/if}}
+tags:
+{{#each tags as |tag i|}}
+ - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
@@ -0,0 +1,47 @@
+---
+description: Pipeline for parsing Hadoop Cluster metrics
+processors:
+ - set:
+ field: ecs.version
+ value: '8.1.0'
+ - set:
+ field: event.type
+ value: info
+ - set:
+ field: event.kind
+ value: metric
+ - set:
+ field: event.category
+ value: database
+ - rename:
+ field: http
+ target_field: hadoop
+ ignore_missing: true
+ ignore_failure: true
+ - pipeline:
+ if: ctx.hadoop.cluster.containsKey('beans')
+ name: '{{ IngestPipeline "pipeline-cluster" }}' 
+ - pipeline:
+ if: ctx.hadoop.cluster.containsKey('clusterMetrics')
+ name: '{{ IngestPipeline "pipeline-expanded-cluster" }}'
+ - script:
+ description: Drops null/empty values recursively
+ lang: painless
+ source: |
+ boolean drop(Object o) {
+ if (o == null || o == "") {
+ return true;
+ } else if (o instanceof Map) {
+ ((Map) o).values().removeIf(v -> drop(v));
+ return (((Map) o).size() == 0);
+ } else if (o instanceof List) {
+ ((List) o).removeIf(v -> drop(v));
+ return (((List) o).length == 0);
+ }
+ return false;
+ }
+ drop(ctx);
+on_failure:
+ - set:
+ field: error.message
+ value: "{{{_ingest.on_failure_message}}}"