Initial commit for Hadoop package

Author: yug-rajani

packages/hadoop/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+ ecs:
+ reference: git@8.0

packages/hadoop/_dev/build/docs/README.md

@@ -0,0 +1,37 @@
+# Hadoop
+
+The Hadoop integration collects and parses data from the Hadoop Events APIs and using the Jolokia Metricbeat Module.
+
+## Compatibility
+
+This module has been tested against `Hadoop version 3.3.1`
+
+## Requirements
+
+In order to ingest data from Hadoop, you must know the full hosts for the NameNode, DataNode, Cluster Metrics, Node Manager and the Hadoop Events API.
+
+## Metrics
+
+### Application Metrics
+
+This is the `application_metrics` dataset.
+
+{{event "application_metrics"}}
+
+{{fields "application_metrics"}}
+
+### Expanded Cluster Metrics
+
+This is the `expanded_cluster_metrics` dataset.
+
+{{event "expanded_cluster_metrics"}}
+
+{{fields "expanded_cluster_metrics"}}
+
+### Jolokia Metrics
+
+This is the `jolokia_metrics` dataset.
+
+{{event "jolokia_metrics"}}
+
+{{fields "jolokia_metrics"}}

packages/hadoop/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,100 @@
+FROM centos:latest
+ARG SERVICE_VERSION=${SERVICE_VERSION:-3.3.1}
+ARG HADOOP_VERSION=${SERVICE_VERSION} 
+
+ARG JAVA_VERSION=8
+ARG JAVA_RELEASE=JDK
+
+RUN set -eux && \
+ pkg="java-1.$JAVA_VERSION.0-openjdk" && \
+ if [ "$JAVA_RELEASE" = "JDK" ]; then \
+ pkg="$pkg-devel"; \
+ else \
+ pkg="$pkg-headless"; \
+ fi; \
+ yum install -y "$pkg" && \
+ yum clean all && \
+ rm -rf /var/cache/yum
+
+COPY profile.d/java.sh /etc/profile.d/
+
+ENV JAVA_HOME=/usr
+
+ARG TAR=hadoop-$HADOOP_VERSION.tar.gz
+
+ENV PATH $PATH:/hadoop/bin
+
+LABEL Description="Hadoop Dev" \
+ "Hadoop Version"="$HADOOP_VERSION"
+
+WORKDIR /
+
+RUN set -eux && \
+ yum install -y openssh-server openssh-clients tar which
+
+ENV JOLOKIA_VERSION=1.6.0 JOLOKIA_HOST=0.0.0.0 JOLOKIA_PORT=7778
+ENV HADOOP_LATEST_VERSION=${SERVICE_VERSION} 
+
+RUN set -eux && \
+ yum install -y wget hostname && \
+
+ wget "http://search.maven.org/remotecontent?filepath=org/jolokia/jolokia-jvm/${JOLOKIA_VERSION}/jolokia-jvm-${JOLOKIA_VERSION}-agent.jar" -O "jolokia-jvm-${JOLOKIA_VERSION}-agent.jar" && \
+ wget -t 10 --max-redirect 1 --retry-connrefused -O "$TAR" "http://www.apache.org/dyn/closer.lua?filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-$HADOOP_VERSION.tar.gz&action=download" || \
+ wget -t 10 --max-redirect 1 --retry-connrefused -O "$TAR" "http://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-$HADOOP_VERSION.tar.gz" && \
+ tar zxf "$TAR" && \
+
+ test -d "hadoop-$HADOOP_VERSION" && \
+ ln -sv "hadoop-$HADOOP_VERSION" hadoop && \
+ mkdir /etc/hadoop && \
+ ln -s /hadoop/etc/hadoop /etc/hadoop/conf && \
+ rm -fv "$TAR" && \
+ { rm -rf hadoop/share/doc; : ; } && \
+ yum autoremove -y && \
+ # gets autoremoved, ensure it's added back as Hadoop scripts need it
+ yum install -y hostname && \
+ yum clean all && \
+ rm -rf /var/cache/yum
+
+ENV JMX_REMOTE=yes JOLOKIA_ENABLED=yes
+
+COPY entrypoint.sh /
+COPY conf/core-site.xml /hadoop/etc/hadoop/
+COPY conf/hdfs-site.xml /hadoop/etc/hadoop/
+COPY conf/yarn-site.xml /hadoop/etc/hadoop/
+COPY conf/mapred-site.xml /hadoop/etc/hadoop/
+COPY profile.d/hadoop.sh /etc/profile.d/
+COPY ssh/config /root/.ssh/
+
+
+RUN set -eux && \
+ /hadoop/bin/hdfs namenode -format && \
+ groupadd hadoop && \
+ useradd -g hadoop hdfs && \
+ useradd -g hadoop yarn && \
+ mkdir -p /dfs/name && \
+ mkdir -p /dfs/data && \
+ mkdir -p /hadoop/logs && \
+ chown -R hdfs:hadoop /dfs/name && \
+ chown -R hdfs:hadoop /dfs/data && \
+ chgrp -R hadoop /hadoop/logs && \
+ chmod -R 0770 /hadoop/logs && \
+ mkdir -p /root/.ssh \
+ /home/hdfs/.ssh \
+ /home/yarn/.ssh && \
+ chown hdfs /home/hdfs/.ssh && \
+ chown yarn /home/yarn/.ssh && \
+ chmod 0700 /root/.ssh \
+ /home/hdfs/.ssh \
+ /home/yarn/.ssh
+
+ENV HDFS_NAMENODE_USER=hdfs
+ENV HDFS_SECONDARYNAMENODE_USER=hdfs
+ENV HDFS_DATANODE_USER=hdfs
+ENV YARN_RESOURCEMANAGER_USER=yarn
+ENV YARN_NODEMANAGER_USER=yarn
+
+
+
+RUN rm -rf /run/nologin
+RUN chmod -R 600 /etc/crypto-policies/
+CMD ["bash","/entrypoint.sh"]

packages/hadoop/_dev/deploy/docker/conf/core-site.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://localhost:8020</value>
+ </property>
+</configuration>

packages/hadoop/_dev/deploy/docker/conf/hdfs-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/dfs/name</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/dfs/data</value>
+ </property>
+ <property>
+ <name>dfs.permissions</name>
+ <value>false</value>
+ </property>
+</configuration>

packages/hadoop/_dev/deploy/docker/conf/mapred-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.env</name>
+ <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
+</property>
+<property>
+ <name>mapreduce.map.env</name>
+ <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
+</property>
+<property>
+ <name>mapreduce.reduce.env</name>
+ <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
+</property>
+</configuration>

packages/hadoop/_dev/deploy/docker/conf/yarn-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>1</value>
+ </property>
+ 
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>3072</value>
+ </property>
+</configuration>

packages/hadoop/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '3'
+services:
+ hadoop:
+ build: .
+ ports:
+ - "7777"
+ - "7779"
+ - "8088"

packages/hadoop/_dev/deploy/docker/entrypoint.sh

@@ -0,0 +1,88 @@
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+
+export JAVA_HOME="${JAVA_HOME:-/usr}"
+
+export PATH="$PATH:/hadoop/sbin:/hadoop/bin"
+
+if [ $# -gt 0 ]; then
+ exec "$@"
+else
+ for x in root hdfs yarn; do
+ if ! [ -f "$x/.ssh/id_rsa" ]; then
+ su - "$x" <<-EOF
+ [ -n "${DEBUG:-}" ] && set -x
+ ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ""
+EOF
+ fi
+ if ! [ -f "$x/.ssh/authorized_keys" ]; then
+ su - "$x" <<-EOF
+ [ -n "${DEBUG:-}" ] && set -x
+ cp -v ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
+ chmod -v 0400 ~/.ssh/authorized_keys
+EOF
+ fi
+ done
+
+ # removed in newer versions of CentOS
+ if ! [ -f /etc/ssh/ssh_host_rsa_key ] && [ -x /usr/sbin/sshd-keygen ]; then
+ /usr/sbin/sshd-keygen || :
+ fi
+ if ! [ -f /etc/ssh/ssh_host_rsa_key ]; then
+ ssh-keygen -q -t rsa -f /etc/ssh/ssh_host_rsa_key -C '' -N ''
+ chmod 0600 /etc/ssh/ssh_host_rsa_key
+ chmod 0644 /etc/ssh/ssh_host_rsa_key.pub
+ fi
+
+ if ! pgrep -x sshd &>/dev/null; then
+ /usr/sbin/sshd
+ fi
+ echo
+ SECONDS=0
+ while true; do
+ if ssh-keyscan localhost 2>&1 | grep -q OpenSSH; then
+ echo "SSH is ready to rock"
+ break
+ fi
+ if [ "$SECONDS" -gt 20 ]; then
+ echo "FAILED: SSH failed to come up after 20 secs"
+ exit 1
+ fi
+ echo "waiting for SSH to come up"
+ sleep 1
+ done
+ echo
+ if ! [ -f /root/.ssh/known_hosts ]; then
+ ssh-keyscan localhost || :
+ ssh-keyscan 0.0.0.0 || :
+ fi | tee -a /root/.ssh/known_hosts
+ hostname="$(hostname -f)"
+ if ! grep -q "$hostname" /root/.ssh/known_hosts; then
+ ssh-keyscan "$hostname" || :
+ fi | tee -a /root/.ssh/known_hosts
+
+ mkdir -pv /hadoop/logs
+
+
+if [ "$JOLOKIA_ENABLED" = 'yes' ]; then
+
+echo "export HDFS_NAMENODE_OPTS="-javaagent\:`echo /jolokia-jvm-${JOLOKIA_VERSION}-agent.jar=host=${JOLOKIA_HOST},port=7777`"" >> "/hadoop-${HADOOP_LATEST_VERSION}/etc/hadoop/hadoop-env.sh"
+ echo "export HDFS_DATANODE_OPTS="-javaagent\:`echo /jolokia-jvm-${JOLOKIA_VERSION}-agent.jar=host=${JOLOKIA_HOST},port=7779`"" >> "/hadoop-${HADOOP_LATEST_VERSION}/etc/hadoop/hadoop-env.sh"
+ echo "export YARN_NODEMANAGER_OPTS="-javaagent\:`echo /jolokia-jvm-${JOLOKIA_VERSION}-agent.jar=host=${JOLOKIA_HOST},port=7782`"" >> "/${HADOOP_LATEST_VERSION}/etc/hadoop/hadoop-env.sh"
+ echo "export YARN_RESOURCEMANAGER_OPTS="-javaagent\:`echo /jolokia-jvm-${JOLOKIA_VERSION}-agent.jar=host=${JOLOKIA_HOST},port=7781`"" >> "/${HADOOP_LATEST_VERSION}/etc/hadoop/hadoop-env.sh"
+ 
+fi
+
+ sed -i "s/localhost/$hostname/" /hadoop/etc/hadoop/core-site.xml
+ echo 'Y' | hdfs namenode -format
+ start-dfs.sh
+ start-yarn.sh
+
+ hdfs dfs -mkdir -p /user/root/input
+ hdfs dfs -put /hadoop-${HADOOP_LATEST_VERSION}/LICENSE.txt /user/root/input/
+ hadoop jar /hadoop-${HADOOP_LATEST_VERSION}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_LATEST_VERSION}.jar wordcount input output
+ 
+ tail -f /dev/null /hadoop/logs/*
+ stop-yarn.sh
+ stop-dfs.sh
+fi

packages/hadoop/_dev/deploy/docker/profile.d/hadoop.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+export HADOOP_HOME=/hadoop