In preparation for moving to Elastic's CI (#862)

* In preperation for moving to Elastic's CI Add dockerfile to build image for testing python client in a docker container add docker-compose to create environment for running tests against specific versions of ES Add Makefile to make it easy to execute tests and cleanup add wait-for-elasticsearch.sh script to wait for elasticsearch before executing the tests. * add pull and comment about pushing requring auth * spelling errors * adding CI directory for jenkins to pick up fixing typo too * fix test-matrix and create working run-tests * add comment about why skipping python 2.6 * adding dockerfiles to create base images for ci

Author: fxdgear

.ci/Dockerfile-py

@@ -0,0 +1,2 @@
+ARG PYTHON_VERSION=3
+FROM python:${PYTHON_VERSION}

.ci/Dockerfile-py26

@@ -0,0 +1,10 @@
+FROM ubuntu:18.04
+RUN apt-get update && apt-get install -y curl build-essential zlib1g-dev libssl1.0-dev
+RUN ln -s /lib/i386-linux-gnu/libz.so.1 /lib/libz.so
+RUN curl -s https://www.python.org/ftp/python/2.6.9/Python-2.6.9.tgz | tar -xzf -
+WORKDIR Python-2.6.9
+RUN LDFLAGS="-L/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)" ./configure
+RUN make
+RUN make install
+RUN curl https://bootstrap.pypa.io/2.6/get-pip.py -o get-pip.py
+RUN python get-pip.py

.ci/run-tests

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#
+# Runs the client tests via Docker with the expectation that the required
+# environment variables have already been exported before running this script.
+#
+# The required environment variables include:
+#
+# - $ELASTICSEARCH_VERSION
+# - $PYTHON_VERSION
+#
+
+# TODO: implement the docker-based testing
+cd ..
+make run_tests

.ci/test-matrix.yml

@@ -0,0 +1,18 @@
+---
+ELASTICSEARCH_VERSION:
+ - 6.4.2
+ - 6.3.2
+ - 6.1.4
+ - 6.0.1
+
+
+
+PYTHON_VERSION:
+ - 3
+ - 2.7
+ - 2.6
+
+exclude:
+ # since a base image for python2.6 doens't exist i need to create one.
+ - PYTHON_VERSION: 2.6
+

Dockerfile

@@ -0,0 +1,12 @@
+ARG PYTHON_VERSION=3
+FROM python:${PYTHON_VERSION}
+RUN apt-get update && apt-get install -y git curl
+RUN pip install ipdb python-dateutil GitPython
+
+RUN git clone https://github.com/elastic/elasticsearch.git /code/elasticsearch
+
+WORKDIR /code/elasticsearch-py
+COPY . .
+RUN pip install .[develop]
+RUN python setup.py develop
+CMD ["/code/wait-for-elasticsearch.sh", "http://elasticsearch:9200", "--", "python", "setup.py", "test"]

Makefile

@@ -0,0 +1,18 @@
+clean:
+docker-compose down --remove-orphans --volumes
+
+build:
+PYTHON_VERSION=${PYTHON_VERSION} docker-compose build client
+
+pull:
+ELASTIC_VERSION=${ELASTIC_VERSION} PYTHON_VERSION=${PYTHON_VERSION} docker-compose pull
+
+push:
+# requires authentication.
+PYTHON_VERSION=${PYTHON_VERSION} docker-compose push client
+
+run_tests:
+ELASTIC_VERSION=${ELASTIC_VERSION} PYTHON_VERSION=${PYTHON_VERSION} docker-compose run client python setup.py test
+
+start_elasticsearch:
+ELASTIC_VERSION=${ELASTIC_VERSION} docker-compose up -d elasticsearch

docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.3'
+services:
+ client:
+ image: docker.elastic.co/clients/elasticsearch-py:${PYTHON_VERSION:-3}
+ build:
+ context: .
+ dockerfile: Dockerfile
+ args:
+ PYTHON_VERSION: ${PYTHON_VERSION:-3}
+ environment:
+ - "TEST_ES_SERVER=http://elasticsearch:9200"
+ volumes:
+ - .:/code/elasticsearch-py
+ - esvol:/tmp
+ networks:
+ - esnet
+ depends_on:
+ - elasticsearch
+ command: ["true"] # dummy command to override the command in the Dockerfile and do nothing.
+ elasticsearch:
+ image: docker.elastic.co/elasticsearch/elasticsearch-oss:${ELASTIC_VERSION:-6.2.4}
+ volumes:
+ - esvol:/tmp
+ networks:
+ - esnet
+ environment:
+ - path.repo=/tmp
+ - "repositories.url.allowed_urls=http://*"
+ - node.attr.testattr=test
+ - bootstrap.memory_lock=false
+ - "discovery.zen.ping.unicast.hosts=elasticsearch"
+ - "http.max_content_length=5mb"
+networks:
+ esnet:
+volumes:
+ esvol:

wait-for-elasticsearch.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+
+until $(curl --output /dev/null --silent --head --fail "$host"); do
+ printf '.'
+ sleep 1
+done
+
+# First wait for ES to start...
+response=$(curl $host)
+
+until [ "$response" = "200" ]; do
+ response=$(curl --write-out %{http_code} --silent --output /dev/null "$host")
+ >&2 echo "Elasticsearch is unavailable - sleeping"
+ sleep 1
+done
+
+
+# next wait for ES status to turn to Green
+health="$(curl -fsSL "$host/_cat/health?h=status")"
+health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ")
+
+until [ "$health" = 'green' ]; do
+ health="$(curl -fsSL "$host/_cat/health?h=status")"
+ health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ")
+ >&2 echo "Elasticsearch is unavailable - sleeping"
+ sleep 1
+done
+
+>&2 echo "Elasticsearch is up"
+exec $cmd