Add ES cleanup module

.gitignore

@@ -1,13 +1,18 @@
-# Compiled files
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
 *.tfstate
-*.tfstate.backup
-*.zip
+*.tfstate.*
+
+# .tfvars files
+*.tfvars
 
-# Module directory
-.terraform
-.idea
-*.iml
+**/.idea
+**/*.iml
 
-.build-harness
-build-harness
+**/.build-harness
+**/build-harness
 **/venv
+
+*.zip

.travis.yml

@@ -5,6 +5,10 @@ addons:
  - make
  - curl
 
+env:
+ global:
+ - TERRAFORM_MODULE_NAME="$(basename $TRAVIS_REPO_SLUG)"
+
 install:
  - make init
 
@@ -14,3 +18,17 @@ script:
  - make terraform/get-modules
  - make terraform/lint
  - make terraform/validate
+ - make build
+
+deploy:
+ # Deploy artifacts to S3
+ - provider: s3
+ region: "us-west-2"
+ access_key_id: $AWS_ACCESS_KEY_ID
+ secret_access_key: $AWS_SECRET_ACCESS_KEY
+ bucket: artifacts.prod.cloudposse.org
+ skip_cleanup: true
+ upload-dir: $TERRAFORM_MODULE_NAME/$TRAVIS_COMMIT
+ local-dir: artifacts
+ on:
+ all_branches: true

Makefile

@@ -1,4 +1,6 @@
-SHELL := /bin/bash
+SHELL := /bin/bash
+LAMBDA_DIR := lambda
+DEPS_CONTAINER := alpine:3.8
 
 # List of targets the `readme` target should call before generating the readme
 export README_DEPS ?= docs/targets.md docs/terraform.md
@@ -7,4 +9,23 @@ export README_DEPS ?= docs/targets.md docs/terraform.md
 
 ## Lint terraform code
 lint:
-$(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate
+$(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate
+
+define docker
+docker run -it -v $(PWD)/$(LAMBDA_DIR)/:/code -w /code $(DEPS_CONTAINER) /bin/sh -c '$(1)'
+endef
+
+## Install dependencies
+dependencies:
+@echo "==> Installing Lambda function dependencies..."
+@$(call docker, apk add --update py-pip && \
+ pip install virtualenv && \
+ virtualenv venv --always-copy && \
+ source ./venv/bin/activate && \
+ ./venv/bin/pip install -qUr requirements.txt)
+
+## Build Lambda function zip
+build: dependencies
+@echo "==> Building Lambda function zip..."
+@mkdir -p artifacts
+@cd $(LAMBDA_DIR) && zip -r ../artifacts/lambda.zip * && cd ../

README.yaml

@@ -0,0 +1,72 @@
+---
+#
+# This is the canonical configuration for the `README.md`
+# Run `make readme` to rebuild the `README.md`
+#
+
+# Name of this project
+name: terraform-aws-lambda-elasticsearch-cleanup
+
+# Logo for this project
+#logo: docs/logo.png
+
+# License of this project
+license: "APACHE2"
+
+# Canonical GitHub repo
+github_repo: cloudposse/terraform-aws-lambda-elasticsearch-cleanup
+
+# Badges to display
+badges:
+ - name: "Build Status"
+ image: "https://travis-ci.org/cloudposse/terraform-aws-lambda-elasticsearch-cleanup.svg?branch=master"
+ url: "https://travis-ci.org/cloudposse/terraform-aws-lambda-elasticsearch-cleanup"
+ - name: "Latest Release"
+ image: "https://img.shields.io/github/release/cloudposse/terraform-aws-lambda-elasticsearch-cleanup.svg"
+ url: "https://github.com/cloudposse/terraform-aws-lambda-elasticsearch-cleanup/releases/latest"
+ - name: "Slack Community"
+ image: "https://slack.cloudposse.com/badge.svg"
+ url: "https://slack.cloudposse.com"
+
+related:
+ - name: "terraform-aws-vpc"
+ description: "Terraform Module that defines a VPC with public/private subnets across multiple AZs with Internet Gateways"
+ url: "https://github.com/cloudposse/terraform-aws-vpc"
+ - name: "terraform-aws-dynamic-subnets"
+ description: "Terraform module for dynamic subnets provisioning."
+ url: "https://github.com/cloudposse/terraform-aws-dynamic-subnets"
+ - name: "terraform-aws-elasticsearch"
+ description: "Terraform module for AWS Elasticsearch provisioning."
+ url: "https://github.com/cloudposse/terraform-aws-elasticsearch"
+
+# Short description of this project
+description: |-
+ Terraform module to provision a scheduled Lambda function which will 
+ delete old Elasticsearch indexes using [SigV4Auth](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) authentication. The
+ lambda function can optionally send output to an SNS topic if the 
+ topic ARN is given. This module was largely inspired by 
+ [aws-lambda-es-cleanup](https://github.com/cloudreach/aws-lambda-es-cleanup)
+
+# How to use this project
+usage: |-
+ ```hcl
+ module "elasticsearch_cleanup" {
+ source = "../"
+ es_endpoint = "${module.elasticsearch.domain_endpoint}"
+ es_domain_arn = "${module.elasticsearch.domain_arn}"
+ es_security_group_id = "${module.elasticsearch.security_group_id}"
+ vpc_id = "${module.vpc.vpc_id}"
+ namespace = "example"
+ stage = "dev"
+ schedule = "rate(5 minutes)"
+ }
+ ```
+
+include:
+ - "docs/targets.md"
+ - "docs/terraform.md"
+
+# Contributors to this project
+contributors:
+ - name: "Josh Myers"
+ github: "joshmyers"

docs/targets.md

@@ -0,0 +1,12 @@
+## Makefile Targets
+```
+Available targets:
+
+ build Build Lambda function zip
+ dependencies Install dependencies
+ help Help screen
+ help/all Display help for all targets
+ help/short This help short screen
+ lint Lint terraform code
+
+```

docs/terraform.md

@@ -0,0 +1,36 @@
+## Module: cloudposse/terraform-aws-lambda-elasticsearch-cleanup
+
+This module creates a scheduled Lambda function which will delete old
+Elasticsearch indexes using SigV4Auth authentication. The lambda
+function can optionally send output to an SNS topic if the topic ARN
+is given
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| attributes | Additional attributes (e.g. `1`) | list | `<list>` | no |
+| delete_after | Number of days to preserve | string | `15` | no |
+| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no |
+| es_domain_arn | The Elasticsearch domain ARN | string | - | yes |
+| es_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | string | - | yes |
+| es_security_group_id | The Elasticsearch cluster security group ID | string | - | yes |
+| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` | string | `all` | no |
+| index_format | Combined with 'index' variable and is used to evaluate the index age | string | `%Y.%m.%d` | no |
+| name | Solution name, e.g. 'app' or 'cluster' | string | `app` | no |
+| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | string | - | yes |
+| python_version | The Python version to use | string | `2.7` | no |
+| schedule | CloudWatch Events rule schedule using cron or rate expression | string | `cron(0 3 * * ? *)` | no |
+| sns_arn | SNS ARN to pusblish alerts | string | `` | no |
+| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | string | - | yes |
+| subnet_ids | Subnet ids | list | - | yes |
+| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map | `<map>` | no |
+| timeout | Timeout for Lambda function in seconds | string | `300` | no |
+| vpc_id | The VPC ID for the Lambda function | string | - | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| security_group_id | Security Group ID of the Lambda |
+

example/main.tf

@@ -0,0 +1,66 @@
+module "label" {
+ source = "git::https://github.com/cloudposse/terraform-terraform-label.git?ref=tags/0.2.1"
+ namespace = "${var.namespace}"
+ name = "${var.name}"
+ stage = "${var.stage}"
+ delimiter = "${var.delimiter}"
+ attributes = "${compact(concat(var.attributes, list("elasticsearch", "cleanup")))}"
+ tags = "${var.tags}"
+ enabled = "true"
+}
+
+module "vpc" {
+ source = "git::https://github.com/cloudposse/terraform-aws-vpc.git?ref=master"
+ name = "${var.name}"
+ namespace = "${var.namespace}"
+ stage = "${var.stage}"
+ tags = "${module.label.tags}"
+}
+
+module "subnets" {
+ source = "git::https://github.com/cloudposse/terraform-aws-dynamic-subnets.git?ref=master"
+ name = "${var.name}"
+ namespace = "${var.namespace}"
+ stage = "${var.stage}"
+ region = "us-west-2"
+ vpc_id = "${module.vpc.vpc_id}"
+ igw_id = "${module.vpc.igw_id}"
+ cidr_block = "10.0.0.0/16"
+ availability_zones = ["us-west-2a", "us-west-2b"]
+ tags = "${module.label.tags}"
+}
+
+module "elasticsearch" {
+ source = "git::https://github.com/cloudposse/terraform-aws-elasticsearch.git?ref=master"
+ name = "${var.name}"
+ namespace = "${var.namespace}"
+ stage = "${var.stage}"
+ dns_zone_id = "Z3SO0TKDDQ0RGG"
+ security_groups = []
+ vpc_id = "${module.vpc.vpc_id}"
+ subnet_ids = ["${module.subnets.public_subnet_ids}"]
+ zone_awareness_enabled = "true"
+ elasticsearch_version = "6.3"
+ instance_type = "t2.small.elasticsearch"
+ instance_count = 4
+ kibana_subdomain_name = "kibana-es"
+ encrypt_at_rest_enabled = "false"
+ ebs_volume_size = 10
+ iam_actions = ["es:*"]
+ iam_role_arns = ["*"]
+ create_iam_service_linked_role = "false"
+ tags = "${module.label.tags}"
+}
+
+module "elasticsearch_cleanup" {
+ source = "../"
+ es_endpoint = "${module.elasticsearch.domain_endpoint}"
+ es_domain_arn = "${module.elasticsearch.domain_arn}"
+ es_security_group_id = "${module.elasticsearch.security_group_id}"
+ subnet_ids = ["${module.subnets.public_subnet_ids}"]
+ vpc_id = "${module.vpc.vpc_id}"
+ namespace = "${var.namespace}"
+ stage = "${var.stage}"
+ schedule = "${var.schedule}"
+ tags = "${module.label.tags}"
+}

example/variables.tf

@@ -0,0 +1,43 @@
+variable "schedule" {
+ default = "rate(5 minutes)"
+}
+
+variable "namespace" {
+ type = "string"
+ description = "Namespace, which could be your organization name, e.g. 'eg' or 'cp'"
+}
+
+variable "stage" {
+ type = "string"
+ description = "Stage, e.g. 'prod', 'staging', 'dev', or 'test'"
+}
+
+variable "environment" {
+ type = "string"
+ default = ""
+ description = "Environment, e.g. 'testing', 'UAT'"
+}
+
+variable "name" {
+ type = "string"
+ default = "app"
+ description = "Solution name, e.g. 'app' or 'cluster'"
+}
+
+variable "delimiter" {
+ type = "string"
+ default = "-"
+ description = "Delimiter to be used between `name`, `namespace`, `stage`, etc."
+}
+
+variable "attributes" {
+ type = "list"
+ default = []
+ description = "Additional attributes (e.g. `1`)"
+}
+
+variable "tags" {
+ type = "map"
+ default = {}
+ description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)"
+}