An Introduction to Chaos Engineering: Trying to Break Stuff

Let me explain

What is Chaos Engineering?

$ pip3 install -U chaostoolkit chaostoolkit-aws $ chaos --version chaos, version 1.3.0

Now, our resilient application

• The Echo server will be a service running on an AWS EC2 instance with an Elastic IP attached to it.
• If the EC2 instance fails, another one will take its place and will reclaim the Elastic IP.
• KeepaliveD will be used to make this happen.

import java.io.*; import java.net.*; import java.lang.Thread; public class EchoServer { public static void main (String[] args) { try { ServerSocket server = new ServerSocket(5566); System.out.println("Listing on port 5566..."); while (true) { Socket client = server.accept(); EchoHandler handler = new EchoHandler(client); handler.start(); } } catch (Exception e) { System.err.println("Exception caught:" + e); } } } class EchoHandler extends Thread { Socket client; InetAddress inetAddress; String configServer; EchoHandler (Socket client) { this.client = client; try { inetAddress = InetAddress.getLocalHost(); } catch(Exception e) {} } public void run () { try { BufferedReader reader = new BufferedReader(new InputStreamReader(client.getInputStream())); PrintWriter writer = new PrintWriter(client.getOutputStream(), true); System.out.println("[connected] " + client.getInetAddress()); while (true) { String line = reader.readLine(); System.out.println(client.getInetAddress() + " says " + line); writer.println(line); if (line.trim().equals("bye")) { break; } } } catch (Exception e) { System.err.println("Exception caught: client disconnected."); } finally { try { System.out.println("[disconnected] " + client.getInetAddress()); client.close(); } catch (Exception e ){ ; } } } } 

Our experiment

{ "version": "1.0.0", "title": "Validating High-Availability of the Echo server.", "description": "Ensure that it will be always an EC2 instance with the ElasticIP attached.", "tags": [ "echoserver", "keepalived", "elasticip", "ha", "master", "backup" ], "configuration": { "aws_region": "us-east-1" }, "steady-state-hypothesis": { "title": "The backup node will take the role of master when the original one is terminated.", "probes": [ { "type": "probe", "name": "port-5566-is-listening-and-working-properly", "tolerance": true, "provider": { "type": "python", "module": "chaospythian.echo.probes", "func": "echoserver", "arguments": { "tcp_ip": "54.71.185.10", "tcp_port": 5566, "message": "testing text\n" } } }, { "type": "probe", "name": "master-and-backup-instances-up-and-running", "tolerance": [1,2], "provider": { "type": "python", "module": "chaosaws.ec2.probes", "func": "count_instances", "arguments": { "filters": [ { "Name": "instance-state-name", "Values": ["running"] }, { "Name": "tag:Name", "Values": ["production-echoserver"] } ] } } }, { "type": "probe", "name": "one-ec2-instance-tagged-as-master", "tolerance": 1, "provider": { "type": "python", "module": "chaosaws.ec2.probes", "func": "count_instances", "arguments": { "filters": [ { "Name": "instance-state-name", "Values": ["running"] }, { "Name": "tag:State", "Values": ["MASTER"] }, { "Name": "tag:Name", "Values": ["production-echoserver"] } ] } } }, { "type": "probe", "name": "one-ec2-instance-tagged-as-backup", "tolerance": [0,1], "provider": { "type": "python", "module": "chaosaws.ec2.probes", "func": "count_instances", "arguments": { "filters": [ { "Name": "instance-state-name", "Values": ["running"] }, { "Name": "tag:State", "Values": ["BACKUP"] }, { "Name": "tag:Name", "Values": ["production-echoserver"] } ] } } } ] }, "method": [ { "type": "action", "name": "terminate-master-node", "provider": { "type": "python", "module": "chaosaws.ec2.actions", "func": "terminate_instance", "arguments": { "filters": [ { "Name": "instance-state-name", "Values": ["running"] }, { "Name": "tag:State", "Values": ["MASTER"] }, { "Name": "tag:Name", "Values": ["production-echoserver"] } ] } }, "pauses": { "after": 10 } } ] } 

Client-side:

$ nc 54.71.185.10 5566 hello hello test test one two three one two three bye bye

Server-side:

Listing on port 5566... [connected] /190.104.119.195 /190.104.119.195 says hello /190.104.119.195 says test /190.104.119.195 says one two three /190.104.119.195 says bye [disconnected] /190.104.119.195

Now the fun part: let's terminate the master node!

# -*- coding: utf-8 -*- import socket __all__ = ["echoserver"] def echoserver(tcp_ip, tcp_port, message): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((tcp_ip, tcp_port)) s.send(message.encode('ascii')) data = s.recv(1024) s.send("bye".encode('ascii')) s.close() return (data.decode('ascii').strip() == message.strip())