chore: update README and fix data directory (conduktor#80)

Author: devshawn

.gitignore

@@ -2,4 +2,6 @@ zk-single-kafka-single/*
 zk-single-kafka-multiple/*
 zk-multiple-kafka-single/*
 zk-multiple-kafka-multiple/*
+zk-multiple-kafka-multiple-schema-registry/*
 full-stack/*
+.idea/

.travis.yml

@@ -25,4 +25,5 @@ script:
 - ./test.sh zk-multiple-kafka-single.yml 4
 - ./test.sh zk-single-kafka-multiple.yml 4
 - ./test.sh zk-multiple-kafka-multiple.yml 6
+- ./test.sh zk-multiple-kafka-multiple-schema-registry.yml 7
 - ./test.sh full-stack.yml 10

README.md

@@ -177,8 +177,8 @@ A: yes. This is for testing only!!! Reduce the KAFKA_LOG_SEGMENT_BYTES to 16MB a
 
 **Q: How do I expose kafka?**
 
-A: Incase you want to expose kafka , you must set ```KAFKA_ADVERTISED_LISTENERS``` to the IP of the machine so that kafka is externally accessible. To achieve this you can set ```LISTENER_DOCKER_EXTERNAL``` to the IP of the machine.
-Say IP is ```50.10.2.3```, follow the sample mapping below:
+A: If you want to expose kafka outside of your local machine, you must set `KAFKA_ADVERTISED_LISTENERS` to the IP of the machine so that kafka is externally accessible. To achieve this you can set `LISTENER_DOCKER_EXTERNAL` to the IP of the machine.
+For example, if the IP of your machine is `50.10.2.3`, follow the sample mapping below:
 
 ```
  kafka1:

full-stack.yml

@@ -8,9 +8,9 @@ services:
  ports:
  - "2181:2181"
  environment:
-  ZOO_MY_ID: 1
-  ZOO_PORT: 2181
-  ZOO_SERVERS: server.1=zoo1:2888:3888
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zoo1:2888:3888
  volumes:
  - ./full-stack/zoo1/data:/data
  - ./full-stack/zoo1/datalog:/datalog

test.sh

@@ -37,11 +37,13 @@ kafka_tests(){
  topic="testtopic"
  if grep -q kafka3 $1; then replication_factor="3"; else replication_factor="1"; fi
  for i in 1 2 3 4 5; do echo "trying to create test topic" && kafka-topics --create --topic $topic --replication-factor $replication_factor --partitions 12 --zookeeper $DOCKER_HOST_IP:2181 && break || sleep 5; done
+ sleep 5
  for x in {1..100}; do echo $x; done | kafka-console-producer --broker-list $DOCKER_HOST_IP:9092 --topic $topic
- rows=`kafka-console-consumer --bootstrap-server $DOCKER_HOST_IP:9092 --topic $topic --from-beginning --timeout-ms 2000 | wc -l`
+ sleep 5
+ rows=`kafka-console-consumer --bootstrap-server $DOCKER_HOST_IP:9092 --topic $topic --from-beginning --timeout-ms 10000 | wc -l`
  # rows=`kafkacat -C -b $DOCKER_HOST_IP:9092 -t $topic -o beginning -e | wc -l `
  if [ "$rows" != "100" ]; then
- kafka-console-consumer --bootstrap-server $DOCKER_HOST_IP:9092 --topic test-topic --from-beginning --timeout-ms 2000 | wc -l
+ kafka-console-consumer --bootstrap-server $DOCKER_HOST_IP:9092 --topic test-topic --from-beginning --timeout-ms 10000 | wc -l
  exit 1
  else
  echo "Kafka Test Success"

zk-multiple-kafka-multiple-schema-registry.yml

@@ -7,38 +7,38 @@ services:
  ports:
  - "2181:2181"
  environment:
-  ZOO_MY_ID: 1
-  ZOO_PORT: 2181
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
- - ./zk-multiple-kafka-multiple/zoo1/data:/data
- - ./zk-multiple-kafka-multiple/zoo1/datalog:/datalog
+ - ./zk-multiple-kafka-multiple-schema-registry/zoo1/data:/data
+ - ./zk-multiple-kafka-multiple-schema-registry/zoo1/datalog:/datalog
 
  zoo2:
  image: zookeeper:3.4.9
  hostname: zoo2
  ports:
  - "2182:2182"
  environment:
-  ZOO_MY_ID: 2
-  ZOO_PORT: 2182
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 2
+ ZOO_PORT: 2182
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
- - ./zk-multiple-kafka-multiple/zoo2/data:/data
- - ./zk-multiple-kafka-multiple/zoo2/datalog:/datalog
+ - ./zk-multiple-kafka-multiple-schema-registry/zoo2/data:/data
+ - ./zk-multiple-kafka-multiple-schema-registry/zoo2/datalog:/datalog
 
  zoo3:
  image: zookeeper:3.4.9
  hostname: zoo3
  ports:
  - "2183:2183"
  environment:
-  ZOO_MY_ID: 3
-  ZOO_PORT: 2183
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 3
+ ZOO_PORT: 2183
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
- - ./zk-multiple-kafka-multiple/zoo3/data:/data
- - ./zk-multiple-kafka-multiple/zoo3/datalog:/datalog
+ - ./zk-multiple-kafka-multiple-schema-registry/zoo3/data:/data
+ - ./zk-multiple-kafka-multiple-schema-registry/zoo3/datalog:/datalog
 
 
  kafka1:
@@ -47,13 +47,14 @@ services:
  ports:
  - "9092:9092"
  environment:
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:19092,PLAINTEXT_HOST://kafka1:9092
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
  KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
  KAFKA_BROKER_ID: 1
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=: 'false'
+ KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
  volumes:
- - ./zk-multiple-kafka-multiple/kafka1/data:/var/lib/kafka/data
+ - ./zk-multiple-kafka-multiple-schema-registry/kafka1/data:/var/lib/kafka/data
  depends_on:
  - zoo1
  - zoo2
@@ -65,13 +66,14 @@ services:
  ports:
  - "9093:9093"
  environment:
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:19093,PLAINTEXT_HOST://kafka2:9093
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
  KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
  KAFKA_BROKER_ID: 2
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=: 'false'
+ KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
  volumes:
- - ./zk-multiple-kafka-multiple/kafka2/data:/var/lib/kafka/data
+ - ./zk-multiple-kafka-multiple-schema-registry/kafka2/data:/var/lib/kafka/data
  depends_on:
  - zoo1
  - zoo2
@@ -83,21 +85,22 @@ services:
  ports:
  - "9094:9094"
  environment:
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:19094,PLAINTEXT_HOST://kafka3:9094
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
  KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
  KAFKA_BROKER_ID: 3
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=: 'false'
+ KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
  volumes:
- - ./zk-multiple-kafka-multiple/kafka3/data:/var/lib/kafka/data
+ - ./zk-multiple-kafka-multiple-schema-registry/kafka3/data:/var/lib/kafka/data
  depends_on:
  - zoo1
  - zoo2
  - zoo3
- schema-registry:
+
+ kafka-schema-registry:
  image: confluentinc/cp-schema-registry:5.5.1
- hostname: schema-registry
- container_name: schema-registry
+ hostname: kafka-schema-registry
  depends_on:
  - zoo1
  - zoo2
@@ -108,5 +111,6 @@ services:
  ports:
  - "8081:8081"
  environment:
- SCHEMA_REGISTRY_HOST_NAME: schema-registry
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:9092,PLAINTEXT://kafka2:9093,PLAINTEXT://kafka3:9094' 
+ SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
+ SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094'
+ SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081

zk-multiple-kafka-multiple.yml

@@ -7,9 +7,9 @@ services:
  ports:
  - "2181:2181"
  environment:
-  ZOO_MY_ID: 1
-  ZOO_PORT: 2181
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
  - ./zk-multiple-kafka-multiple/zoo1/data:/data
  - ./zk-multiple-kafka-multiple/zoo1/datalog:/datalog
@@ -20,9 +20,9 @@ services:
  ports:
  - "2182:2182"
  environment:
-  ZOO_MY_ID: 2
-  ZOO_PORT: 2182
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 2
+ ZOO_PORT: 2182
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
  - ./zk-multiple-kafka-multiple/zoo2/data:/data
  - ./zk-multiple-kafka-multiple/zoo2/datalog:/datalog
@@ -33,9 +33,9 @@ services:
  ports:
  - "2183:2183"
  environment:
-  ZOO_MY_ID: 3
-  ZOO_PORT: 2183
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 3
+ ZOO_PORT: 2183
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
  - ./zk-multiple-kafka-multiple/zoo3/data:/data
  - ./zk-multiple-kafka-multiple/zoo3/datalog:/datalog

zk-multiple-kafka-single.yml

@@ -7,9 +7,9 @@ services:
  ports:
  - "2181:2181"
  environment:
-  ZOO_MY_ID: 1
-  ZOO_PORT: 2181
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
  - ./zk-multiple-kafka-single/zoo1/data:/data
  - ./zk-multiple-kafka-single/zoo1/datalog:/datalog
@@ -20,9 +20,9 @@ services:
  ports:
  - "2182:2182"
  environment:
-  ZOO_MY_ID: 2
-  ZOO_PORT: 2182
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 2
+ ZOO_PORT: 2182
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
  - ./zk-multiple-kafka-single/zoo2/data:/data
  - ./zk-multiple-kafka-single/zoo2/datalog:/datalog
@@ -33,9 +33,9 @@ services:
  ports:
  - "2183:2183"
  environment:
-  ZOO_MY_ID: 3
-  ZOO_PORT: 2183
-  ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+ ZOO_MY_ID: 3
+ ZOO_PORT: 2183
+ ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  volumes:
  - ./zk-multiple-kafka-single/zoo3/data:/data
  - ./zk-multiple-kafka-single/zoo3/datalog:/datalog

zk-single-kafka-multiple.yml

@@ -7,9 +7,9 @@ services:
  ports:
  - "2181:2181"
  environment:
-  ZOO_MY_ID: 1
-  ZOO_PORT: 2181
-  ZOO_SERVERS: server.1=zoo1:2888:3888
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zoo1:2888:3888
  volumes:
  - ./zk-single-kafka-multiple/zoo1/data:/data
  - ./zk-single-kafka-multiple/zoo1/datalog:/datalog

zk-single-kafka-single.yml

@@ -7,9 +7,9 @@ services:
  ports:
  - "2181:2181"
  environment:
-  ZOO_MY_ID: 1
-  ZOO_PORT: 2181
-  ZOO_SERVERS: server.1=zoo1:2888:3888
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zoo1:2888:3888
  volumes:
  - ./zk-single-kafka-single/zoo1/data:/data
  - ./zk-single-kafka-single/zoo1/datalog:/datalog