Skip to content

Commit bfa8ae6

Browse files
author
dongliang
committed
kafka-v2.12-2.3.0
1 parent 561c8be commit bfa8ae6

File tree

2 files changed

+363
-0
lines changed

2 files changed

+363
-0
lines changed

‎kafka/2.12-2.3.0/kafka-cluster.sh‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
#!/bin/bash
2+
3+
HOMELOG='/home/logs'
4+
KAFKACLUSTER=$HOMELOG'/kafka-cluster'
5+
KAFKACLUSTERLOG_1=$KAFKACLUSTER'/kafka1-logs'
6+
KAFKACLUSTERLOG_2=$KAFKACLUSTER'/kafka2-logs'
7+
KAFKACLUSTERLOG_3=$KAFKACLUSTER'/kafka3-logs'
8+
KAFKAZKCLUSTERPATH_1=$KAFKACLUSTER'/zk1'
9+
KAFKAZKCLUSTERPATH_2=$KAFKACLUSTER'/zk2'
10+
KAFKAZKCLUSTERPATH_3=$KAFKACLUSTER'/zk3'
11+
12+
#basicpath
13+
if [ ! -d$HOMELOG ];then
14+
echo '::: '$HOMELOG' :::'
15+
mkdir $HOMELOG
16+
fi
17+
18+
if [ ! -d$KAFKACLUSTER ];then
19+
echo '::: '$KAFKACLUSTER' :::'
20+
mkdir $KAFKACLUSTER
21+
fi
22+
23+
#kafka logs
24+
if [ ! -d$KAFKACLUSTERLOG_1 ];then
25+
echo '::: '$KAFKACLUSTERLOG_1' not exsit folder :::'
26+
mkdir $KAFKACLUSTERLOG_1
27+
fi
28+
29+
if [ ! -d$KAFKACLUSTERLOG_2 ];then
30+
echo '::: '$KAFKACLUSTERLOG_2' not exsit folder :::'
31+
mkdir $KAFKACLUSTERLOG_2
32+
fi
33+
34+
if [ ! -d$KAFKACLUSTERLOG_3 ];then
35+
echo '::: '$KAFKACLUSTERLOG_3' not exsit folder :::'
36+
mkdir $KAFKACLUSTERLOG_3
37+
fi
38+
39+
#kafka zk cluster path
40+
if [ ! -d$KAFKAZKCLUSTERPATH_1 ];then
41+
echo '::: '$KAFKAZKCLUSTERPATH_1' not exsit folder :::'
42+
mkdir $KAFKAZKCLUSTERPATH_1
43+
fi
44+
45+
if [ ! -d$KAFKAZKCLUSTERPATH_2 ];then
46+
echo '::: '$KAFKAZKCLUSTERPATH_2' not exsit folder :::'
47+
mkdir $KAFKAZKCLUSTERPATH_2
48+
fi
49+
50+
if [ ! -d$KAFKAZKCLUSTERPATH_3 ];then
51+
echo '::: '$KAFKAZKCLUSTERPATH_3' not exsit folder :::'
52+
mkdir $KAFKAZKCLUSTERPATH_3
53+
fi
54+
55+
#kafka zk data
56+
if [ ! -d"/home/logs/kafka/zk1/data" ];then
57+
echo '::: /home/logs/kafka/zk1/data :::'
58+
mkdir /home/logs/kafka/zk1/data
59+
fi
60+
61+
if [ ! -d"/home/logs/kafka/zk2/data" ];then
62+
echo '::: /home/logs/kafka/zk2/data :::'
63+
mkdir /home/logs/kafka/zk2/data
64+
fi
65+
66+
if [ ! -d"/home/logs/kafka/zk3/data" ];then
67+
echo '::: /home/logs/kafka/zk3/data :::'
68+
mkdir /home/logs/kafka/zk3/data
69+
fi
70+
71+
#kafka zk log
72+
if [ ! -d"/home/logs/kafka/zk1/log" ];then
73+
echo '::: /home/logs/kafka/zk1/log :::'
74+
mkdir /home/logs/kafka/zk1/log
75+
fi
76+
77+
if [ ! -d"/home/logs/kafka/zk2/log" ];then
78+
echo '::: /home/logs/kafka/zk2/log :::'
79+
mkdir /home/logs/kafka/zk2/log
80+
fi
81+
82+
if [ ! -d"/home/logs/kafka/zk3/log" ];then
83+
echo '::: /home/logs/kafka/zk3/log :::'
84+
mkdir /home/logs/kafka/zk3/log
85+
fi
86+
87+
88+
89+
echo '::: start build zookeeper cluster :::'
90+
91+
#zookeeper-cluster-ip
92+
#default cluster 3
93+
MACHINE_HOST="10.1.62.21"
94+
95+
#zookeeper-cluster-port
96+
#default port 2181
97+
ZOOKEEPER_PORT_1=2151
98+
ZOOKEEPER_PORT_2=2152
99+
ZOOKEEPER_PORT_3=2153
100+
101+
#zookeeper-leader-port
102+
#default port 2888
103+
ZOOKEEPER_LEADERPORT_1=2887
104+
ZOOKEEPER_LEADERPORT_2=2888
105+
ZOOKEEPER_LEADERPORT_3=2889
106+
107+
#zookeeper-vote-port
108+
#default port 3888
109+
ZOOKEEPER_VOTEPORT_1=3887
110+
ZOOKEEPER_VOTEPORT_2=3888
111+
ZOOKEEPER_VOTEPORT_3=3889
112+
113+
#build zk1、zk2、zk3
114+
docker rm -f kafka-zk1
115+
docker rm -f kafka-zk2
116+
docker rm -f kafka-zk3
117+
118+
docker run -d \
119+
--restart=always \
120+
--name kafka-zk1 \
121+
-p $ZOOKEEPER_PORT_1:2181 \
122+
-p $ZOOKEEPER_LEADERPORT_1:2888 \
123+
-p $ZOOKEEPER_VOTEPORT_1:3888 \
124+
-e "ZOO_INIT_LIMIT=2000" \
125+
-e "ZOO_MAX_CLIENT_CNXNS=10000" \
126+
-e "TZ=Asia/Shanghai" \
127+
-e "ZOO_MY_ID=1" \
128+
-e "ZOO_SERVERS=server.1=0.0.0.0:2888:3888 server.2=$MACHINE_HOST:$ZOOKEEPER_LEADERPORT_2:$ZOOKEEPER_VOTEPORT_2 server.3=$MACHINE_HOST:$ZOOKEEPER_LEADERPORT_3:$ZOOKEEPER_VOTEPORT_3" \
129+
-v /home/logs/kafka/zk1/data:/data \
130+
-v /home/logs/kafka/zk1/log:/datalog \
131+
zookeeper:3.5.5
132+
133+
docker run -d \
134+
--restart=always \
135+
--name kafka-zk2 \
136+
-p $ZOOKEEPER_PORT_2:2181 \
137+
-p $ZOOKEEPER_LEADERPORT_2:2888 \
138+
-p $ZOOKEEPER_VOTEPORT_2:3888 \
139+
-e "ZOO_INIT_LIMIT=2000" \
140+
-e "ZOO_MAX_CLIENT_CNXNS=10000" \
141+
-e "TZ=Asia/Shanghai" \
142+
-e "ZOO_MY_ID=2" \
143+
-e "ZOO_SERVERS=server.1=$MACHINE_HOST:$ZOOKEEPER_LEADERPORT_1:$ZOOKEEPER_VOTEPORT_1 server.2=0.0.0.0:2888:3888 server.3=$MACHINE_HOST:$ZOOKEEPER_LEADERPORT_3:$ZOOKEEPER_VOTEPORT_3" \
144+
-v /home/logs/kafka/zk2/data:/data \
145+
-v /home/logs/kafka/zk2/log:/datalog \
146+
zookeeper:3.5.5
147+
148+
docker run -d \
149+
--restart=always \
150+
--name kafka-zk3 \
151+
-p $ZOOKEEPER_PORT_3:2181 \
152+
-p $ZOOKEEPER_LEADERPORT_3:2888 \
153+
-p $ZOOKEEPER_VOTEPORT_3:3888 \
154+
-e "ZOO_INIT_LIMIT=2000" \
155+
-e "ZOO_MAX_CLIENT_CNXNS=10000" \
156+
-e "TZ=Asia/Shanghai" \
157+
-e "ZOO_MY_ID=3" \
158+
-e "ZOO_SERVERS=server.1=$MACHINE_HOST:$ZOOKEEPER_LEADERPORT_1:$ZOOKEEPER_VOTEPORT_1 server.2=$MACHINE_HOST:$ZOOKEEPER_LEADERPORT_2:$ZOOKEEPER_VOTEPORT_2 server.3=0.0.0.0:2888:3888" \
159+
-v /home/logs/kafka/zk3/data:/data \
160+
-v /home/logs/kafka/zk3/log:/datalog \
161+
zookeeper:3.5.5
162+
163+
164+
echo '::: build zookeeper cluster successfully :::'
165+
166+
167+
echo '::: start build kafka cluster :::'
168+
169+
KAFKA_PORT_1=9092
170+
KAFKA_PORT_2=9093
171+
KAFKA_PORT_3=9094
172+
173+
#kafka broker
174+
#docker rm -f kafka-cluster-1
175+
#docker rm -f kafka-cluster-2
176+
#docker rm -f kafka-cluster-3
177+
178+
179+
# docker run -d \
180+
# --restart=always \
181+
# --name kafka-cluster-1 \
182+
# -p $KAFKA_PORT_1:9092 \
183+
# -e KAFKA_ZOOKEEPER_CONNECT=$MACHINE_HOST:$ZOOKEEPER_PORT_1,$MACHINE_HOST:$ZOOKEEPER_PORT_2,$MACHINE_HOST:$ZOOKEEPER_PORT_3 \
184+
# -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$MACHINE_HOST:$KAFKA_PORT_1 \
185+
# -e KAFKA_LISTENERS=PLAINTEXT://:$KAFKA_PORT_1 \
186+
# -e KAFKA_BROKER_ID=0 \
187+
# -e KAFKA_HEAP_OPTS="-Xmx4G -Xms4G" \
188+
# -e KAFKA_LOG_RETENTION_HOURS=24 \
189+
# -v /etc/localtime:/etc/localtime \
190+
# -v /home/logs/kafka1-logs/:/kafka/logs \
191+
# wurstmeister/kafka:2.12-2.3.0
192+
193+
# docker run -d \
194+
# --restart=always \
195+
# --name kafka-cluster-2 \
196+
# -p $KAFKA_PORT_2:9092 \
197+
# -e KAFKA_ZOOKEEPER_CONNECT=$MACHINE_HOST:$ZOOKEEPER_PORT_1,$MACHINE_HOST:$ZOOKEEPER_PORT_2,$MACHINE_HOST:$ZOOKEEPER_PORT_3 \
198+
# -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$MACHINE_HOST:$KAFKA_PORT_2 \
199+
# -e KAFKA_LISTENERS=PLAINTEXT://:$KAFKA_PORT_2 \
200+
# -e KAFKA_BROKER_ID=0 \
201+
# -e KAFKA_HEAP_OPTS="-Xmx4G -Xms4G" \
202+
# -e KAFKA_LOG_RETENTION_HOURS=24 \
203+
# -v /etc/localtime:/etc/localtime \
204+
# -v /home/logs/kafka2-logs/:/kafka/logs \
205+
# wurstmeister/kafka:2.12-2.3.0
206+
207+
# docker run -d \
208+
# --restart=always \
209+
# --name kafka-cluster-3 \
210+
# -p $KAFKA_PORT_3:9092 \
211+
# -e KAFKA_ZOOKEEPER_CONNECT=$MACHINE_HOST:$ZOOKEEPER_PORT_1,$MACHINE_HOST:$ZOOKEEPER_PORT_2,$MACHINE_HOST:$ZOOKEEPER_PORT_3 \
212+
# -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$MACHINE_HOST:$KAFKA_PORT_3 \
213+
# -e KAFKA_LISTENERS=PLAINTEXT://:$KAFKA_PORT_3 \
214+
# -e KAFKA_BROKER_ID=0 \
215+
# -e KAFKA_HEAP_OPTS="-Xmx4G -Xms4G" \
216+
# -e KAFKA_LOG_RETENTION_HOURS=24 \
217+
# -v /etc/localtime:/etc/localtime \
218+
# -v /home/logs/kafka3-logs/:/kafka/logs \
219+
# wurstmeister/kafka:2.12-2.3.0
220+
221+
echo '::::::::: build kafka cluster successfully! ::::::::::::::'
222+
223+
224+
echo '::: start build kafka-manager :::'
225+
226+
KAFKAMANAGER_PORT=14672
227+
228+
#kafka manager
229+
#docker rm -f kafka-manager
230+
231+
# docker run -d \
232+
# --restart=always \
233+
# --name kafka-manager \
234+
# -p $KAFKAMANAGER_PORT:9000 \
235+
# -e ZK_HOSTS=$MACHINE_HOST:$ZOOKEEPER_PORT_1,$MACHINE_HOST:$ZOOKEEPER_PORT_2,$MACHINE_HOST:$ZOOKEEPER_PORT_3 \
236+
# -e APPLICATION_SECRET=letmein \
237+
# sheepkiller/kafka-manager
238+
239+
echo '::::::::: build kafka-manager successfully! ::::::::::::::'

‎kafka/2.12-2.3.0/kafka-single.sh‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/bin/bash
2+
3+
HOMELOG='/home/logs'
4+
KAFKASINGLE=$HOMELOG'/kafka-single'
5+
KAFKALOG=$KAFKASINGLE'/kafka-logs'
6+
KAFKAZK=$KAFKASINGLE'/zk'
7+
KAFKAZKDATA=$KAFKAZK'/data'
8+
KAFKAZKLOG=$KAFKAZK'/log'
9+
10+
#defind zk-version
11+
ZK_VERSION=3.5.5
12+
13+
#defind kafka-version
14+
KAFKA_VERSION=2.12-2.3.0
15+
16+
#defind kafka-manager-version
17+
KAFKAMANAGER_VERSION=stable
18+
19+
if [ ! -d$KAFKALOG ];then
20+
echo '::: '$KAFKALOG' not exsit folder :::'
21+
mkdir $KAFKALOG
22+
fi
23+
24+
if [ ! -d$KAFKASINGLE ];then
25+
echo '::: '$KAFKASINGLE' not exsit folder :::'
26+
mkdir $KAFKASINGLE
27+
fi
28+
29+
if [ ! -d$KAFKAZK ];then
30+
echo '::: '$KAFKAZK' not exsit folder :::'
31+
mkdir $KAFKAZK
32+
fi
33+
34+
#kafka logs
35+
if [ ! -d$KAFKALOG ];then
36+
echo '::: '$KAFKALOG' not exsit folder :::'
37+
mkdir $KAFKALOG
38+
fi
39+
40+
#kafka zk data
41+
if [ ! -d$KAFKAZKDATA ];then
42+
echo '::: '$KAFKAZKDATA' :::'
43+
mkdir $KAFKAZKDATA
44+
fi
45+
46+
#kafka zk log
47+
if [ ! -d$KAFKAZKLOG ];then
48+
echo '::: '$KAFKAZKLOG' :::'
49+
mkdir $KAFKAZKLOG
50+
fi
51+
52+
53+
54+
echo '::: start build zookeeper-single :::'
55+
56+
#zookeeper-cluster-ip
57+
#default cluster 3
58+
MACHINE_HOST="10.1.62.21"
59+
60+
#zookeeper-cluster-port
61+
#default port 2181
62+
ZOOKEEPER_PORT_1=2150
63+
64+
65+
#build zk
66+
docker rm -f kafka-single-zk1
67+
68+
docker run -d \
69+
--restart=always \
70+
--name kafka-single-zk1 \
71+
-p $ZOOKEEPER_PORT_1:2181 \
72+
-e "ZOO_INIT_LIMIT=2000" \
73+
-e "ZOO_MAX_CLIENT_CNXNS=10000" \
74+
-e "TZ=Asia/Shanghai" \
75+
-v $KAFKAZKDATA:/data \
76+
-v $KAFKAZKLOG:/datalog \
77+
zookeeper:$ZK_VERSION
78+
79+
echo '::: build zookeeper successfully :::'
80+
81+
82+
echo '::: start build kafka single :::'
83+
84+
KAFKA_PORT_1=9090
85+
86+
#kafka broker
87+
#eg ==> send : ./bin/kafka-console-producer.sh --broker-list localhost:9092 --topic mykafka
88+
#eg ==> receive : ./bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic mykafka --from-beginning
89+
docker rm -f kafka-single
90+
91+
92+
docker run -d \
93+
--restart=always \
94+
--name kafka-single \
95+
-p $KAFKA_PORT_1:9090 \
96+
-e KAFKA_ZOOKEEPER_CONNECT=$MACHINE_HOST:$ZOOKEEPER_PORT_1 \
97+
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$MACHINE_HOST:$KAFKA_PORT_1 \
98+
-e KAFKA_LISTENERS=PLAINTEXT://:$KAFKA_PORT_1 \
99+
-e KAFKA_BROKER_ID=0 \
100+
-e KAFKA_HEAP_OPTS="-Xmx4G -Xms4G" \
101+
-e KAFKA_LOG_RETENTION_HOURS=24 \
102+
-v /etc/localtime:/etc/localtime \
103+
-v $KAFKALOG:/kafka/logs \
104+
wurstmeister/kafka:$KAFKA_VERSION
105+
106+
echo '::::::::: build kafka single successfully! ::::::::::::::'
107+
108+
109+
echo '::: start build kafka-single-manager :::'
110+
111+
KAFKAMANAGER_PORT=14670
112+
113+
#kafka manager
114+
docker rm -f kafka-single-manager
115+
116+
docker run -d \
117+
--restart=always \
118+
--name kafka-single-manager \
119+
-p $KAFKAMANAGER_PORT:9000 \
120+
-e ZK_HOSTS=$MACHINE_HOST:$ZOOKEEPER_PORT_1 \
121+
-e APPLICATION_SECRET=letmein \
122+
sheepkiller/kafka-manager:$KAFKAMANAGER_VERSION
123+
124+
echo '::::::::: build kafka-single-manager successfully! ::::::::::::::'

0 commit comments

Comments
 (0)