1. kafka集群

master	slave1	slave2
zookeeper	zookeeper	zookeeper
kafka	kafka	kafka

2. 下载kafka安装包

[root@master local]# wget https://archive.apache.org/dist/kafka/0.10.2.1/kafka_2.11-0.10.2.1.tgz

3. kafka集群部署

3.1 解压至安装路径

[root@master local]# tar -zxvf kafka_2.11-0.10.2.1.tgz -C /usr/local

3.2 在$KAFKA_HOME目录下新建logs目录

[root@master kafka_2.11-0.10.2.1]# mkdir logs

3.3 修改$KAFKA_HOME/config目录下的配置文件

[root@master config]# vim server.properties

############################# Server Basics #############################

# The id of the broker. This must be set to a unique integer for each broker.
broker.id=0

# Switch to enable topic deletion or not, default value is false
delete.topic.enable=true

############################# Log Basics #############################

# A comma seperated list of directories under which to store log files
log.dirs=/usr/local/kafka_2.11-0.10.2.1/logs

############################# Zookeeper #############################

# Zookeeper connection string (see zookeeper docs for details).
# This is a comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect=master:2181,slave1:2181,slave2:2181

3.4 配置环境变量

[root@master config]# vim ~/.bashrc

#kafka
export KAFKA_HOME=/usr/local/kafka_2.11-0.10.2.1
export PATH=$PATH:$KAFKA_HOME/bin

3.5 source环境变量文件

[root@master config]# source ~/.bashrc

3.6 将配置好的kafka分发slave1、slave2

[root@master local]# scp -r kafka_2.11-0.10.2.1/ root@slave1:/usr/local/

[root@master local]# scp -r kafka_2.11-0.10.2.1/ root@slave1:/usr/local/

4. 修改slave1和slave2的配置文件

slave1

[root@slave1 config]# vim server.properties

# 修改slave节点的broker id
broker.id=1

slave2

[root@slave2 config]# vim server.properties

# 修改slave节点的broker id
broker.id=2

配置slave1和slave2的环境变量

vim ~/.bashrc

#kafka
export KAFKA_HOME=/usr/local/kafka_2.11-0.10.2.1
export PATH=$PATH:$KAFKA_HOME/bin

source环境变量文件

source ~/.bashrc

5. 启动kafka集群

在master、slave1、slave2分别启动kafka

[root@master bin]# JMX_PORT=9988 $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties  #JMX是kafka manager监控kafka的服务，指定端口号

[root@slave1 bin]# JMX_PORT=9988 $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties  #JMX是kafka manager监控kafka的服务，指定端口号

[root@slave2 bin]# JMX_PORT=9988 $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties  #JMX是kafka manager监控kafka的服务，指定端口号

6. 查看kafka JVM进程

master

[root@master bin]# jps
7441 Jps
3058 ResourceManager
2902 SecondaryNameNode
2748 NameNode
3644 QuorumPeerMain
7230 Kafka

slave1

[root@slave1 kafka_2.11-0.10.2.1]# jps
6146 Jps
5971 Kafka
2532 NodeManager
2455 DataNode
2955 QuorumPeerMain

slave2

[root@slave2 kafka_2.11-0.10.2.1]# jps
6113 Jps
2500 NodeManager
2423 DataNode
2887 QuorumPeerMain
5928 Kafka