欧拉操作系统安装 kafka 和 zookeeper 分布式集群

timedatectl set-timezone Asia/Shanghaicat >> /etc/sysctl.conf << EOFkernel.shmmax = kernel.shmall = kernel.sysrq = 1vm.swappiness = 1fs.inotify.max_user_watches = 10000000net.core.wmem_max = 16777216net.core.rmem_max = 16777216net.ipv4.conf.all.send_redirects = 0net.ipv4.conf.default.send_redirects = 0net.ipv4.conf.all.secure_redirects = 0net.ipv4.conf.default.secure_redirects = 0net.ipv4.conf.all.accept_redirects = 0net.ipv4.conf.default.accept_redirects = 0fs.inotify.max_queued_events = 327679net.ipv4.neigh.default.gc_thresh1 = 2048net.ipv4.neigh.default.gc_thresh2 = 4096net.ipv4.neigh.default.gc_thresh3 = 8192net.ipv6.conf.all.disable_ipv6 = 0net.ipv6.conf.default.disable_ipv6 = 0net.ipv6.conf.lo.disable_ipv6 = 0net.ipv4.tcp_keepalive_time = 1200net.ipv4.tcp_keepalive_intvl = 15net.ipv4.tcp_keepalive_probes = 5net.ipv4.tcp_max_syn_backlog = 4096net.core.somaxconn = 4096net.core.netdev_max_backlog = 3000net.ipv4.tcp_tw_reuse = 1net.ipv4.tcp_fin_timeout = 30net.ipv4.tcp_fack = 1net.ipv4.tcp_sack = 1net.core.wmem_default = 256960net.core.wmem_max = 4088000net.core.rmem_default = 256960net.core.rmem_max = 4088000net.ipv4.tcp_wmem = 8760 256960 4088000net.ipv4.tcp_rmem = 8760 256960 4088000net.ipv4.tcp_mem = 131072 262144 524288vm.max_map_count=655360EOFsysctl -pecho never > /sys/kernel/mm/transparent_hugepage/enabledecho never > /sys/kernel/mm/transparent_hugepage/defrag

cat >> /etc/security/limits.conf << EOF* soft nofile 1024000* hard nofile 1024000* soft nproc 655350* hard nproc 655350* soft stack unlimited* hard stack unlimited* soft core unlimited* hard core unlimited root soft nofile 1024000root hard nofile 1024000root soft nproc 655350root hard nproc 655350root soft stack unlimitedroot hard stack unlimitedroot soft core unlimitedroot hard core unlimited yunwei soft nofile 1024000yunwei hard nofile 1024000yunwei soft nproc 655350yunwei hard nproc 655350yunwei soft stack unlimitedyunwei hard stack unlimitedyunwei soft core unlimitedyunwei hard core unlimitedEOF

lsblk

mkdir /datavgcreate data /dev/sdblvcreate -n lv1 -l 100%FREE datamkfs.xfs /dev/mapper/data-lv1

cat >> /etc/fstab << EOF/dev/data/lv1/data xfs defaults 0 0EOFmount -adf -h

mkdir /data/soft# 上传相关软件到这个目录ls /data/softsystemctl stop firewalld && systemctl disable firewalldsetenforce 0sed -i  's/SELINUX/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config sed -i  's/SELINUX/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config 

cd /data/softtar xvf jdk-8u421-linux-x64.tar.gzmv jdk1.8.0_421 /usr/local/jdkcat >>  /etc/profile << EOFexport PATH=$PATH:/usr/local/jdk/binexport JAVA_HOME=/usr/local/jdkEOFsource /etc/profile

yum install gcc net-tools tar vim htop bash-co* rsync wget jq unzip lrzsz make -y

yum install ntp ntpdate -y

vim /etc/ntp.conf server ntp.aliyun.com # poll 开头的都注释了 systemctl enable ntp && systemctl start ntp crontab -e */5 * * * * ntpdate -u ntp.aliyun.com

vim /etc/ntp.conf server node1 # poll 开头的都注释了 systemctl enable ntp && systemctl start ntpcrontab -e */5 * * * * ntpdate -u node1

ssh-keygenssh-copy-id node1ssh-copy-id node2ssh-copy-id node3 # 测试ssh node1ssh node2ssh node3

wget https://archive.apache.org/dist/kafka/3.4.1/kafka_2.13-3.4.1.tgzls -alkafka_2.13-3.4.1.tgz

cd /data/softtar xvf kafka_2.13-3.4.1.tgz mv kafka_2.13-3.4.1 ../kafkacd /data/kafka# 目录的创建，这里的创建需要和配置文件中对应。 可以按自己需求定义mkdir -p data zookeeper/log

cd /data/kafka/configmv zookeeper.properties zookeeper.properties.bakcat > zookeeper.properties << EOFdataDir=/data/kafka/zookeeperdataLogDir=/data/kafka/zookeeper/logclientPort=2181maxClientCnxns=0admin.enableServer=falseserver.1=node1:2888:3888server.2=node2:2888:3888server.3=node3:2888:3888tickTime=2000 initLimit=10 syncLimit=5EOF

echo 1 > /data/kafka/zookeeper/myid

cat > /etc/systemd/system/zookeeper.service << EOF[Unit]Description=Apache zookeeperAfter=network.target Requires=network.target  [Service]Environment="JAVA_HOME=/usr/local/jdk"ExecStart=/data/kafka/bin/zookeeper-server-start.sh  /data/kafka/config/zookeeper.propertiesExecStop=/data/kafka/bin/zookeeper-server-stop.shRestart=alwaysRestartSec=10StartLimitInterval=5LimitNOFILE=265535LimitNPROC=65535 [Install]WantedBy=multi-user.targetEOF

cd /data/kafka/configmv server.properties server.properties.bak

cat > server.properties << EOFbroker.id=0listeners=PLAINTEXT://node1:9092num.network.threads=3num.io.threads=8socket.send.buffer.bytes=102400socket.receive.buffer.bytes=102400socket.request.max.bytes=104857600log.dirs=/data/kafka/datanum.partitions=3num.recovery.threads.per.data.dir=1offsets.topic.replication.factor=3transaction.state.log.replication.factor=3transaction.state.log.min.isr=3log.retention.hours=168log.retention.check.interval.ms=300000zookeeper.connect=node1:2181,node2:2181,node3:2181zookeeper.connection.timeout.ms=18000group.initial.rebalance.delay.ms=0EOF

cat > /etc/systemd/system/kafka.service << EOF[Unit]Description=Apache Kafka serverAfter=network.target Requires=network.target  [Service]Type=forkingEnvironment="JAVA_HOME=/usr/local/jdk"ExecStart=/data/kafka/bin/kafka-server-start.sh -daemon /data/kafka/config/server.propertiesExecStop=/data/kafka/bin/kafka-server-stop.shRestart=alwaysRestartSec=10StartLimitInterval=5LimitNOFILE=265535 [Install]WantedBy=multi-user.targetEOF

cd /data/kafka/config/kraftmv server.properties server.properties.bak

cat > server.properties << EOFprocess.roles=broker,controllernode.id=1controller.quorum.voters=1@node1:9093,2@node2:9093,3@node3:9093listeners=PLAINTEXT://node1:9092,CONTROLLER://node1:9093inter.broker.listener.name=PLAINTEXTadvertised.listeners=PLAINTEXT://node1:9092controller.listener.names=CONTROLLERlistener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSLnum.network.threads=3num.io.threads=8socket.send.buffer.bytes=102400socket.receive.buffer.bytes=102400socket.request.max.bytes=104857600log.dirs=/data/kafka/datanum.partitions=3num.recovery.threads.per.data.dir=1offsets.topic.replication.factor=3transaction.state.log.replication.factor=3transaction.state.log.min.isr=3log.retention.hours=168log.segment.bytes=1073741824log.retention.check.interval.ms=300000max.request.size=2097152log.cleanup.policy=deleteEOF

cat > /etc/systemd/system/kafka.service << EOF[Unit]Description=Apache Kafka serverAfter=network.target Requires=network.target  [Service]Type=forkingEnvironment="JAVA_HOME=/usr/local/jdk"ExecStart=/data/kafka/bin/kafka-server-start.sh -daemon /data/kafka/config/kraft/server.propertiesExecStop=/data/kafka/bin/kafka-server-stop.shRestart=alwaysRestartSec=10StartLimitInterval=5LimitNOFILE=265535 [Install]WantedBy=multi-user.targetEOF

rsync -avz /data/kafka node2:/datarsync -avz /data/kafka node3:/data # 同步 service rsync -avz  /etc/systemd/system/zookeeper.service  node2:/etc/systemd/system/zookeeper.servicersync -avz  /etc/systemd/system/zookeeper.service  node3:/etc/systemd/system/zookeeper.service rsync -avz /etc/systemd/system/kafka.service node2:/etc/systemd/system/kafka.servicersync -avz /etc/systemd/system/kafka.service node3:/etc/systemd/system/kafka.service

# node1 执行， 前面已经执行过echo 1 > /data/kafka/zookeeper/myid # node2 执行echo 2 > /data/kafka/zookeeper/myid # node3 执行echo 3 > /data/kafka/zookeeper/myid # 修改 kafka 配置文件 server.properties 中的这两行  zk模式修改cd /data/kafka/configvim server.properties# node1  前面修改可以不用改broker.id=0listeners=PLAINTEXT://node1:9092 # node2broker.id=1listeners=PLAINTEXT://node2:9092 # node3broker.id=2listeners=PLAINTEXT://node3:9092  # 修改 kafka 配置文件 server.properties 中的这两行  kraft 模式修改cd /data/kafka/config/kraftvim server.properties # node1 前面修改了node.id=1listeners=PLAINTEXT://node1:9092,CONTROLLER://node1:9093advertised.listeners=PLAINTEXT://node1:9092 # node2 node.id=2listeners=PLAINTEXT://node2:9092,CONTROLLER://node2:9093advertised.listeners=PLAINTEXT://node2:9092 # node3node.id=3listeners=PLAINTEXT://node3:9092,CONTROLLER://node3:9093advertised.listeners=PLAINTEXT://node3:9092

 # 生成 uudicd /data/kafka/bin/./kafka-storage.sh random-uuidMcHv1APORp-y-_gHoIRzBQ  # 这个命令每一台执行./kafka-storage.sh format \-t McHv1APORp-y-_gHoIRzBQ \-c /data/kafka/config/kraft/server.properties

systemctl enable zookeeper.servicesystemctl enable kafka.service

# 启动- 三台先都启动好 zk 后，观察没有问题，在启动 kafkasystemctl start zookeeper.servicesystemctl start kafka.service

[root@node1 config]# jps -l3283 kafka.Kafka1395 org.apache.zookeeper.server.quorum.QuorumPeerMain3711 sun.tools.jps.Jps

#如果没外网可以把镜像本地下载，然后上传到服务器中# 我这里准备了kafka-ui.img.zipunzip kafka-ui.img.zipdocker load -i kafka-ui.img docker run -itd -p 8080:8080 --restart=always -e DYNAMIC_CONFIG_ENABLED=true provectuslabs/kafka-ui:v0.7.1