Openresty + keepalived 实现负载均衡 + IPV6 验证
·
文章目录
1. 准备工作
环境
| IP | software |
|---|---|
| 192.168.10.30 | keepalived-MASTER，openresty，web1 |
| 192.168.10.40 | keepalived-BACKUP，openresty，web2 |
| 192.168.10.73 | VIP |
两台机器同步同步时间
yum -y install ntpdate
ntpdate ntp1.aliyun.com
# 如果时间未进行同步执行
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
2. 安装 docker
curl http://49.232.8.65/shell/docker/docker.sh | bash
3. docker 运行 tomcat 容器
两个节点各运行一个 tomcat 容器
docker pull tomcat
docker run -itd --name tomcat-01/tomcat-02 -p 8888:8080 tomcat
tomcat 访问 404 解决：版本太高的原因，我这是 10.x
docker exec -it gifted_faraday bash
cp -r ./webapps.dist/* ./webapps
exit
浏览器再次访问。
配置修改
mkdir /usr/local/tomcat/webapps/web_demo
# 修改动态页面内容
cat > /usr/local/tomcat/webapps/web_demo/index.jsp <<EOF
<%@ page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
<% out.println("动态页面 1,http://www.tomcat01.com");%>
</body>
</html>
EOF
###编辑 tomcat 主配置文件，添加虚拟主机配置，这里要先删掉原先的主机名等配置
vim /usr/local/tomcat/conf/server.xml
#找到这个原配置删掉，否则重启 tomcat 会报错
<Host name="localhost" appBase="webapps" unpackWARs="true" autoDeploy="true"
#在行尾 162 行处插入下面配置，注意 Host 位置，看下面的示例图
<Host name="localhost" appBase="webapps" unpackWARs="true" autoDeploy="true" xmlValidation="false" xmlNamespaceAware="false">
<Context docBase="/usr/local/tomcat/webapps/web_demo" path="" reloadable="true" />
</Host>
#重启 tomcat
shutdown.sh # 容器内
docker start tomcat-01 # 宿主机
浏览器再次访问两个链接，应该f分别显示：
动态页面 1,http://www.tomcat01.com
动态页面 2,http://www.tomcat02.com
4. 安装 openresty 服务
两个节点
文件
[root@tengxun-02 /data/openresty]#ls
Dockerfile start.sh
[root@tengxun-02 /data/openresty]#cat start.sh
#!/bin/bash
/usr/local/openresty/nginx/sbin/nginx
[root@tengxun-02 /data/openresty]#cat Dockerfile
FROM centos:centos7
MAINTAINER zc
ENV TIME_ZOME=Asia/Shanghai
WORKDIR /usr/local/openresty
RUN yum -y install readline-devel pcre-devel openssl-devel zlib-devel vim gcc gcc-c++ perl make kernel-headers kernel-devel curl wget postgresql-devel &> /dev/null && \
wget https://openresty.org/download/openresty-1.21.4.1.tar.gz &> /dev/null && \
tar zxvf openresty-1.21.4.1.tar.gz &> /dev/null && \
cd openresty-1.21.4.1/ &> /dev/null && \
./configure &> /dev/null && \
make -j 4 &> /dev/null && make install &> /dev/null && \
ln -s /usr/local/openresty/nginx/sbin/nginx /usr/local/sbin/ &> /dev/null
ENV PATH=/usr/local/openresty/nginx/sbin:$PATH
ENV export PATH
EXPOSE 80
RUN echo "daemon off;">>/usr/local/openresty/nginx/conf/nginx.conf
workdir /
ADD start.sh /start.sh
RUN chmod 755 /start.sh
CMD ["/start.sh"]
制作 openresty 镜像
[root@tengxun-02 /data/openresty]#docker build -t openresty:1.21.4.1 .
......
[root@tengxun-02 /data/openresty]#docker images | grep open
openresty 1.21.4.1 0d1dd3c10e13 46 seconds ago 780MB
运行
[root@tengxun-02 /data/openresty]#docker run -itd --name openresty_1.21.4.1_20220724 -p 2000:80 openresty:1.21.4.1
......
[root@tengxun-02 /data/openresty]#docker ps -a | grep open
f7782a2ded4f openresty:1.21.4.1 "/start.sh" 14 seconds ago Up 14 seconds 0.0.0.0:2000->80/tcp openresty_1.21.4.1_20220724
# ------------------------------------------------------
/usr/local/openresty/nginx/html/index.html
/usr/local/openresty/nginx/logs
/usr/local/openresty/nginx/conf/nginx.conf
修改配置文件
[root@c7-2 ~]#docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
5a1da877780a openresty:1.21.4.1 "/start.sh" 9 seconds ago Up 8 seconds 0.0.0.0:2000->80/tcp, :::2000->80/tcp openresty_1.21.4.1_20220724
f470233bc199 tomcat "catalina.sh run" 27 minutes ago Up 14 minutes 0.0.0.0:8888->8080/tcp, :::8888->8080/tcp tomcat-01
[root@c7-2 ~]#docker exec -it openresty_1.21.4.1_20220724 bash
[root@5a1da877780a /]# ls
anaconda-post.log bin dev etc home lib lib64 media mnt opt proc root run sbin srv start.sh sys tmp usr var
[root@5a1da877780a /]# cd /usr/local/openresty/nginx/conf/
[root@5a1da877780a conf]# vim nginx.conf
......
#配置负载均衡的服务器列表，weight 参数表示权重，权重越高，被分配到的概率越大
#gzip on; # 33 行下面加入以下内容
upstream tomcat_server {
server 192.168.10.30:8888 weight=1;
server 192.168.10.40:8888 weight=1;
}
......
#access_log logs/host.access.log main; # 45 行下加入下面配置参数
location ~ .*\.jsp$ {
proxy_pass http://tomcat_server;
proxy_set_header HOST $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
......
[root@5a1da877780a conf]# nginx -t
nginx: the configuration file /usr/local/openresty/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/openresty/nginx/conf/nginx.conf test is successful
[root@5a1da877780a conf]# nginx -s reload
[root@5a1da877780a conf]# exit
exit
访问测试 openresty 是否能代理后端服务 tomcat，且轮询访问
访问 10.30 节点
[root@c7-2 ~]#curl http://192.168.10.30:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 2,http://www.tomcat02.com
</body>
</html>
[root@c7-2 ~]#curl http://192.168.10.30:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 1,http://www.tomcat01.com
</body>
</html>
访问 10.40 节点
[10:41:56 root@c7-3~]#curl http://192.168.10.40:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 1,http://www.tomcat01.com
</body>
</html>
[10:42:08 root@c7-3~]#curl http://192.168.10.40:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 2,http://www.tomcat02.com
</body>
</html>
5. 安装 keepalived 服务
两台机器
yum -y install keepalived vim lsof wget curl lrzsz
备份配置文件
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf{.`date +%F`}
修改配置文件
MASTER
vim /etc/keepalived/keepalived.conf
# 检查 nginx 是否挂掉的脚本，需要自己编写脚本逻辑
vrrp_script chk_nginx {
script "/home/keepalived/nginx_check.sh"
interval 2
weight -20
}
#一个 vrrp_instance 就是定义一个虚拟路由器的，实例名称
vrrp_instance VI_1_NGINX {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.73/24 # VIP 地址
}
}
BACKUP
vim /etc/keepalived/keepalived.conf
# 检查 nginx 是否挂掉的脚本，需要自己编写脚本逻辑
vrrp_script chk_nginx {
script "/home/keepalived/nginx_check.sh"
interval 2
weight -20
}
vrrp_instance VI_1_NGINX {
state BACKUP
interface ens33
virtual_router_id 51
priority 90
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.73/24 # VIP 地址
}
}
nginx 服务心跳检测
两次检测服务是否存活，nginx 服务停止则关闭 keepalived 服务，防止 keepalived 脑裂状况的产生。
mkdir -p /home/keepalived/
vim /home/keepalived/nginx_check.sh
#!/bin/bash
nginxpid=$(ps -C nginx --no-header|wc -l)
if [ $nginxpid -eq 0 ];then
docker start openresty_1.21.4.1_20220724
sleep 3
nginxpid=$(ps -C nginx --no-header|wc -l)
if [ $nginxpid -eq 0 ];then
systemctl stop keepalived
fi
fi
chmod +x /home/keepalived/nginx_check.sh
#---------------------其他写法-----------------------------------
root@10.45.80.25[/root]#cat /usr/local/bin/chk_haproxy.sh
#!/bin/bash
if [ $(ps -C haproxy --no-header | wc -l) -ne 0 ]; then
exit 0
else
#restart keepalived to evict vip
exit 1
fi
启动 keepalived：两台主机都启动
systemctl daemon-reload
systemctl enable keepalived
systemctl start keepalived
# -------------------------------------------------------
[root@c7-2 ~]#systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
Active: active (running) since Sun 2022-07-24 11:46:01 CST; 14s ago
Process: 39493 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 39495 (keepalived)
Tasks: 3
Memory: 7.5M
CGroup: /system.slice/keepalived.service
├─39495 /usr/sbin/keepalived -D
├─39496 /usr/sbin/keepalived -D
└─39497 /usr/sbin/keepalived -D
Jul 24 11:46:03 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:03 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:03 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:03 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:08 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:08 c7-2 Keepalived_vrrp[39497]: VRRP_Instance(VI_1_NGINX) Sending/queueing gratuitous ARPs on ens33 for 192.168.10.73
Jul 24 11:46:08 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:08 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:08 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
Jul 24 11:46:08 c7-2 Keepalived_vrrp[39497]: Sending gratuitous ARP on ens33 for 192.168.10.73
查看日志
tail -f /var/log/messages
查看 VIP 绑定
[root@c7-2 ~]#ip a s ens33 # 可以看到 VIP 在 192.168.10.30 这台主机上
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:c4:cf:09 brd ff:ff:ff:ff:ff:ff
inet 192.168.10.30/24 brd 192.168.10.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.10.73/24 scope global secondary ens33
valid_lft forever preferred_lft forever
inet6 fe80::fd95:f79c:11fe:c64d/64 scope link noprefixroute
valid_lft forever preferred_lft forever
6. 测试访问 VIP
[root@c7-2 ~]#curl http://192.168.10.73:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 1,http://www.tomcat01.com
</body>
</html>
[root@c7-2 ~]#curl http://192.168.10.73:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 2,http://www.tomcat02.com
</body>
</html>
[root@c7-2 ~]#curl http://192.168.10.73:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 1,http://www.tomcat01.com
</body>
</html>
[root@c7-2 ~]#curl http://192.168.10.73:2000/index.jsp
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 2,http://www.tomcat02.com
</body>
</html>
7. 使用 IPV6 验证是否可以通信
修改 Keepalived 服务：将 vip 换成 ipv6 格式：两台机器一样
[root@c7-2 ~]#vi /etc/keepalived/keepalived.conf
# 检查 nginx 是否挂掉的脚本，需要自己编写脚本逻辑
vrrp_script chk_nginx {
script "/home/keepalived/nginx_check.sh"
interval 2
weight -20
}
#一个 vrrp_instance 就是定义一个虚拟路由器的，实例名称
vrrp_instance VI_1_NGINX {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
# 192.168.10.73/24 # VIP 地址
2001:fecc:0:622::a/64
}
}
[root@c7-2 ~]#systemctl daemon-reload
[root@c7-2 ~]#systemctl restart keepalived
查看网卡
[root@c7-2 ~]#ip a s ens33
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:c4:cf:09 brd ff:ff:ff:ff:ff:ff
inet 192.168.10.30/24 brd 192.168.10.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 2001:fecc:0:622::a/64 scope global nodad
valid_lft forever preferred_lft forever
inet6 fe80::fd95:f79c:11fe:c64d/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[15:42:35 root@c7-3~]#ip a s ens33
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:2b:52:fc brd ff:ff:ff:ff:ff:ff
inet 192.168.10.40/24 brd 192.168.10.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::830f:61b3:ffd5:7233/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@c7-2 ~]#ping6 -I ens33 -c 3 2001:fecc:0:622::a
PING 2001:fecc:0:622::a(2001:fecc:0:622::a) from 2001:fecc:0:622::a ens33: 56 data bytes
64 bytes from 2001:fecc:0:622::a: icmp_seq=1 ttl=64 time=0.067 ms
64 bytes from 2001:fecc:0:622::a: icmp_seq=2 ttl=64 time=0.106 ms
64 bytes from 2001:fecc:0:622::a: icmp_seq=3 ttl=64 time=0.050 ms
--- 2001:fecc:0:622::a ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2003ms
rtt min/avg/max/mdev = 0.050/0.074/0.106/0.024 ms
- Linux 可能不支持，需要修改 IPV6 配置
浏览器访问：http://[IPV6地址]:2000/index.jsp
curl 访问：
curl -i -g -d -k -v http://[IPV6地址]:2000/index.jsp
# cmd 访问，Linux 里不知道怎么回事无法访问
# cmd 复制方法：单击顶部的空白处，然后点击编辑下面的标记，框选复制的内容，ctrl + c
C:\Users\15205>curl -i -g -d -k -v http://[fe80::54b5:65e0:690:9a7e]:2000/index.jsp
* Trying fe80::54b5:65e0:690:9a7e:2000...
* Connected to fe80::54b5:65e0:690:9a7e (fe80::54b5:65e0:690:9a7e) port 2000 (#0)
> POST /index.jsp HTTP/1.1
> Host: [fe80::54b5:65e0:690:9a7e]:2000
> User-Agent: curl/7.83.1
> Accept: */*
> Content-Length: 2
> Content-Type: application/x-www-form-urlencoded
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200
HTTP/1.1 200
< Server: openresty/1.21.4.1
Server: openresty/1.21.4.1
< Date: Sun, 24 Jul 2022 08:33:05 GMT
Date: Sun, 24 Jul 2022 08:33:05 GMT
< Content-Type: text/html;charset=UTF-8
Content-Type: text/html;charset=UTF-8
< Content-Length: 121
Content-Length: 121
< Connection: keep-alive
Connection: keep-alive
< Set-Cookie: JSESSIONID=DF50139C7209399F1C960597CAC6C175; Path=/; HttpOnly
Set-Cookie: JSESSIONID=DF50139C7209399F1C960597CAC6C175; Path=/; HttpOnly
<
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 1,http://www.tomcat01.com
</body>
</html>
* Connection #0 to host fe80::54b5:65e0:690:9a7e left intact
C:\Users\15205>curl -i -g -d -k -v http://[fe80::54b5:65e0:690:9a7e]:2000/index.jsp
* Trying fe80::54b5:65e0:690:9a7e:2000...
* Connected to fe80::54b5:65e0:690:9a7e (fe80::54b5:65e0:690:9a7e) port 2000 (#0)
> POST /index.jsp HTTP/1.1
> Host: [fe80::54b5:65e0:690:9a7e]:2000
> User-Agent: curl/7.83.1
> Accept: */*
> Content-Length: 2
> Content-Type: application/x-www-form-urlencoded
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200
HTTP/1.1 200
< Server: openresty/1.21.4.1
Server: openresty/1.21.4.1
< Date: Sun, 24 Jul 2022 08:33:12 GMT
Date: Sun, 24 Jul 2022 08:33:12 GMT
< Content-Type: text/html;charset=UTF-8
Content-Type: text/html;charset=UTF-8
< Content-Length: 121
Content-Length: 121
< Connection: keep-alive
Connection: keep-alive
< Set-Cookie: JSESSIONID=45720079D67322F16C4CF9F3DF1BC750; Path=/; HttpOnly
Set-Cookie: JSESSIONID=45720079D67322F16C4CF9F3DF1BC750; Path=/; HttpOnly
<
<html>
<head>
<title>JSP tomcat02 page </title>
</head>
<body>
动态页面 2,http://www.tomcat02.com
</body>
</html>
* Connection #0 to host fe80::54b5:65e0:690:9a7e left intact
浏览器访问
北京人形旗下天工造物具身智能开源社区，聚焦具身天工与慧思开物两大平台
更多推荐
1
0- 0
已为社区贡献3条内容
所有评论(0)