```sh
高可用K8S ApiServer服务，通过nginx的stream做四层负载均衡，nginx高可用通过keepalived实现。实现的效果172.16.100.111:16433--> 172.16.100.100:6433/172.16.100.101:6433
```
配置规划
```sh
+-------------+----------------+-------+----------------------------+
| Host | IP | Port | SoftWare |
+-------------+----------------+-------+----------------------------+
| k8s-master1 | 172.16.100.100 | 6433 | Nginx,Keepalived,ApiServer |
| k8s-master2 | 172.16.100.101 | 6433 | Nginx,Keepalived,ApiServer |
| VIP | 172.16.100.111 | 16433 | / |
+-------------+----------------+-------+----------------------------+
```
主从LVS节点
```sh
$ yum install nginx nginx-mod-stream keepalived -y #nginx-mod-stream 四层负载均衡stream模块
```
配置主从节点nginx
```sh
$ cat /etc/nginx/nginx.conf
...
events {
worker_connections 1024;
}
# 四层负载均衡，为两台Master apiserver组件提供负载均衡
stream {
log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
access_log /var/log/nginx/k8s-access.log main;
upstream k8s-apiserver {
server 172.16.100.100:6443; # Master1 APISERVER IP:PORT
server 172.16.100.101:6443; # Master1 APISERVER IP:PORT
}
server {
listen 16443; # 由于nginx与master节点复用，这个监听端口不能是6443，否则会冲突
proxy_pass k8s-apiserver;
}
}
...
```
配置主节点keepalived
```sh
$cat > /etc/keepalived/keepalived.conf <<EOF
global_defs {
router_id keepalived_100
}
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_1 {
state MASTER
interface ens192 # 修改为实际网卡名
virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的
priority 100 # 优先级，备服务器设置 90
advert_int 1 # 指定VRRP 心跳包通告间隔时间，默认1秒
authentication {
auth_type PASS
auth_pass 1111
}
# 虚拟IP
virtual_ipaddress {
172.16.100.111/24
}
track_script {
check_nginx
}
}
EOF
```
nginx检查脚本
```sh
$ cat >/etc/keepalived/check_nginx.sh <<EOF
#!/bin/bash
count=$(ss -antp |grep 16443 |egrep -cv "grep|$$")
if [ "$count" -eq 0 ];then
exit 1
else
exit 0
fi
EOF
```
```sh
systemctl restart nginx
systemctl enable nginx
systemctl restart keepalived
systemctl enable keepalived
```
配置从nginx节点keepalived
```sh
$ cat >/etc/keepalived/keepalived.conf <<EOF
global_defs {
router_id keepalived_101
}
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_1 {
state BACKUP
interface ens192 # 修改为实际网卡名
virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的
priority 90 # 优先级，备服务器设置 90
advert_int 1 # 指定VRRP 心跳包通告间隔时间，默认1秒
authentication {
auth_type PASS
auth_pass 1111
}
# 虚拟IP
virtual_ipaddress {
172.16.100.111/24
}
track_script {
check_nginx
}
}
EOF
```
主从启动nginx和keepalived
```sh
systemctl restart nginx
systemctl enable nginx
systemctl restart keepalived
systemctl enable keepalived
```
查看vip是否绑定
```sh
$ ip a
2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:50:56:93:6a:3a brd ff:ff:ff:ff:ff:ff
inet 172.16.100.101/24 brd 172.16.100.255 scope global noprefixroute ens192
valid_lft forever preferred_lft forever
inet 172.16.100.111/32 scope global ens192
```
测试
```sh
$ curl http://172.16.100.111
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx</center>
</body>
</html>
```
