兜兜    2021-09-22 10:20:24    2021-10-27 15:14:23   

kubernetes k8s nginx ingress
#### 环境介绍 ```sh k8s版本:1.18.20 ingress-nginx: 3.4.0 ``` #### 安装ingress-nginx ##### 下载helm安装包 ```sh $ wget https://github.com/kubernetes/ingress-nginx/releases/download/ingress-nginx-3.4.0/ingress-nginx-3.4.0.tgz $ tar xvf ingress-nginx-3.4.0.tgz $ cd ingress-nginx ``` #### 配置参数 ```sh $ cat > values.yaml <<EOF controller: image: repository: registry.cn-hangzhou.aliyuncs.com/google_containers/nginx-ingress-controller #更好阿里云镜像 tag: "v0.40.1" digest: sha256:abffcf2d25e3e7c7b67a315a7c664ec79a1588c9c945d3c7a75637c2f55caec6 pullPolicy: IfNotPresent runAsUser: 101 allowPrivilegeEscalation: true containerPort: http: 80 https: 443 config: {} configAnnotations: {} proxySetHeaders: {} addHeaders: {} dnsConfig: {} dnsPolicy: ClusterFirst reportNodeInternalIp: false hostNetwork: true #开启主机网络 hostPort: enabled: true #开启主机端口 ports: http: 80 https: 443 electionID: ingress-controller-leader ingressClass: nginx podLabels: {} podSecurityContext: {} sysctls: {} publishService: enabled: true pathOverride: "" scope: enabled: false tcp: annotations: {} udp: annotations: {} extraArgs: {} extraEnvs: [] kind: DaemonSet #DaemonSet运行 annotations: {} labels: {} updateStrategy: {} minReadySeconds: 0 tolerations: [] affinity: {} topologySpreadConstraints: [] terminationGracePeriodSeconds: 300 nodeSelector: ingress: nginx #配置部署选择ingress=nginx节点 livenessProbe: failureThreshold: 5 initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 port: 10254 readinessProbe: failureThreshold: 3 initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 port: 10254 healthCheckPath: "/healthz" podAnnotations: {} #replicaCount: 1 #关闭 minAvailable: 1 resources: requests: cpu: 100m memory: 90Mi autoscaling: enabled: false minReplicas: 1 maxReplicas: 11 targetCPUUtilizationPercentage: 50 targetMemoryUtilizationPercentage: 50 autoscalingTemplate: [] enableMimalloc: true customTemplate: configMapName: "" configMapKey: "" service: enabled: true annotations: {} labels: {} externalIPs: [] loadBalancerSourceRanges: [] enableHttp: true enableHttps: true ports: http: 80 https: 443 targetPorts: http: http https: https type: LoadBalancer nodePorts: http: "" https: "" tcp: {} udp: {} internal: enabled: false annotations: {} extraContainers: [] extraVolumeMounts: [] extraVolumes: [] extraInitContainers: [] admissionWebhooks: enabled: false #关闭 failurePolicy: Fail port: 8443 service: annotations: {} externalIPs: [] loadBalancerSourceRanges: [] servicePort: 443 type: ClusterIP patch: enabled: true image: repository: docker.io/jettech/kube-webhook-certgen tag: v1.3.0 pullPolicy: IfNotPresent priorityClassName: "" podAnnotations: {} nodeSelector: {} tolerations: [] runAsUser: 2000 metrics: port: 10254 enabled: false service: annotations: {} externalIPs: [] loadBalancerSourceRanges: [] servicePort: 9913 type: ClusterIP serviceMonitor: enabled: false additionalLabels: {} namespace: "" namespaceSelector: {} scrapeInterval: 30s targetLabels: [] metricRelabelings: [] prometheusRule: enabled: false additionalLabels: {} rules: [] lifecycle: preStop: exec: command: - /wait-shutdown priorityClassName: "" revisionHistoryLimit: 10 maxmindLicenseKey: "" defaultBackend: enabled: false image: repository: k8s.gcr.io/defaultbackend-amd64 tag: "1.5" pullPolicy: IfNotPresent runAsUser: 65534 extraArgs: {} serviceAccount: create: true name: extraEnvs: [] port: 8080 livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 readinessProbe: failureThreshold: 6 initialDelaySeconds: 0 periodSeconds: 5 successThreshold: 1 timeoutSeconds: 5 tolerations: [] affinity: {} podSecurityContext: {} podLabels: {} nodeSelector: {} podAnnotations: {} replicaCount: 1 minAvailable: 1 resources: {} service: annotations: {} externalIPs: [] loadBalancerSourceRanges: [] servicePort: 80 type: ClusterIP priorityClassName: "" rbac: create: true scope: false podSecurityPolicy: enabled: false serviceAccount: create: true name: imagePullSecrets: [] tcp: {} udp: {} EOF ``` #### 创建命名空间 ```sh $ kubectl create namespace ingress-nginx ``` #### 节点打标签 ```sh $ kubectl label nodes k8s-master1 ingress=nginx $ kubectl label nodes k8s-master2 ingress=nginx $ kubectl label nodes k8s-node1 ingress=nginx ``` #### 安装nginx-ingress ```sh $ helm -n ingress-nginx upgrade -i ingress-nginx . ``` #### 卸载ingress-nginx ```sh $ helm -n ingress-nginx uninstall ingress-nginx ``` #### 测试nginx-ingress #### 部署一个测试nginx服务 ```sh $ cat > nginx-deployment.yml <<EOF apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deploy spec: replicas: 3 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-service spec: selector: app: nginx ports: - protocol: TCP port: 80 targetPort: 80 type: ClusterIP EOF ``` 配置ingress对象 创建TLS证书 ```sh $ kubectl create secret tls shudoon-com-tls --cert=5024509__example.com.pem --key=5024509__example.com.key ``` #### 创建ingress规则 ```sh $ cat >tnginx-ingress.yaml <<EOF apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: kubernetes.io/ingress.class: nginx name: tnginx-ingress spec: rules: - host: tnginx.example.com http: paths: - path: / backend: serviceName: nginx-service servicePort: 80 # This section is only required if TLS is to be enabled for the Ingress tls: - hosts: - tnginx.example.com secretName: shudoon-com-tls EOF ``` 测试 https://tnginx.example.com/ #### 问题 `问题:创建自定义ingress报错:Internal error occurred: failed calling webhook “validate.nginx.ingress.kubernetes.io` 查看策略 ```sh $ kubectl get validatingwebhookconfigurations ``` 删除策略 ```sh $ kubectl delete -A ValidatingWebhookConfiguration ingress-nginx-admission ```
阅读 46 评论 0 收藏 0
阅读 46
评论 0
收藏 0

兜兜    2021-09-15 15:56:38    2021-10-27 15:14:16   

Keepalived LVS ingress
```sh 防火墙端口映射80/443到LVS的VIP对应80/443,LVS负载均衡K8S节点IP的80/443端口。ingress-nginx-controller服务暴露方式通过(HostNetwork:80/443)。实现的效果119.x.x.x:80/443-->172.16.100.99:80/443(LVS VIP)--> 172.16.100.100:80/443,172.16.100.101:80/443,172.16.100.102:80/443 ``` 配置规划 ```sh +----------------+----------------+--------+--------------------------+ | Host | IP | Port | SoftWare | +----------------+----------------+--------+--------------------------+ | LVS01 | 172.16.100.27 | 80/443 | LVS,Keepalived | | LVS02 | 172.16.100.28 | 80/443 | LVS,Keepalived | | RS/k8s-master1 | 172.16.100.100 | 80/443 | ingress-nginx-controller | | RS/k8s-master2 | 172.16.100.101 | 80/443 | ingress-nginx-controller | | RS/k8s-node1 | 172.16.100.102 | 80/443 | ingress-nginx-controller | | VIP | 172.16.100.99 | 80/443 | / | +----------------+----------------+--------+--------------------------+ ``` 安装lvs和keepalived(`172.16.100.27/172.16.100.28`) ```sh $ yum install ipvsadm keepalived -y $ systemctl enable keepavlied ``` 配置keepalived(`172.16.100.27`) ```sh $ cat > /etc/keepalived/keepalived.conf <<EOF ! Configuration File for keepalived global_defs { router_id LVS_27 #route id } vrrp_instance VI_1 { state MASTER #主节点 interface ens192 #网卡 virtual_router_id 51 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 172.16.100.99 } } virtual_server 172.16.100.99 443 { delay_loop 3 lb_algo rr lb_kind DR persistence_timeout 50 protocol TCP real_server 172.16.100.100 443 { weight 1 TCP_CHECK { connect_port 443 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 172.16.100.101 443 { weight 1 TCP_CHECK { connect_port 443 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 172.16.100.102 443 { weight 1 TCP_CHECK { connect_port 443 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } virtual_server 172.16.100.99 80 { delay_loop 3 lb_algo rr lb_kind DR persistence_timeout 50 protocol TCP real_server 172.16.100.100 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 172.16.100.101 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 172.16.100.102 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } EOF ``` 配置keepalived(`172.16.100.28`) ```sh $ cat /etc/keepalived/keepalived.conf ... global_defs { router_id LVS_28 #route id,两台机器配置不一样 } vrrp_instance VI_1 { state BACKUP #备份节点 interface ens192 #网卡名 virtual_router_id 51 priority 99 #优先级 ... ``` 配置RS节点(`172.16.100.100/172.16.100.101/172.16.100.102`) ```sh $ cat >/etc/init.d/lvs_rs.sh <<EOF vip=172.16.100.99 mask='255.255.255.255' dev=lo:1 case $1 in start) echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce ifconfig $dev $vip netmask $mask #broadcast $vip up echo "The RS Server is Ready!" ;; stop) ifconfig $dev down echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce echo "The RS Server is Canceled!" ;; *) echo "Usage: $(basename $0) start|stop" exit 1 ;; esac EOF ``` 启动脚本 ```sh $ chmod +x /etc/init.d/lvs_rs.sh $ /etc/init.d/lvs_rs.sh start ``` ```sh $ ip a #查看lo:1接口VIP是否绑定 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet 172.16.100.99/32 scope global lo:1 valid_lft forever preferred_lft forever ``` 启动keepalived(`172.16.100.27/172.16.100.28`) ```sh $ systemctl start keepavlied ``` 查看VIP是否绑定 ```sh $ ip a 2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:50:56:93:ed:a4 brd ff:ff:ff:ff:ff:ff inet 172.16.100.27/24 brd 172.16.100.255 scope global noprefixroute ens192 valid_lft forever preferred_lft forever inet 172.16.100.99/32 scope global ens192 valid_lft forever preferred_lft forever ``` 查看LVS信息 ```sh $ ipvsadm -L -n IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP 172.16.100.99:80 rr persistent 50 -> 172.16.100.100:80 Route 1 0 0 -> 172.16.100.101:80 Route 1 0 0 -> 172.16.100.102:80 Route 1 0 0 TCP 172.16.100.99:443 rr persistent 50 -> 172.16.100.100:443 Route 1 0 0 -> 172.16.100.101:443 Route 1 0 0 -> 172.16.100.102:443 Route 1 0 0 ```
阅读 105 评论 0 收藏 0
阅读 105
评论 0
收藏 0