荣陶博客

荣陶笔记

0%

高可用多master节点集群

发表于

每个master节点都需要执行命令

允许用户ssh远程登录

注：在root用户修改sshd_config文件

vi /etc/ssh/sshd_config

修改 PasswordAuthentication yes

设置系统主机名以及 Host 文件的相互解析

1
2
3
4
# 不同机器rt-master11需要不同
hostnamectl set-hostname rt-master11
# 将配置复制到各个节点
scp /etc/hosts root@192.168.180.31:/etc/hosts

安装依赖包

1
yum install -y iptables ipset wget ipvsadm curl wget net-tools

设置防火墙为 Iptables 并设置空规则

1
2
systemctl stop firewalld && systemctl disable firewalld
yum -y install iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save

关闭 虚拟内存 及 SELINUX

1
2
3
4
5
6
7
# 关闭虚拟内存
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 关闭 SELINUX
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
reboot
# 确认selinux已经被禁用
getenforce

创建安装目录

1
mkdir ~/install-k8s ~/install-k8s/core ~/install-k8s/pluging ~/install-k8s/pluging/flannel

调整内核参数，对于 K8S

在~/install-k8s/core创建

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
cat > kubernetes.conf <<EOF 
net.bridge.bridge-nf-call-iptables=1 
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它 
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM 
fs.inotify.max_user_instances=8192 
fs.inotify.max_user_watches=1048576 
fs.file-max=52706963 
fs.nr_open=52706963 
net.ipv6.conf.all.disable_ipv6=1 
net.netfilter.nf_conntrack_max=2310720 
EOF

modprobe br_netfilter模块开机加载

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
vi /etc/rc.sysinit
# 添加以下文本
#!/bin/bash
for file in /etc/sysconfig/modules/*.modules ; do
[ -x $file ] && $file
done
# 添加启动模块
cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF 
modprobe br_netfilter
modprobe ip_conntrack
EOF
# 检查启动模块
vi /etc/sysconfig/modules/br_netfilter.modules
# 增加权限
chmod 755 /etc/sysconfig/modules/br_netfilter.modules
reboot
# 重启后 模块自动加载
lsmod |grep br_netfilter

复制配置到开机启动

1
2
3
4
modprobe br_netfilter
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
# 启动配置
sysctl -p /etc/sysctl.d/kubernetes.conf

调整系统时区

1
2
3
4
5
6
7
# 设置系统时区为 中国/上海
timedatectl set-timezone Asia/Shanghai
# 将当前的 UTC 时间写入硬件时钟 
timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务
systemctl restart rsyslog
systemctl restart crond

关闭系统不需要服务

1
systemctl stop postfix && systemctl disable postfix

使用Vagrant设置虚拟机，设置默认网卡

1
2
3
4
5
6
7
8
9
route add default gw 192.168.180.1
[root@rt-master21 hightUse]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.180.1   0.0.0.0         UG    0      0        0 eth1
0.0.0.0         10.0.2.2        0.0.0.0         UG    100    0        0 eth0
10.0.2.0        0.0.0.0         255.255.255.0   U     100    0        0 eth0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.180.0   0.0.0.0         255.255.255.0   U     101    0        0 eth1

设置 rsyslogd 和 systemd journald

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# 持久化保存日志的目录
mkdir /var/log/journal 
mkdir /etc/systemd/journald.conf.d

cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF 
[Journal]
# 持久化保存到磁盘
Storage=persistent
# 压缩历史日志 
Compress=yes
SyncIntervalSec=5m 
RateLimitInterval=30s 
RateLimitBurst=1000
# 最大占用空间 10G 
SystemMaxUse=10G
# 单日志文件最大 200M 
SystemMaxFileSize=200M
# 日志保存时间 2 周 
MaxRetentionSec=2week
# 不将日志转发到 syslog 
ForwardToSyslog=no
EOF

# 启动日志记录服务
systemctl restart systemd-journald

关闭 NUMA

1
2
3
4
5
6
7
8
9
10
11
12
# 备份默认文件
cp /etc/default/grub{,.bak}
# 在 GRUB_CMDLINE_LINUX 一行添加 `numa=off` 参数
vi /etc/default/grub
# 检查
# diff /etc/default/grub.bak /etc/default/grub
# 6c6
# < GRUB_CMDLINE_LINUX="no_timer_check console=tty0 console=ttyS0,115200n8 net.ifnames=0 biosdevname=0 elevator=noop crashkernel=auto"
# ---
# > GRUB_CMDLINE_LINUX="no_timer_check console=tty0 console=ttyS0,115200n8 net.ifnames=0 biosdevname=0 elevator=noop crashkernel=auto numa=off"
cp /boot/grub2/grub.cfg{,.bak}
grub2-mkconfig -o /boot/grub2/grub.cfg

kube-proxy开启ipvs的前置条件

1
2
3
4
5
6
7
8
9
10
modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules <<EOF 
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

安装 Docker 软件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 安装yum工具
yum install -y yum-utils
# 设置稳定的存储库
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
# 安装docker
yum install -y docker-ce
# 配置 daemon
mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF 
{
  "registry-mirrors": ["https://77xsmdni.mirror.aliyuncs.com/"],
  "exec-opts": ["native.cgroupdriver=systemd"], 
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m" 
  }
}
EOF

systemctl daemon-reload
# 启动docker服务
systemctl start docker && systemctl enable docker

在主节点启动 Haproxy 与 Keepalived 容器

创建文件夹~/install-k8s/hightUse

创建haproxy.cfg配置文件

1
2
mkdir -p data/lb/etc
vi data/lb/etc/haproxy.cfg

逐个进行负载，避免节点没启动而报错，先负载180.21的节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon

defaults
    log     global
    mode    http
    option  httplog
    option  dontlognull
    retries 3
    option redispatch
    timeout connect  5000
    timeout client  50000
    timeout server  50000

frontend stats-front
  bind *:8081
  mode http
  default_backend stats-back

frontend fe_k8s_6444
  bind *:6444
  mode tcp
  timeout client 1h
  log global
  option tcplog
  default_backend be_k8s_6443
  acl is_websocket hdr(Upgrade) -i WebSocket
  acl is_websocket hdr_beg(Host) -i ws

backend stats-back
  mode http
  balance roundrobin
  stats uri /haproxy/stats
  stats auth pxcstats:secret

backend be_k8s_6443
  mode tcp
  timeout queue 1h
  timeout server 1h
  timeout connect 1h
  log global
  balance roundrobin
  server rancher01 192.168.180.21:6443

创建启动haproxy的脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
#!/bin/bash
MasterIP1=192.168.180.11
MasterIP2=192.168.180.21
MasterIP3=192.168.180.31
MasterPort=6443

docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
        -e MasterIP1=$MasterIP1 \
        -e MasterIP2=$MasterIP2 \
        -e MasterIP3=$MasterIP3 \
        -e MasterPort=$MasterPort \
        -v /root/install-k8s/hightUse/data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \
        wise2c/haproxy-k8s

下载haproxy镜像

1
2
3
4
5
# 每个主节点都需要
docker pull wise2c/haproxy-k8s
# 运行haproxy镜像
chmod a+x data/lb/start-haproxy.sh
./data/lb/start-haproxy.sh

启动keepalived

vi /root/install-k8s/hightUse/data/lb/start-keepalived.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/bin/bash
VIRTUAL_IP=192.168.180.100
INTERFACE=eth1
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18

docker run -itd --restart=always --name=Keepalived-K8S \
        --net=host --cap-add=NET_ADMIN \
        -e VIRTUAL_IP=$VIRTUAL_IP \
        -e INTERFACE=$INTERFACE \
        -e CHECK_PORT=$CHECK_PORT \
        -e RID=$RID \
        -e VRID=$VRID \
        -e NETMASK_BIT=$NETMASK_BIT \
        -e MCAST_GROUP=$MCAST_GROUP \
        wise2c/keepalived-k8s

运行keepalived镜像

1
2
chmod a+x data/lb/start-keepalived.sh
./data/lb/start-keepalived.sh

安装 Kubeadm (主从配置)

1
2
3
4
5
6
7
8
9
10
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes 
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 
enabled=1
gpgcheck=0
repo_gpgcheck=0 
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg 
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg 
EOF

安装k8s

1
2
3
4
5
6
7
8
9
10
11
yum -y install kubeadm kubectl kubelet
systemctl enable kubelet.service
# 设置节点ip，处理node的INTERNAL-IP为10.0.2.15
vi /etc/sysconfig/kubelet
# 修改为以下内容
KUBELET_EXTRA_ARGS="--node-ip=192.168.180.21"
# 重启服务
systemctl restart kubelet.service
# 使kubectl可以自动补充
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

创建k8s初始化文件

1
kubeadm config print init-defaults > kubeadm-config.yaml

修改以下内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 节点ip地址
advertiseAddress: 192.168.180.21
# 镜像仓库
imageRepository: registry.aliyuncs.com/google_containers
# 高可用多master节点需要配置改vip地址, 在clusterName下配置
controlPlaneEndpoint: "192.168.180.100:6444"
# k8s版本
kubernetesVersion: v1.18.3
# 在networking:下添加 podSubnet: "10.244.0.0/16" ，因为这是网络插件的默认网段
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"
  serviceSubnet: 10.96.0.0/12
  
# 将k8s代理改为ipvs, 最后一行添加
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

集群初始化（注：需要在12小时内完成整个集群部署，否则证书会过期）

1
2
3
4
5
6
7
8
9
10
11
12
13
kubeadm init --config=kubeadm-config.yaml --upload-certs | tee kubeadm-init.log
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 添加master节点
kubeadm join 192.168.180.100:6444 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:04d5aa7619d1ebe925a9ac5f7ad8d69dc511d18ee158c40f737837692b3bb41a \
    --control-plane --certificate-key fcc0f31d68fff8886666649f3363ae888e02915bdaea213a8d74d9add061893a
    
# 添加工作节点
kubeadm join 192.168.180.100:6444 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:440a02125afb176cb517893deda010b2252c041973cdd4ea01bc6ba712d7b280

将配置文件复制到其他master节点

1
2
scp -r  ~/install-k8s/hightUse/* root@192.168.180.11:/root/install-k8s/hightUse
scp -r  ~/install-k8s/hightUse/* root@192.168.180.31:/root/install-k8s/hightUse

启动其他master节点

在文件夹~/install-k8s/hightUse下运行

1
2
3
4
chmod a+x data/lb/start-haproxy.sh
./data/lb/start-haproxy.sh
chmod a+x data/lb/start-keepalived.sh
./data/lb/start-keepalived.sh

安装k8s（同上）

将master节点加入到主master节点

1
2
3
kubeadm join 192.168.180.100:6444 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:440a02125afb176cb517893deda010b2252c041973cdd4ea01bc6ba712d7b280 \
    --control-plane --certificate-key e6fe032d63e01c1c1a61824b46b4a6e28535fd34ea527656a409ec65433f126a

每个节点都启动后，重启HAProxy-K8S

修改vi ~/install-k8s/hightUse/data/lb/etc/haproxy.cfg

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon

defaults
    log     global
    mode    http
    option  httplog
    option  dontlognull
    retries 3
    option redispatch
    timeout connect  5000
    timeout client  50000
    timeout server  50000

frontend stats-front
  bind *:8081
  mode http
  default_backend stats-back

frontend fe_k8s_6444
  bind *:6444
  mode tcp
  timeout client 1h
  log global
  option tcplog
  default_backend be_k8s_6443
  acl is_websocket hdr(Upgrade) -i WebSocket
  acl is_websocket hdr_beg(Host) -i ws

backend stats-back
  mode http
  balance roundrobin
  stats uri /haproxy/stats
  stats auth pxcstats:secret

backend be_k8s_6443
  mode tcp
  timeout queue 1h
  timeout server 1h
  timeout connect 1h
  log global
  balance roundrobin
  server rancher01 192.168.180.21:6443
  server rancher02 192.168.180.11:6443
  server rancher03 192.168.180.31:6443

重启HAProxy-K8S容器

1
2
3
4
docker rm -f HAProxy-K8S && bash /root/install-k8s/hightUse/data/lb/start-haproxy.sh

# 若连接不上180.100，可重启keepalived
# docker rm -f Keepalived-K8S && bash /root/install-k8s/hightUse/data/lb/start-keepalived.sh

将配置覆盖其他master节点的配置，并重启HAProxy-K8S容器

1
2
scp ~/install-k8s/hightUse/data/lb/etc/haproxy.cfg root@192.168.180.31:/root/install-k8s/hightUse/data/lb/etc/haproxy.cfg
scp ~/install-k8s/hightUse/data/lb/etc/haproxy.cfg root@192.168.180.11:/root/install-k8s/hightUse/data/lb/etc/haproxy.cfg

Etcd 集群状态查看

1
2
3
[root@rt-master21 hightUse]$ kubectl -n kube-system exec etcd-rt-master21 -- etcdctl --endpoints=https://192.168.180.21:2379,https://192.168.180.11:2379,https://192.168.180.31:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key endpoint health
kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml
kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml

处理某个master节点关闭后，无法使用kubectl

vi ~/.kube/config

1
2
# 修改server: https://192.168.180.100:6443 为本机ip
server: https://192.168.180.11:6443

K8S重新加入MASTER节点，避免ETCD错误

我们有时候会有删除，再重新加入master节点的需求，比如master机器改名。这里注意重新加入时，经常会出现etcd报错，如下

1
[check-etcd] Checking that the etcd cluster is healthy

删除etcd信息

在master01节点上执行命令，进入etcd的容器里

1
2
3
4
5
6
7
[root@rt-master21 vagrant]# kubectl exec -it etcd-rt-master21 sh -n kube-system
#输入命令
etcdctl --endpoints 127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key member list
# bf50d6c7bfb98b43, started, rt-master21, https://192.168.180.21:2380, https://192.168.180.21:2379, false
# da4f3d6e4d869225, started, rt-master11, https://10.0.2.15:2380, https://10.0.2.15:2379, false
# 因为我们的rt-master11机器对应的hash是da4f3d6e4d869225。我们下一步就是根据hash删除etcd信息，执行如下命令
etcdctl --endpoints 127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key member remove da4f3d6e4d869225

获取master加入的token

1
2
3
4
5
6
kubeadm init phase upload-certs --upload-certs
# b = 2f530afd934070166d4520badf1c86c56d018358e018d6e7d97ef5f08fef9d4f
kubeadm token create --print-join-command
# a = kubeadm join 192.168.180.100:6444 --token 2rmyg7.qorrwlcux6pz3r0s     --discovery-token-ca-cert-hash sha256:418f932dd8fe8d516f980980285688e5bc299e746f9697c94638aafd23814eca
# 加入master的命令为 a + --control-plane --certificate-key + b
kubeadm join 192.168.180.100:6444 --token 2rmyg7.qorrwlcux6pz3r0s     --discovery-token-ca-cert-hash sha256:418f932dd8fe8d516f980980285688e5bc299e746f9697c94638aafd23814eca --control-plane --certificate-key 2f530afd934070166d4520badf1c86c56d018358e018d6e7d97ef5f08fef9d4f