kubernetes 高可用部署方法笔记

#主机名解析
vim /etc/hosts
#添加内容

192.168.17.101 master1
192.168.17.102 master2
192.168.17.103 master3
192.168.17.104 node1
192.168.17.105 node2

#启动chrony服务
systemctl start chronyd
#开机自启
systemctl enable chronyd
#date

systemctl stop firewalld && systemctl disable firewalld
systemctl stop iptables && systemctl disable iptables

vim /etc/selinux/config

vim /etc/fstab
#/dev/mapper/centos-swap swap 

vim /etc/sysctl.d/kubernetes.conf

net.bridge.bridge-nf-call-ip6tables  = 1
net.bridge.bridge-nf-call-iptables  = 1 
net.ipv4.ip_forward =1

#重新加载配置
sysctl -p
#加载网桥过滤模块
modprobe br_netfilter
#查看网桥过滤模块是否加载成功
lsmod | grep br_netfilter

yum install ipset ipvsadm -y

cat <<EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

 #为脚本添加执行权限
chmod +x /etc/sysconfig/modules/ipvs.modules
#执行脚本
/bin/bash /etc/sysconfig/modules/ipvs.modules
#查看模块是否加载成功
lsmod |grep -e ip_vs -e nf_conntrack_ipv4

reboot

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

#查看当前镜像源中支持的docker版本
yum list docker-ce --showduplicates
#安装最新版本docker-ce
yum install  docker-ce

#添加配置文件
mkdir /etc/docker
cat <<EOF > /etc/docker/daemon.json
{
   "registry-mirrors": ["https://eeu0s9el.mirror.aliyuncs.com"],
   "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

#启动docker，设置开机自启
systemctl restart docker
systemctl enable docker

#检查版本
docker version

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg 
　　　　 https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubelet kubeadm kubectl

# 为了实现Docker使用的cgroup drvier和kubelet使用的cgroup drver一致，建议修改"/etc/sysconfig/kubelet"文件的内容：
vim /etc/sysconfig/kubelet
# 修改为以下内容
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
KUBE_PROXY_MODE="ipvs"

# 设置为开机启动即可
systemctl enable kubelet

kubeadm config images list

images=( 
     kube-apiserver:v1.22.2
     kube-controller-manager:v1.22.2
     kube-scheduler:v1.22.2
     kube-proxy:v1.22.2
     pause:3.5
     etcd:3.5.0-0
)

for imageName in ${images[@]} ; do
   docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
   docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName k8s.gcr.io/$imageName
   docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
done

docker pull coredns/coredns:1.8.4
docker images
docker tag coredns/coredns:1.8.4 k8s.gcr.io/coredns/coredns:v1.8.4
docker rmi coredns/coredns:1.8.4
docker images

yum install -y keepalived

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived

global_defs {        #全局定义部分
   router_id master #Keepalived服务器的路由标识，自定义
}
vrrp_script check-haproxy { #定义vrrp脚本
    #检查所在节点的 haproxy 进程是否正常。如果异常则将权重减少3
    script "killall -0 haproxy" #要执行的命令
    interval 5                  #脚本调用时间
    weight -3                   #根据权重值调整vrrp实例的优先级，默认为0
    fall 10                     #需要失败多少次，vrrp才进行角色状态切换(从正常切换为fault)
    rise 2                      #需要成功多少次，vrrp才进行角色状态切换(从fault切换为正常)
}
vrrp_instance VI_1 {
    state MASTER      #指定Master或者BACKUP，必须大写
    interface ens33   #监听的实际网口,VIP所在接口
    dont_track_primary
    virtual_router_id 10  #VIP末段
    priority 100          #权重值，数字越大优先级越高，谁的优先级高谁就是master，该项取值范围是1-255（在此范围之外会被识别成默认值100）
    advert_int 3          #发送组播包的间隔时间，默认为1秒
    track_script {
        check-haproxy
    }
    virtual_ipaddress {
        192.168.17.10/24  #vip地址
    }
}
EOF

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id master
}
vrrp_script check-haproxy {
    script "killall -0 haproxy"
    interval 5
    weight -3
    fall 10
    rise 2
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 10
    dont_track_primary
    priority 99
    advert_int 3
    track_script {
        check-haproxy
    }
    virtual_ipaddress {
        192.168.17.10/24
    }
}
EOF

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id master
}
vrrp_script check-haproxy {
    script "killall -0 haproxy"
    interval 5
    weight -3
    fall 10
    rise 2
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 10
    dont_track_primary
    priority 98
    advert_int 3
    track_script {
        check-haproxy
    }
    virtual_ipaddress {
        192.168.17.10/24
    }
}
EOF

systemctl start keepalived && systemctl enable keepalived

yum install -y haproxy

cat > /etc/haproxy/haproxy.cfg <<EOF
global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /var/run/haproxy-admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    nbproc 1

defaults
    log     global
    timeout connect 5000
    timeout client  10m
    timeout server  10m

listen  admin_stats
    bind 0.0.0.0:10080
    mode http
    log 127.0.0.1 local0 err
    stats refresh 30s
    stats uri /status
    stats realm welcome login Haproxy
    stats auth admin:123456
    stats hide-version
    stats admin if TRUE

listen kube-master
    bind 0.0.0.0:8443
    mode tcp
    option tcplog
    balance source
    server 192.168.17.101 192.168.17.101:6443 check inter 2000 fall 2 rise 2 weight 1
    server 192.168.17.102 192.168.17.102:6443 check inter 2000 fall 2 rise 2 weight 1
    server 192.168.17.103 192.168.17.103:6443 check inter 2000 fall 2 rise 2 weight 1
EOF

#设置开机自启
systemctl enable haproxy && systemctl restart haproxy

#查看是否启动完成
netstat -lntup|grep haproxy

cat > kubeadm-config.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.22.2
apiServer:
  certSANs:
  - 192.168.17.101  #master1
  - 192.168.17.102  #master2
  - 192.168.17.103  #master3
  - 192.168.17.10   #VIP
  extraArgs:
    authorization-mode: Node,RBAC
  timeoutForControlPlane: 4m0s
#vip地址
controlPlaneEndpoint: 192.168.17.10:6443
#已预拉镜像，初始化可不指定
#imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers

networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12

EOF

kubeadm init --config=kubeadm-config.yaml --upload-certs

# 在master1节点执行，这里的命令是kubeadm init命令执行后，提示出来的
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 查看部署CNI网络插件运行状态
kubectl get pods -n kube-system
# 在状态都为Running时，查看节点状态
kubectl get nodes

#创建测试命名空间
kubectl create ns dev
#部署nginx
kubectl create deployment  nginx --image=nginx:1.17.1 -n dev
#暴露端口
kubectl expose deployment nginx -n dev --port=80 --type=NodePort
#查看服务状态
kubectl get pods,svc -n dev

kubeadm token create --print-join-command

#如果环境出现问题，尽最大努力还原通过 'kubeadm init' 或者 'kubeadm join' 操作对主机所做的更改
kubeadm reset
rm -rf $HOME/.kube