无极最新招聘信息优化公司结构
一、
前言和写作原因
本文做一个kubernetes集群部署记录，实在是部署的东西太多了，害怕忘记，kubernetes集群的部署又细节比较多，因此，在这里做一个尽量详细的记录
三个VMware虚拟机，IP分别为192.168.123.11，192.168.123.12，192.168.123.13， 11服务器为master，12，13服务器为工作节点，node1和node2
操作系统为centos7，kubernetes的版本为1.25.7，下面是部署成功的截图：
[root@k8s-master ~]# k get no
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 3h55m v1.25.7
k8s-node1 Ready <none> 3h45m v1.25.7
k8s-node2 Ready <none> 3h45m v1.25.7
[root@k8s-master ~]# k get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-2 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
etcd-0 Healthy {"health":"true"}
[root@k8s-master ~]# k get csr
NAME AGE SIGNERNAME REQUESTOR REQUESTEDDURATION CONDITION
csr-kx5cd 3h45m kubernetes.io/kube-apiserver-client-kubelet system:bootstrap:abcdef <none> Approved,Issued
csr-p55mg 3h46m kubernetes.io/kube-apiserver-client-kubelet system:bootstrap:abcdef <none> Approved,Issued
csr-x2p5m 3h55m kubernetes.io/kube-apiserver-client-kubelet system:node:k8s-master <none> Approved,Issued
[root@k8s-master ~]# k get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-c676cc86f-r5f89 1/1 Running 1 (46m ago) 52m
kube-system coredns-c676cc86f-rmbt5 1/1 Running 1 (46m ago) 52m
kube-system kube-apiserver-k8s-master 1/1 Running 4 (46m ago) 3h55m
kube-system kube-controller-manager-k8s-master 1/1 Running 5 (46m ago) 3h55m
kube-system kube-flannel-ds-hkbb4 1/1 Running 1 (46m ago) 48m
kube-system kube-flannel-ds-p5whh 1/1 Running 1 (46m ago) 48m
kube-system kube-flannel-ds-r2ltp 1/1 Running 1 (46m ago) 48m
kube-system kube-proxy-6tcvm 1/1 Running 3 (46m ago) 3h46m
kube-system kube-proxy-bfjlz 1/1 Running 3 (46m ago) 3h45m
kube-system kube-proxy-m99t4 1/1 Running 4 (46m ago) 3h55m
kube-system kube-scheduler-k8s-master 1/1 Running 4 (46m ago) 3h55m
这里需要说明一下，该集群是使用的一年证书，因此，是不适用于在生产使用的，全部操作可以离线化完成
云原生|kubernetes|kubeadm部署的集群的100年证书_kubernetes 百年证书-CSDN博客 这个是关于证书时间的问题解决博客
相关资料下载地址：
通过网盘分享的文件：kubeadm部署1.25.7
链接: https://pan.baidu.com/s/1AG2qSw3zufs884afM38Ufg?pwd=7a97 提取码: 7a97
--来自百度网盘超级会员v6的分享
二、
kubernetes集群部署前的环境准备
1、
服务器环境初始化
大部分环境方面还是使用脚本来快速处理，脚本内容如下：
注意。里面有三个IP，需要根据实际修改好，脚本执行方式为：bash 脚本名称 主机名
例如，在master节点，那么，执行命令为 bash os7init.sh k8s-master，在node1节点也就是192.168.123.12，那么执行命令为 bash os7init.sh k8s-node1
别忘了把kernel-lt-5.4.266-1.el7.elrepo.x86_64.rpm 这个文件丢到root目录，脚本执行完毕后，建议重启一下服务器
#!/bin/bash
# init centos7 ./centos7-init.sh 主机名# 检查是否为root用户，脚本必须在root权限下运行
if [[ "$(whoami)" != "root" ]]; thenecho "please run this script as root !" >&2exit 1
fi
echo -e "\033[31m the script only Support CentOS_7 x86_64 \033[0m"
echo -e "\033[31m system initialization script, Please Seriously. press ctrl+C to cancel \033[0m"# 检查是否为64位系统，这个脚本只支持64位脚本
platform=`uname -i`
if [ $platform != "x86_64" ];thenecho "this script is only for 64bit Operating System !"exit 1
fiif [ "$1" == "" ];thenecho "The host name is empty."exit 1
elsehostnamectl --static set-hostname $1hostnamectl set-hostname $1
ficat << EOF
+---------------------------------------+
| your system is CentOS 7 x86_64 |
| start optimizing |
+---------------------------------------+
EOF
sleep 1# 安装必要支持工具及软件工具
yum_update(){
yum update -y
yum install -y nmap unzip wget vim lsof xz net-tools iptables-services ntpdate ntp-doc psmisc lrzsz expect conntrack telnet net-tools }# 设置时间同步 set time
zone_time(){
timedatectl set-timezone Asia/Shanghai
/usr/sbin/ntpdate 0.cn.pool.ntp.org > /dev/null 2>&1
/usr/sbin/hwclock --systohc
/usr/sbin/hwclock -w
cat > /var/spool/cron/root << EOF
10 0 * * * /usr/sbin/ntpdate 0.cn.pool.ntp.org > /dev/null 2>&1
* * * * */1 /usr/sbin/hwclock -w > /dev/null 2>&1
EOF
chmod 600 /var/spool/cron/root
/sbin/service crond restart
sleep 1
}# 修改文件打开数 set the file limit
limits_config(){
cat > /etc/rc.d/rc.local << EOF
#!/bin/bashtouch /var/lock/subsys/local
ulimit -SHn 1024000
EOFsed -i "/^ulimit -SHn.*/d" /etc/rc.d/rc.local
echo "ulimit -SHn 1024000" >> /etc/rc.d/rc.localsed -i "/^ulimit -s.*/d" /etc/profile
sed -i "/^ulimit -c.*/d" /etc/profile
sed -i "/^ulimit -SHn.*/d" /etc/profilecat >> /etc/profile << EOF
ulimit -c unlimited
ulimit -s unlimited
ulimit -SHn 1024000
EOFsource /etc/profile
ulimit -a
cat /etc/profile | grep ulimitif [ ! -f "/etc/security/limits.conf.bak" ]; thencp /etc/security/limits.conf /etc/security/limits.conf.bak
ficat > /etc/security/limits.conf << EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOFif [ ! -f "/etc/security/limits.d/20-nproc.conf.bak" ]; thencp /etc/security/limits.d/20-nproc.conf /etc/security/limits.d/20-nproc.conf.bak
ficat > /etc/security/limits.d/20-nproc.conf << EOF
* soft nproc 409600
root soft nproc unlimited
EOFsleep 1
}# 优化内核参数 tune kernel parametres
sysctl_config(){
if [ ! -f "/etc/sysctl.conf.bak" ]; thencp /etc/sysctl.conf /etc/sysctl.conf.bak
fi#add
cat > /etc/sysctl.conf << EOF
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 5
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_max_tw_buckets = 60000
net.ipv4.tcp_max_orphans = 32768
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_wmem = 4096 16384 13107200
net.ipv4.tcp_rmem = 4096 87380 17476000
net.ipv4.ip_local_port_range = 1024 65000
net.ipv4.ip_forward = 1
net.ipv4.route.gc_timeout = 100
net.core.somaxconn = 32768
net.core.netdev_max_backlog = 32768
net.nf_conntrack_max = 6553500
net.netfilter.nf_conntrack_max = 6553500
net.netfilter.nf_conntrack_tcp_timeout_established = 180
vm.overcommit_memory = 1
vm.swappiness = 1
fs.file-max = 1024000
EOF#reload sysctl
/sbin/sysctl -p
sleep 1
}# 设置UTF-8 LANG="zh_CN.UTF-8"
LANG_config(){
echo "LANG=\"en_US.UTF-8\"">/etc/locale.conf
source /etc/locale.conf
}#关闭SELINUX disable selinux
selinux_config(){
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
setenforce 0
sleep 1
}#日志处理
log_config(){
setenforce 0
systemctl start systemd-journald
systemctl status systemd-journald
}# 关闭防火墙
firewalld_config(){
/usr/bin/systemctl stop firewalld.service
/usr/bin/systemctl disable firewalld.service
}# SSH配置优化 set sshd_config
sshd_config(){
if [ ! -f "/etc/ssh/sshd_config.bak" ]; thencp /etc/ssh/sshd_config /etc/ssh/sshd_config.bak
ficat >/etc/ssh/sshd_config<<EOF
Port 22
AddressFamily inet
ListenAddress 0.0.0.0
Protocol 2
HostKey /etc/ssh/ssh_host_rsa_key
HostKey /etc/ssh/ssh_host_ecdsa_key
HostKey /etc/ssh/ssh_host_ed25519_key
SyslogFacility AUTHPRIV
PermitRootLogin yes
MaxAuthTries 6
RSAAuthentication yes
PubkeyAuthentication yes
AuthorizedKeysFile .ssh/authorized_keys
PasswordAuthentication yes
ChallengeResponseAuthentication no
UsePAM yes
UseDNS no
X11Forwarding yes
UsePrivilegeSeparation sandbox
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
AcceptEnv XMODIFIERS
Subsystem sftp /usr/libexec/openssh/sftp-server
EOF
/sbin/service sshd restart
}# 关闭ipv6 disable the ipv6
ipv6_config(){
echo "NETWORKING_IPV6=no">/etc/sysconfig/network
echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6
echo 1 > /proc/sys/net/ipv6/conf/default/disable_ipv6
echo "127.0.0.1 localhost localhost.localdomain">/etc/hosts
#sed -i 's/IPV6INIT=yes/IPV6INIT=no/g' /etc/sysconfig/network-scripts/ifcfg-enp0s8for line in $(ls -lh /etc/sysconfig/network-scripts/ifcfg-* | awk -F '[ ]+' '{print $9}')
do
if [ -f $line ]thensed -i 's/IPV6INIT=yes/IPV6INIT=no/g' $lineecho $i
fi
done
}# 设置历史命令记录格式 history
history_config(){
export HISTFILESIZE=10000000
export HISTSIZE=1000000
export PROMPT_COMMAND="history -a"
export HISTTIMEFORMAT="%Y-%m-%d_%H:%M:%S "
##export HISTTIMEFORMAT="{\"TIME\":\"%F %T\",\"HOSTNAME\":\"\$HOSTNAME\",\"LI\":\"\$(who -u am i 2>/dev/null| awk '{print \$NF}'|sed -e 's/[()]//g')\",\"LU\":\"\$(who am i|awk '{print \$1}')\",\"NU\":\"\${USER}\",\"CMD\":\""
cat >>/etc/bashrc<<EOF
alias vi='vim'
HISTDIR='/var/log/command.log'
if [ ! -f \$HISTDIR ];then
touch \$HISTDIR
chmod 666 \$HISTDIR
fi
export HISTTIMEFORMAT="{\"TIME\":\"%F %T\",\"IP\":\"\$(ip a | grep -E '192.168|172' | head -1 | awk '{print \$2}' | cut -d/ -f1)\",\"LI\":\"\$(who -u am i 2>/dev/null| awk '{print \$NF}'|sed -e 's/[()]//g')\",\"LU\":\"\$(who am i|awk '{print \$1}')\",\"NU\":\"\${USER}\",\"CMD\":\""
export PROMPT_COMMAND='history 1|tail -1|sed "s/^[ ]\+[0-9]\+ //"|sed "s/$/\"}/">> /var/log/command.log'
EOF
source /etc/bashrc
}# 服务优化设置
service_config(){
/usr/bin/systemctl stop postfix.service
/usr/bin/systemctl disable postfix.service
chmod +x /etc/rc.local
chmod +x /etc/rc.d/rc.local
#ls -l /etc/rc.d/rc.local
}# VIM设置
vim_config(){
cat > /root/.vimrc << EOF
set history=1000EOF#autocmd InsertLeave * se cul
#autocmd InsertLeave * se nocul
#set nu
#set bs=2
#syntax on
#set laststatus=2
#set tabstop=4
#set go=
#set ruler
#set showcmd
#set cmdheight=1
#hi CursorLine cterm=NONE ctermbg=blue ctermfg=white guibg=blue guifg=white
#set hls
#set cursorline
#set ignorecase
#set hlsearch
#set incsearch
#set helplang=cn
}# done
done_ok(){
touch /var/log/init-ok
cat << EOF
+-------------------------------------------------+
| optimizer is done |
| it's recommond to restart this server ! |
| Please Reboot system |
+-------------------------------------------------+
EOF
}update_kernel(){
yum install /root/kernel-lt-5.4.266-1.el7.elrepo.x86_64.rpm -y
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
yum install ipvsadm ipset sysstat conntrack libseccomp -y
cat >> /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
overlay
br_netfilter
EOFsystemctl restart systemd-modules-load.service}
zidingyihosts(){
cat >/etc/hosts<<EOF
127.0.0.1 localhost localhost.localdomain
192.168.123.11 k8s-master
192.168.123.12 k8s-node1
192.168.123.13 k8s-node2
EOF
}
# main
main(){yum_updatezone_timelimits_configsysctl_configLANG_configselinux_configlog_configfirewalld_config#sshd_configipv6_confighistory_configservice_configvim_configdone_okupdate_kernelzidingyihosts
}
main2、
快速免密脚本
这个别忘了把自己的密码写正确了
#!/usr/bin/expect -f# 参数初始化
set ip_list [lrange $argv 0 end]
set user "root"
set password "自己的密码"# 生成密钥对（完整交互处理）
spawn ssh-keygen -t rsa -b 4096 -N ""expect {"Enter file in which to save the key" { send "\r"; exp_continue }"Overwrite (y/n)" { send "y\r"; exp_continue }"Are you sure" { send "yes\r"; exp_continue }"Enter passphrase (empty for no passphrase)" { send "\r"; exp_continue }"Enter same passphrase again" { send "\r"; exp_continue }
# eof {
3 exit 1
# }
}# 批量部署公钥（带超时和重试）
foreach ip $ip_list {if {![regexp {^([0-9]{1,3}\.){3}[0-9]{1,3}$} $ip]} {puts "⚠️ 跳过无效IP: $ip"continue}puts "🔧 正在部署公钥到 $ip..."spawn ssh-copy-id $user@$ipexpect {"(yes/no)" { send "yes\r"; exp_continue }"*assword:" { send "$password\r"; exp_continue }"success" { puts "✅ 公钥部署成功: $ip"; close $spawn_id;continue }eof { puts "❌ 连接失败: $ip (密码错误或网络问题)";close $spawn_id;continue }timeout { puts "⏳ 连接超时: $ip (5秒未响应)";close $spawn_id;continue }}
}
puts "\n 所有服务器免密登录部署完成！"
该脚本执行命令为：expect expect.sh 192.168.123.11 192.168.123.12 192.168.123.13
正确输出为，如果不正确可以反复执行：
❌ 连接失败: 192.168.123.12 (密码错误或网络问题)
🔧 正在部署公钥到 192.168.123.13...
spawn ssh-copy-id root@192.168.123.13
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
root@192.168.123.13's password: Number of key(s) added: 1Now try logging into the machine, with: "ssh 'root@192.168.123.13'"
and check to make sure that only the key(s) you wanted were added.❌ 连接失败: 192.168.123.13 (密码错误或网络问题)所有服务器免密登录部署完成！
3、
docker服务和etcd集群安装仍然是使用ansible来进行安装：
cfssl.tar.gz和docker-27.5.0.tgz文件别忘记放到master的root目录下，在此之前，要在master服务器安装ansible，安装命令为：yum install ansible -y
docker-27.5.0.tgz需要放置到root目录下，不需要解压
进入ansible-deployment-dcoker目录后直接执行命令ansible-playbook -i hosts multi-deploy-docker.yaml 就可以快速安装好docker服务了
最终输出应该为：
" Total Memory: 3.806GiB", " Name: k8s-node2", " ID: 7d2a48be-8b73-4380-a8ca-faabd8877532", " Docker Root Dir: /var/lib/docker", " Debug Mode: false", " Experimental: false", " Insecure Registries:", " 127.0.0.0/8", " Registry Mirrors:", " http://bc437cce.m.daocloud.io/", " Live Restore Enabled: false", " Product License: Community Engine"]
}PLAY RECAP *********************************************************************************************************************************************************************************************************************************************************************************************************************************
192.168.123.11 : ok=9 changed=8 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.123.12 : ok=9 changed=8 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.123.13 : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
可参考文章：利用ansible的角色快速批量一键部署基础docker环境_ansible-playbook docker role-CSDN博客
etcd集群：
cfssl.tar.gz和etcd-v3.4.9-linux-amd64.tar.gz需要放到root目录下
修改hosts文件，内容如下：
修改group_vars/all.yml文件，内容如下：
进入ansible-deployment-etcd目录后，直接执行命令ansible-playbook -i hosts deployment-etcd-cluster.yaml 就可以快速安装好etcd集群了
最终输出大概像这样的：
ASK [etcd : debug] ************************************************************************************************************************************************************************************************************************************************************************************************************************
ok: [192.168.123.11] => {"status.stdout_lines": ["+------------------+---------+--------+-----------------------------+-----------------------------+------------+", "| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER |", "+------------------+---------+--------+-----------------------------+-----------------------------+------------+", "| 8ef8187eebb59092 | started | etcd-2 | https://192.168.123.12:2380 | https://192.168.123.12:2379 | false |", "| badb2f4024bbdf87 | started | etcd-3 | https://192.168.123.13:2380 | https://192.168.123.13:2379 | false |", "| d9fb26556fe7b4a5 | started | etcd-1 | https://192.168.123.11:2380 | https://192.168.123.11:2379 | false |", "+------------------+---------+--------+-----------------------------+-----------------------------+------------+"]
}
ok: [192.168.123.12] => {"status.stdout_lines": ["+------------------+---------+--------+-----------------------------+-----------------------------+------------+", "| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER |", "+------------------+---------+--------+-----------------------------+-----------------------------+------------+", "| 8ef8187eebb59092 | started | etcd-2 | https://192.168.123.12:2380 | https://192.168.123.12:2379 | false |", "| badb2f4024bbdf87 | started | etcd-3 | https://192.168.123.13:2380 | https://192.168.123.13:2379 | false |", "| d9fb26556fe7b4a5 | started | etcd-1 | https://192.168.123.11:2380 | https://192.168.123.11:2379 | false |", "+------------------+---------+--------+-----------------------------+-----------------------------+------------+"]
}
ok: [192.168.123.13] => {"status.stdout_lines": ["+------------------+---------+--------+-----------------------------+-----------------------------+------------+", "| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER |", "+------------------+---------+--------+-----------------------------+-----------------------------+------------+", "| 8ef8187eebb59092 | started | etcd-2 | https://192.168.123.12:2380 | https://192.168.123.12:2379 | false |", "| badb2f4024bbdf87 | started | etcd-3 | https://192.168.123.13:2380 | https://192.168.123.13:2379 | false |", "| d9fb26556fe7b4a5 | started | etcd-1 | https://192.168.123.11:2380 | https://192.168.123.11:2379 | false |", "+------------------+---------+--------+-----------------------------+-----------------------------+------------+"]
}PLAY RECAP *********************************************************************************************************************************************************************************************************************************************************************************************************************************
192.168.123.11 : ok=11 changed=5 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.123.12 : ok=11 changed=5 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.123.13 : ok=11 changed=4 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
localhost : ok=6 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
可参考文章；centos7操作系统 ---ansible剧本离线快速部署etcd集群_ansible 部署etcd集群-CSDN博客
4、
安装cni（三个节点都做）
cni-plugins-linux-amd64-v1.6.2.tgz放置到root目录下
mkdir -p /opt/cni/bin
tar xvf cni-plugins-linux-amd64-v1.6.2.tgz -C /opt/cni/bin5、
cri-docker 中间件部署（三个节点都做）
下载地址：
https://github.com/Mirantis/cri-dockerd/releases
也可以直接使用离线包内的，这个无所谓，解压命令：tar zxvf cri-dockerd-0.3.16.amd64.tgz && mv cri-dockerd/cri-dockerd /usr/bin/&&chmod a+x /usr/bin/cri-dockerd
创建启动脚本文件：
cat >/usr/lib/systemd/system/cri-docker.service <<EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
#Requires=cri-docker.socket[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --cri-dockerd-root-directory=/var/lib/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.10 --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d
ExecReload=/bin/kill -s HUP
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process[Install]
WantedBy=multi-user.target
EOF启动cri-docker
systemctl enable --now cri-docker最终查看状态，绿色为正确，特别需要注意要有Connecting to docker on the Endpoint unix:///var/run/docker.sock：
[root@k8s-node1 ~]# systemctl status cri-docker
● cri-docker.service - CRI Interface for Docker Application Container EngineLoaded: loaded (/usr/lib/systemd/system/cri-docker.service; enabled; vendor preset: disabled)Active: active (running) since Sat 2025-03-22 23:16:27 CST; 911ms agoDocs: https://docs.mirantis.comMain PID: 62457 (cri-dockerd)Tasks: 9Memory: 14.6MCGroup: /system.slice/cri-docker.service└─62457 /usr/bin/cri-dockerd --cri-dockerd-root-directory=/var/lib/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.10 --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.dMar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Connecting to docker on the Endpoint unix:///var/run/docker.sock"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Start docker client with request timeout 0s"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Hairpin mode is set to none"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Loaded network plugin cni"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Docker cri networking managed by network plugin cni"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Setting cgroupDriver systemd"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Docker cri received runtime config &RuntimeConfig{NetworkConfig:&NetworkConfig{PodCidr:,},}"
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Starting the GRPC backend for the Docker CRI interface."
Mar 22 23:16:27 k8s-node1 cri-dockerd[62457]: time="2025-03-22T23:16:27+08:00" level=info msg="Start cri-dockerd grpc backend"
Mar 22 23:16:27 k8s-node1 systemd[1]: Started CRI Interface for Docker Application Container Engine.
三、
etcd集群的证书处理
（建立目录在三个节点都执行，在master节点拷贝好文件后，scp到工作节点12和13）
mkdir -p /etc/kubernetes/pki/etcd/
cp /opt/etcd/ssl/ca.pem /etc/kubernetes/pki/etcd/
cp /opt/etcd/ssl/server.pem /etc/kubernetes/pki/etcd/apiserver-etcd-client.pem
cp /opt/etcd/ssl/server-key.pem /etc/kubernetes/pki/etcd/apiserver-etcd-client-key.pem
scp /etc/kubernetes/pki/etcd/* k8s-node1:/etc/kubernetes/pki/etcd/
scp /etc/kubernetes/pki/etcd/* k8s-node2:/etc/kubernetes/pki/etcd/四、
安装kubeadm，kubelet和kubectl三个rpm包并执行集群初始化
yum 安装这三个文件就不废话了，有手就可以的事情，不在这废话了，三个节点都安装，
安装之前还是需要配一个yum源，配置命令如下；
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
三个节点都导入离线镜像：
导入命令为：for i in `ls /root/kubeadm的镜像/*.tar`;do docker load <$i;done
安装完毕后，使用下面这个配置文件初始化集群：
配置文件里主要是四个IP，需要根据自己实际修改，还有就是
imagePullPolicy: IfNotPresent
name: k8s-master 这个也是必须准确的主机名称，其它就不需要改动，初始化命令为：kubeadm init --config=kubeadm-init.yaml
cat >/root/kubeadm-init.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: "0"usages:- signing- authentication
kind: InitConfiguration
localAPIEndpoint:advertiseAddress: 192.168.123.11bindPort: 6443
nodeRegistration:criSocket: /var/run/cri-dockerd.sockimagePullPolicy: IfNotPresentname: k8s-mastertaints: null
---
apiServer:timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:external:endpoints: #下面为自定义etcd集群地址- https://192.168.123.11:2379- https://192.168.123.12:2379- https://192.168.123.13:2379caFile: /etc/kubernetes/pki/etcd/ca.pemcertFile: /etc/kubernetes/pki/etcd/apiserver-etcd-client.pemkeyFile: /etc/kubernetes/pki/etcd/apiserver-etcd-client-key.pem
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.25.7
networking:dnsDomain: cluster.localpodSubnet: "10.244.0.0/16"serviceSubnet: "10.96.0.0/12"
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:acceptContentTypes: ""burst: 0contentType: ""kubeconfig: /var/lib/kube-proxy/kubeconfig.confqps: 0
clusterCIDR: "10.244.0.0/16"
configSyncPeriod: 0s
conntrack:maxPerCore: nullmin: nulltcpCloseWaitTimeout: nulltcpEstablishedTimeout: null
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: "k8s-master"
iptables:masqueradeAll: falsemasqueradeBit: nullminSyncPeriod: 0ssyncPeriod: 0s
ipvs:excludeCIDRs: nullminSyncPeriod: 0sscheduler: ""strictARP: falsesyncPeriod: 0stcpFinTimeout: 0stcpTimeout: 0sudpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: ""
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
udpIdleTimeout: 0s
winkernel:enableDSR: falsenetworkName: ""sourceVip: ""
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:anonymous:enabled: falsewebhook:cacheTTL: 0senabled: truex509:clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:mode: Webhookwebhook:cacheAuthorizedTTL: 0scacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging: {}
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
EOF初始化非常快，基本十几秒就好了，正确输出如下：
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxyYour Kubernetes control-plane has initialized successfully!To start using your cluster, you need to run the following as a regular user:mkdir -p $HOME/.kubesudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/configsudo chown $(id -u):$(id -g) $HOME/.kube/configAlternatively, if you are the root user, you can run:export KUBECONFIG=/etc/kubernetes/admin.confYou should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:https://kubernetes.io/docs/concepts/cluster-administration/addons/Then you can join any number of worker nodes by running the following on each as root:kubeadm join 192.168.123.11:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:6de53fdadf43dd8197b311b5eacb99fd92094cf6e43cbcb67c549ea81060f0e0根据提示命令在master节点执行：
mkdir -p $HOME/.kubesudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/configsudo chown $(id -u):$(id -g) $HOME/.kube/config
在工作节点执行：
kubeadm join 192.168.123.11:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:6de53fdadf43dd8197b311b5eacb99fd92094cf6e43cbcb67c549ea81060f0e0工作节点正确的输出如下：
[root@k8s-node1 ~]# kubeadm join 192.168.123.11:6443 --token abcdef.0123456789abcdef \
> --discovery-token-ca-cert-hash sha256:6de53fdadf43dd8197b311b5eacb99fd92094cf6e43cbcb67c549ea81060f0e0
[preflight] Running pre-flight checks[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.Run 'kubectl get nodes' on the control-plane to see this node join the cluster五、
flannel网络插件的安装：
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
主要是修改namespace ，命令为sed -i "s@namespace: kube-flannel@namespace: kube-system@g" kube-flannel.yml
删除创建namespace相关片段
修改后的文件：
cat >kube-flannel.yml <<EOF
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:labels:k8s-app: flannelname: flannel
rules:
- apiGroups:- ""resources:- podsverbs:- get
- apiGroups:- ""resources:- nodesverbs:- get- list- watch
- apiGroups:- ""resources:- nodes/statusverbs:- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:labels:k8s-app: flannelname: flannel
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: flannel
subjects:
- kind: ServiceAccountname: flannelnamespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:labels:k8s-app: flannelname: flannelnamespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:name: kube-flannel-cfgnamespace: kube-systemlabels:tier: nodek8s-app: flannelapp: flannel
data:cni-conf.json: |{"name": "cbr0","cniVersion": "0.3.1","plugins": [{"type": "flannel","delegate": {"hairpinMode": true,"isDefaultGateway": true}},{"type": "portmap","capabilities": {"portMappings": true}}]}net-conf.json: |{"Network": "10.244.0.0/16","EnableNFTables": false,"Backend": {"Type": "vxlan"}}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:name: kube-flannel-dsnamespace: kube-systemlabels:tier: nodeapp: flannelk8s-app: flannel
spec:selector:matchLabels:app: flanneltemplate:metadata:labels:tier: nodeapp: flannelspec:affinity:nodeAffinity:requiredDuringSchedulingIgnoredDuringExecution:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/osoperator: Invalues:- linuxhostNetwork: truepriorityClassName: system-node-criticaltolerations:- operator: Existseffect: NoScheduleserviceAccountName: flannelinitContainers:- name: install-cni-pluginimage: ghcr.io/flannel-io/flannel-cni-plugin:v1.6.2-flannel1command:- cpargs:- -f- /flannel- /opt/cni/bin/flannelvolumeMounts:- name: cni-pluginmountPath: /opt/cni/bin- name: install-cniimage: ghcr.io/flannel-io/flannel:v0.26.4command:- cpargs:- -f- /etc/kube-flannel/cni-conf.json- /etc/cni/net.d/10-flannel.conflistvolumeMounts:- name: cnimountPath: /etc/cni/net.d- name: flannel-cfgmountPath: /etc/kube-flannel/containers:- name: kube-flannelimage: ghcr.io/flannel-io/flannel:v0.26.4command:- /opt/bin/flanneldargs:- --ip-masq- --kube-subnet-mgrresources:requests:cpu: "100m"memory: "50Mi"securityContext:privileged: falsecapabilities:add: ["NET_ADMIN", "NET_RAW"]env:- name: POD_NAMEvalueFrom:fieldRef:fieldPath: metadata.name- name: POD_NAMESPACEvalueFrom:fieldRef:fieldPath: metadata.namespace- name: EVENT_QUEUE_DEPTHvalue: "5000"volumeMounts:- name: runmountPath: /run/flannel- name: flannel-cfgmountPath: /etc/kube-flannel/- name: xtables-lockmountPath: /run/xtables.lockvolumes:- name: runhostPath:path: /run/flannel- name: cni-pluginhostPath:path: /opt/cni/bin- name: cnihostPath:path: /etc/cni/net.d- name: flannel-cfgconfigMap:name: kube-flannel-cfg- name: xtables-lockhostPath:path: /run/xtables.locktype: FileOrCreate
EOF六、
kubectl命令补全
先安装一下命令补全：
yum install bash-completion -y下面的命令在执行一下：
source /usr/share/bash-completion/bash_completion
echo "source <(kubectl completion bash)" >>/etc/profile
echo "source /usr/share/bash-completion/bash_completion" >>/etc/profile
kubectl completion bash | sudo tee /etc/bash_completion.d/kubectl > /dev/nullecho "alias k=kubectl">>/etc/profile
echo "complete -F __start_kubectl k">>/etc/profile
source /etc/profile七、
检查集群是否正常：
随便部署一个nginx：
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80
[root@k8s-master ~]# k get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
etcd-0 Unhealthy Get "https://192.168.123.11:2379/health": dial tcp 192.168.123.11:2379: connect: connection refused
scheduler Healthy ok
controller-manager Healthy ok
etcd-2 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
[root@k8s-master ~]# k get no
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 6h29m v1.25.7
k8s-node1 Ready <none> 6h20m v1.25.7
k8s-node2 Ready <none> 6h19m v1.25.7
[root@k8s-master ~]# k get po -A -owide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default nginx-deployment-7fb96c846b-dk476 1/1 Running 0 67s 10.244.1.7 k8s-node1 <none> <none>
default nginx-deployment-7fb96c846b-gtqcz 1/1 Running 0 67s 10.244.1.6 k8s-node1 <none> <none>
default nginx-deployment-7fb96c846b-rdmtl 1/1 Running 0 67s 10.244.2.7 k8s-node2 <none> <none>
kube-system coredns-c676cc86f-r5f89 1/1 Running 2 (16m ago) 3h26m 10.244.2.6 k8s-node2 <none> <none>
kube-system coredns-c676cc86f-rmbt5 1/1 Running 2 (16m ago) 3h26m 10.244.1.5 k8s-node1 <none> <none>
kube-system kube-apiserver-k8s-master 1/1 Running 6 (16m ago) 6h29m 192.168.123.11 k8s-master <none> <none>
kube-system kube-controller-manager-k8s-master 1/1 Running 1 (16m ago) 6h29m 192.168.123.11 k8s-master <none> <none>
kube-system kube-flannel-ds-hkbb4 1/1 Running 2 (16m ago) 3h22m 192.168.123.13 k8s-node2 <none> <none>
kube-system kube-flannel-ds-p5whh 1/1 Running 2 (16m ago) 3h22m 192.168.123.12 k8s-node1 <none> <none>
kube-system kube-flannel-ds-r2ltp 1/1 Running 2 (16m ago) 3h22m 192.168.123.11 k8s-master <none> <none>
kube-system kube-proxy-6tcvm 1/1 Running 4 (16m ago) 6h20m 192.168.123.12 k8s-node1 <none> <none>
kube-system kube-proxy-bfjlz 1/1 Running 4 (16m ago) 6h19m 192.168.123.13 k8s-node2 <none> <none>
kube-system kube-proxy-m99t4 1/1 Running 5 (16m ago) 6h29m 192.168.123.11 k8s-master <none> <none>
kube-system kube-scheduler-k8s-master 1/1 Running 1 (16m ago) 6h29m 192.168.123.11 k8s-master <none> <none>