目录

kubernetes-kubeadm搭建kubernetes

环境准备

  1. 修改主机名,添加 host 解析

    hostnamectl set-hostname k8s-master hostnamectl set-hostname k8s-node1 hostnamectl set-hostname k8s-node2

  2. 免密登录

       ssh-keygen
       ssh-copy-id -i root@master
       ssh-copy-id -i root@node1
       ssh-copy-id -i root@node2
    
  3. 关闭防火墙、swap 分区、selinux

       systemctl stop firewalld
       systemctl disable firewalld
       swapoff -a
       sed -i '/swap/s/^/#/' /etc/fstab
       setenforce 0
       sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
    
  4. 优化ssh连接

       sed -ri 's@^#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config
       sed -ri 's#^GSSAPIAuthentication yes#GSSAPIAuthentication no#g' /etc/ssh/sshd_config
       grep ^UseDNS /etc/ssh/sshd_config
       grep ^GSSAPIAuthentication /etc/ssh/sshd_config
       systemctl restart sshd
    
  5. 所有节点配置模块自动加载

       modprobe ip_vs
       modprobe ip_vs_rr
       modprobe ip_vs_wrr
       modprobe ip_vs_sh
       modprobe nf_conntrack
       modprobe br_netfilter
       modprobe ip_conntrack
       cat >>/etc/rc.sysinit<<EOF
       #!/bin/bash
       for file in /etc/sysconfig/modules/*.modules ; do
       [ -x $file ] && $file
       done
       EOF
       echo "modprobe ip_vs" >/etc/sysconfig/modules/ip_vs.modules
       echo "modprobe ip_vs_rr" >/etc/sysconfig/modules/ip_vs_rr.modules
       echo "modprobe ip_vs_wrr" >/etc/sysconfig/modules/ip_vs_wrr.modules
       echo "modprobe ip_vs_sh" >/etc/sysconfig/modules/ip_vs_sh.modules
       echo "modprobe nf_conntrack" >/etc/sysconfig/modules/nf_conntrack.modules
       echo "modprobe br_netfilter" >/etc/sysconfig/modules/br_netfilter.modules
       echo "modprobe ip_conntrack" >/etc/sysconfig/modules/ip_conntrack.modules
       chmod 755 /etc/sysconfig/modules/ip_vs.modules
       chmod 755 /etc/sysconfig/modules/ip_vs_rr.modules
       chmod 755 /etc/sysconfig/modules/ip_vs_wrr.modules
       chmod 755 /etc/sysconfig/modules/ip_vs_sh.modules
       chmod 755 /etc/sysconfig/modules/nf_conntrack.modules
       chmod 755 /etc/sysconfig/modules/br_netfilter.modules
       chmod 755 /etc/sysconfig/modules/ip_conntrack.modules
       lsmod | grep -E "ip_vs|ip_vs_rr|ip_vs_wrr|ip_vs_sh|nf_conntrack|br_netfilter|ip_conntrack"
    

    输出以下内容为配置成功

       ip_vs_sh               16384  0
       ip_vs_wrr              16384  0
       ip_vs_rr               16384  0
       ip_vs                 180224  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
       nf_conntrack          176128  1 ip_vs
       nf_defrag_ipv6         20480  2 nf_conntrack,ip_vs
       nf_defrag_ipv4         16384  1 nf_conntrack
       br_netfilter           28672  0
       bridge                294912  1 br_netfilter
       libcrc32c              16384  3 nf_conntrack,xfs,ip_vs
    
    
  6. 同步集群时间

    可参考👉NTP时间服务器

  7. 安装常用软件

       yum -y install expect wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git chrony bind-utils rsync unzip git
    
  8. 安装 IPVS

       dnf -y install ipvsadm
    
  9. 修改内核参数

       cat > /etc/sysctl.d/k8s.conf << EOF
       net.ipv4.ip_forward = 1
       net.bridge.bridge-nf-call-iptables = 1
       net.bridge.bridge-nf-call-ip6tables = 1
       fs.may_detach_mounts = 1
       vm.overcommit_memory=1
       vm.panic_on_oom=0
       fs.inotify.max_user_watches=89100
       fs.file-max=52706963
       fs.nr_open=52706963
       net.ipv4.tcp_keepalive_time = 600
       net.ipv4.tcp_keepalive_probes = 3
       net.ipv4.tcp_keepalive_intvl = 15
       net.ipv4.tcp_max_tw_buckets = 36000
       net.ipv4.tcp_tw_reuse = 1
       net.ipv4.tcp_max_orphans = 327680
       net.ipv4.tcp_orphan_retries = 3
       net.ipv4.tcp_syncookies = 1
       net.ipv4.tcp_max_syn_backlog = 16384
       net.ipv4.ip_conntrack_max = 65536
       net.ipv4.tcp_max_syn_backlog = 16384
       net.ipv4.tcp_timestamps = 0
       net.core.somaxconn = 16384
       EOF
    
       # 立即生效
       sysctl --system
    
  10. 需要在所有机器上安装容器运行时(containerd、docker等)

    推荐使用containerd作为容器运行时,使用docker配置会稍许麻烦,而且新版本的k8s已不再支持docker需要安装cri-dockerd来作为中转 👉安装cri-dockerd

    👉 containerd安装

    可查看👉Docker离线安装 ansible批量安装docker和docker-compose

安装Kubernetes

  1. 修改镜像源

       cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
       [kubernetes]
       name=Kubernetes
       baseurl=https://pkgs.k8s.io/core:/stable:/v1.34/rpm/
       enabled=1
       gpgcheck=1
       gpgkey=https://pkgs.k8s.io/core:/stable:/v1.34/rpm/repodata/repomd.xml.key
       exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
       EOF
    
  2. 安装软件包(kubelet、kubeadm 和 kubectl)master节点

       sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
       sudo systemctl enable --now kubelet
    
  3. 设置TAB键补全

       kubectl completion bash >/etc/bash_completion.d/kubectl
       kubeadm completion bash >/etc/bash_completion.d/kubeadm
    

部署(主节点)

  1. 查看所需镜像

       # 在主节点上操作,查看镜像列表
       kubeadm config images list
    
  2. 下载所需镜像并打包上传到私有仓库内

  3. 生成配置文件

       kubeadm config print init-defaults > kubeadm-init.yml
    
  4. 修改配置文件

       apiVersion: kubeadm.k8s.io/v1beta4
       bootstrapTokens:
       - groups:
         - system:bootstrappers:kubeadm:default-node-token
         token: abcdef.0123456789abcdef
         ttl: 0s
         usages:
         - signing
         - authentication
       kind: InitConfiguration
       localAPIEndpoint:
         advertiseAddress: 10.10.10.254   #决定api装在哪台机器上
         bindPort: 6443
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock  #设置容器运行时Unix 域套接字
         imagePullPolicy: IfNotPresent
         imagePullSerial: true
         name: node
         taints: null
       timeouts:
         controlPlaneComponentHealthCheck: 4m0s
         discovery: 5m0s
         etcdAPICall: 2m0s
         kubeletHealthCheck: 4m0s
         kubernetesAPICall: 1m0s
         tlsBootstrap: 5m0s
         upgradeManifests: 5m0s
       ---
       apiServer: {}
       apiVersion: kubeadm.k8s.io/v1beta4
       caCertificateValidityPeriod: 87600h0m0s
       certificateValidityPeriod: 8760h0m0s
       certificatesDir: /etc/kubernetes/pki
       clusterName: kubernetes
       controllerManager: {}
       dns: {}
       encryptionAlgorithm: RSA-2048
       etcd:
         local:
           dataDir: /var/lib/etcd
       imageRepository: registry.godone.top/pub   #设置私有仓库地址
       kind: ClusterConfiguration
       kubernetesVersion: 1.34.1
       networking:
         dnsDomain: cluster.local
         serviceSubnet: 10.254.0.0/16
         podSubnet: 10.244.0.0/16
       proxy: {}
       scheduler: {}
    
       ---
       #申明cgroup用 systemd
       apiVersion: kubelet.config.k8s.io/v1beta1
       kind: KubeletConfiguration
       cgroupDriver: systemd
       failSwapOn: false
       ---
       #启用ipvs
       apiVersion: kubeproxy.config.k8s.io/v1alpha1
       kind: KubeProxyConfiguration
       mode: ipvs
    
    
  5. 下载镜像

    kubeadm config images pull --config=kubeadm-init.yml

  6. 执行初始化并将日志记录到 kubeadm-init.log 日志文件中

    kubeadm init --config=kubeadm-init.yml | tee kubeadm-init.log

    如果初始化失败可执行下面命令重置 kubeadm 环境

    注:系统必须有/etc/resolv.conf​​文件(全部节点都需要,否则会出现错误)

       sudo kubeadm reset -f
       sudo rm -rf /etc/kubernetes /var/lib/etcd /var/lib/kubelet /etc/cni/net.d
    
  7. 查看集群是否处于健康状态

       [root@k8s-master init]# kubectl get cs
       Warning: v1 ComponentStatus is deprecated in v1.19+
       NAME                 STATUS    MESSAGE   ERROR
       controller-manager   Healthy   ok
       scheduler            Healthy   ok
       etcd-0               Healthy   ok
    

集群加入node节点

安装

  1. node节点无需安装 kubectl 客户端工具。

       yum install -y kubeadm-1.26.0 kubelet-1.26.0
    
  2. node节点也需要安装containerd软件包

加入集群

提示:加入集群前,请务必确认 /etc/containerd/config.toml​​ 中配置修改正确,可以直接拷贝 k8s-master 节点配置文件到此。

scp k8s-master:/etc/containerd/config.toml /etc/containerd/config.toml

配置完成后,重启服务

systemctl restart containerd

设置 kubelet 开机启动:

systemctl enable kubelet

加入集群,该命令在 k8s-master 节点初始化成功后,日志直接给出。

kubeadm join 10.10.10.254:6443 --token abcdef.0123456789abcdef         --discovery-token-ca-cert-hash sha256:a05ac810ead05a9d659331b2958c42d8088816fcf277cbb8a5ac6e91ef2c06e5

kubectl相关命令

命令 说明
kubectl get nodes -o wide 查看集群
kubectl get nodes -o wide 集群节点状态
kubectl get po -o wide -A 集群所有Pod状态
kubectl get svc -A 集群所有Service状态
kubectl get configmaps -A kubectl get configmaps -A

安装网络插件Flanel

  1. 导入相关Flanel镜像包

  2. 配置kube-flannel.yml文件

       apiVersion: v1
       kind: Namespace
       metadata:
         labels:
           k8s-app: flannel
           pod-security.kubernetes.io/enforce: privileged
         name: kube-flannel
       ---
       apiVersion: v1
       kind: ServiceAccount
       metadata:
         labels:
           k8s-app: flannel
         name: flannel
         namespace: kube-flannel
       ---
       apiVersion: rbac.authorization.k8s.io/v1
       kind: ClusterRole
       metadata:
         labels:
           k8s-app: flannel
         name: flannel
       rules:
       - apiGroups:
         - ""
         resources:
         - pods
         verbs:
         - get
       - apiGroups:
         - ""
         resources:
         - nodes
         verbs:
         - get
         - list
         - watch
       - apiGroups:
         - ""
         resources:
         - nodes/status
         verbs:
         - patch
       ---
       apiVersion: rbac.authorization.k8s.io/v1
       kind: ClusterRoleBinding
       metadata:
         labels:
           k8s-app: flannel
         name: flannel
       roleRef:
         apiGroup: rbac.authorization.k8s.io
         kind: ClusterRole
         name: flannel
       subjects:
       - kind: ServiceAccount
         name: flannel
         namespace: kube-flannel
       ---
       apiVersion: v1
       data:
         cni-conf.json: |
           {
             "name": "cbr0",
             "cniVersion": "0.3.1",
             "plugins": [
               {
                 "type": "flannel",
                 "delegate": {
                   "hairpinMode": true,
                   "isDefaultGateway": true
                 }
               },
               {
                 "type": "portmap",
                 "capabilities": {
                   "portMappings": true
                 }
               }
             ]
           }
         net-conf.json: |
           {
             "Network": "10.244.0.0/16",  # 前面kubeadm-init.yml文件内配置的podSubnet地址
             "EnableNFTables": false,
             "Backend": {
               "Type": "vxlan"
             }
           }
       kind: ConfigMap
       metadata:
         labels:
           app: flannel
           k8s-app: flannel
           tier: node
         name: kube-flannel-cfg
         namespace: kube-flannel
       ---
       apiVersion: apps/v1
       kind: DaemonSet
       metadata:
         labels:
           app: flannel
           k8s-app: flannel
           tier: node
         name: kube-flannel-ds
         namespace: kube-flannel
       spec:
         selector:
           matchLabels:
             app: flannel
             k8s-app: flannel
         template:
           metadata:
             labels:
               app: flannel
               k8s-app: flannel
               tier: node
           spec:
             affinity:
               nodeAffinity:
                 requiredDuringSchedulingIgnoredDuringExecution:
                   nodeSelectorTerms:
                   - matchExpressions:
                     - key: kubernetes.io/os
                       operator: In
                       values:
                       - linux
             containers:
             - args:
               - --ip-masq
               - --kube-subnet-mgr
               command:
               - /opt/bin/flanneld
               env:
               - name: POD_NAME
                 valueFrom:
                   fieldRef:
                     fieldPath: metadata.name
               - name: POD_NAMESPACE
                 valueFrom:
                   fieldRef:
                     fieldPath: metadata.namespace
               - name: EVENT_QUEUE_DEPTH
                 value: "5000"
               - name: CONT_WHEN_CACHE_NOT_READY
                 value: "false"
               image: hub.godone.top/pub/flannel:v0.27.4  #设置私有仓库地址
               name: kube-flannel
               resources:
                 requests:
                   cpu: 100m
                   memory: 50Mi
               securityContext:
                 capabilities:
                   add:
                   - NET_ADMIN
                   - NET_RAW
                 privileged: false
               volumeMounts:
               - mountPath: /run/flannel
                 name: run
               - mountPath: /etc/kube-flannel/
                 name: flannel-cfg
               - mountPath: /run/xtables.lock
                 name: xtables-lock
             hostNetwork: true
             initContainers:
             - args:
               - -f
               - /flannel
               - /opt/cni/bin/flannel
               command:
               - cp
               image: hub.godone.top/pub/flannel-cni-plugin:v1.8.0-flannel1  #设置私有仓库地址
               name: install-cni-plugin
               volumeMounts:
               - mountPath: /opt/cni/bin
                 name: cni-plugin
             - args:
               - -f
               - /etc/kube-flannel/cni-conf.json
               - /etc/cni/net.d/10-flannel.conflist
               command:
               - cp
               image: hub.godone.top/pub/flannel:v0.27.4  #设置私有仓库地址
               name: install-cni
               volumeMounts:
               - mountPath: /etc/cni/net.d
                 name: cni
               - mountPath: /etc/kube-flannel/
                 name: flannel-cfg
             priorityClassName: system-node-critical
             serviceAccountName: flannel
             tolerations:
             - effect: NoSchedule
               operator: Exists
             volumes:
             - hostPath:
                 path: /run/flannel
               name: run
             - hostPath:
                 path: /opt/cni/bin
               name: cni-plugin
             - hostPath:
                 path: /etc/cni/net.d
               name: cni
             - configMap:
                 name: kube-flannel-cfg
               name: flannel-cfg
             - hostPath:
                 path: /run/xtables.lock
                 type: FileOrCreate
               name: xtables-lock
    
    

部署网络插件(master节点执行)

注:所有节点必须有/etc/resolv.conf文件

kubectl apply -f kube-flannel.yml

推荐文章

kubernetes-kubectl