操作系统： win10 on Ubuntu 18.04.3 LTS (GNU/Linux 4.4.0-18362-Microsoft x86_64)
GO 版本 go version go1.12.9 linux/amd64
GCC 版本 gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)
make 版本 Make 4.1
部署服务器： etcd 192.168.30.31 kubernetes 192.168.30.32
部署环境：centos 7.6
业务部署目录：/apps/业务
K8S 网段 10.66.0.0/16
docker pod 网段 10.67.0.0/16
使用kube-router 提供网络服务
cd /usr/local/src wget https://dl.google.com/go/go1.12.9.linux-amd64.tar.gz tar -xvf go1.12.9.linux-amd64.tar.gz mv go ../ vi /etc/profile export GOPATH=/mnt/e/work/go export GOBIN=/mnt/e/work/go/bin PATH=$PATH:/usr/local/go/bin:$HOME/bin:$GOBIN export PATH source /etc/profile go version root@Qist:~# go version go version go1.12.9 linux/amd64 创建go 工作目录BIN目录 mkdir -p /mnt/e/work/go/{bin,src,pkg}apt -y install make gccwget https://github.com/kubernetes/kubernetes/archive/v1.15.3.tar.gz tar -xvf v1.15.3.tar.gz cd kubernetes-1.15.3/ make cd ./_output/local/bin/linux/amd64 mkdir -p /mnt/e/work/k8s/bin cp -pdr kube* /mnt/e/work/k8s/bin/go get github.com/cloudflare/cfssl/cmd/cfssl go get github.com/cloudflare/cfssl/cmd/cfssljson#etcd 使用二进制方式部署编译依赖会用到墙外的东西环境设置比较麻烦 etcd 节点服务器操作 wget https://github.com/etcd-io/etcd/releases/download/v3.4.0/etcd-v3.4.0-linux-amd64.tar.gz mkdir -p /apps/etcd/{bin,conf,ssl,data} # 数据存储目录 mkdir -p /apps/etcd/data/default.etcd # 创建 etcd 用户 useradd etcd -s /sbin/nologin -M # 解压etcd tar -xvf etcd-v3.4.0-linux-amd64.tar.gz # cp 可执行文件到工作目录 cd etcd-v3.4.0-linux-amd64/ cp -pdr etcd etcdctl /apps/etcd/bin/ # etcd 证书 在win on Ubuntu 操作 mkdir -p /apps/work/k8s/cfssl/ && \ cat << EOF | tee /apps/work/k8s/cfssl/ca-config.json { "signing": { "default": { "expiry": "87600h" }, "profiles": { "kubernetes": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" } } } } EOF # 创建etcd ca证书配置 mkdir -p /apps/work/k8s/cfssl/etcd cat << EOF | tee /apps/work/k8s/cfssl/etcd/etcd-ca-csr.json { "CN": "etcd", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF # 生成 ETCD CA 证书和私钥 mkdir -p /apps/work/k8s/cfssl/pki/etcd cfssl gencert -initca /apps/work/k8s/cfssl/etcd/etcd-ca-csr.json | cfssljson -bare /apps/work/k8s/cfssl/pki/etcd/etcd-ca # 创建 ETCD Server 证书 export ETCD_SERVER_IPS=" \ \"192.168.30.31\" \ " && \ export ETCD_SERVER_HOSTNAMES=" \ \"etcd\" \ " && \ cat << EOF | tee /apps/work/k8s/cfssl/etcd/etcd_server.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_SERVER_IPS}, ${ETCD_SERVER_HOSTNAMES} ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF # 生成 ETCD Server 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/etcd/etcd-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/etcd/etcd-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/etcd/etcd_server.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/etcd/etcd_server # 创建 ETCD Member 证书 export ETCD_MEMBER_1_IP=" \ \"192.168.30.31\" \ " && \ export ETCD_MEMBER_1_HOSTNAMES="etcd\ " && \ cat << EOF | tee /apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json { "CN": "etcd", "hosts": [ "127.0.0.1", ${ETCD_MEMBER_1_IP}, "${ETCD_MEMBER_1_HOSTNAMES}" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF ##### 生成 ETCD Member 1 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/etcd/etcd-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/etcd/etcd-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/etcd/etcd_member_${ETCD_MEMBER_1_HOSTNAMES} # 创建 ETCD Client 配置文件 cat << EOF | tee /apps/work/k8s/cfssl/etcd/etcd_client.json { "CN": "client", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF #生成 ETCD Client 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/etcd/etcd-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/etcd/etcd-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/etcd/etcd_client.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/etcd/etcd_client # 复制证书到etcd 节点服务器 scp -r /apps/work/k8s/cfssl/pki/etcd/* 192.168.30.31:/apps/etcd/ssl # 配置etcd 启动文件 etcd 服务器操作 vi /apps/etcd/conf/etcd ETCD_OPTS="--name=etcd \ --data-dir=/apps/etcd/data/default.etcd \ --listen-peer-urls=https://192.168.30.31:2380 \ --listen-client-urls=https://192.168.30.31:2379,https://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.30.31:2379 \ --initial-advertise-peer-urls=https://192.168.30.31:2380 \ --initial-cluster=etcd=https://192.168.30.31:2380\ --initial-cluster-token=node4=etcd=https://192.168.30.31:2380 \ --initial-cluster-state=new \ --heartbeat-interval=6000 \ --election-timeout=30000 \ --snapshot-count=5000 \ --auto-compaction-retention=1 \ --max-request-bytes=33554432 \ --quota-backend-bytes=17179869184 \ --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \ --cert-file=/apps/etcd/ssl/etcd_server.pem \ --key-file=/apps/etcd/ssl/etcd_server-key.pem \ --peer-cert-file=/apps/etcd/ssl/etcd_member_etcd.pem \ --peer-key-file=/apps/etcd/ssl/etcd_member_etcd-key.pem \ --peer-client-cert-auth \ --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem" # 配置etcd 启动service 文件 vi /usr/lib/systemd/system/etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify LimitNOFILE=1024000 LimitNPROC=1024000 LimitCORE=infinity LimitMEMLOCK=infinity User=etcd Group=etcd EnvironmentFile=-/apps/etcd/conf/etcd ExecStart=/apps/etcd/bin/etcd $ETCD_OPTS Restart=on-failure [Install] WantedBy=multi-user.target # etcd 目录 etcd 用户权限 chown -R etcd.etcd /apps/etcd # 启动 etcd systemctl start etcd # 设置开机启动 systemctl enable etcd # 设置环境变量 /etc/profile export ETCDCTL_API=3 export ENDPOINTS=https://192.168.30.31:2379 #执行 source /etc/profile vim ~/.bashrc alias etcdctl='/apps/etcd/bin/etcdctl --endpoints=${ENDPOINTS} --cacert=/apps/etcd/ssl/etcd-ca.pem' source ~/.bashrc # 查看状态 etcdctl endpoint health [root@etcd ~]# etcdctl endpoint health https://192.168.30.31:2379 is healthy: successfully committed proposal: took = 16.707114ms# win on Ubuntu 操作 #创建kube-apiserver ca证书配置 mkdir -p /apps/work/k8s/cfssl/k8s cat << EOF | tee /apps/work/k8s/cfssl/k8s/k8s-ca-csr.json { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF #生成 Kubernetes CA 证书和私钥 mkdir -p /apps/work/k8s/cfssl/pki/k8s cfssl gencert -initca /apps/work/k8s/cfssl/k8s/k8s-ca-csr.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/k8s-ca #创建 Kubernetes API Server 证书配置文件 export K8S_APISERVER_VIP=" \ \"192.168.30.32\" \ " && \ export K8S_APISERVER_SERVICE_CLUSTER_IP="10.66.0.1" && \ export K8S_APISERVER_HOSTNAME="api.k8s.cluster.local" && \ export K8S_CLUSTER_DOMAIN_SHORTNAME="cluster" && \ export K8S_CLUSTER_DOMAIN_FULLNAME="cluster.local" && \ cat << EOF | tee /apps/work/k8s/cfssl/k8s/k8s_apiserver.json { "CN": "kubernetes", "hosts": [ "127.0.0.1", ${K8S_APISERVER_VIP}， "${K8S_APISERVER_SERVICE_CLUSTER_IP}", "${K8S_APISERVER_HOSTNAME}", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.${K8S_CLUSTER_DOMAIN_SHORTNAME}", "kubernetes.default.svc.${K8S_CLUSTER_DOMAIN_FULLNAME}" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF #生成 Kubernetes API Server 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/k8s_apiserver.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/k8s_server # 创建 Kubernetes webhook 证书配置文件 cat << EOF | tee /apps/work/k8s/cfssl/k8s/aggregator.json { "CN": "aggregator", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF ##### 生成 Kubernetes webhook 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/aggregator.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/aggregator # 远程服务器创建目录 mkdir -p /apps/kubernetes/{bin,conf,config,kubelet-plugins,log,ssl} #证书到分发到 192.168.30.32 scp -r /apps/work/k8s/cfssl/pki/k8s 192.168.30.32:/apps/kubernetes/ssl/k8s # cp etcd_client 证书 scp -r /apps/work/k8s/cfssl/pki/etcd/etcd_client* 192.168.30.32:/apps/kubernetes/ssl/etcd scp -r /apps/work/k8s/cfssl/pki/etcd/etcd-ca.pem 192.168.30.32:/apps/kubernetes/ssl/etcd # 分发kubernetes 二进制文件到远程服务器把所有的二进制都cp 过去 scp -r /mnt/e/work/k8s/bin/* 192.168.30.32:/apps/kubernetes/bin # 远程服务器操作192.168.30.32 # 创建 k8s 用户 useradd k8s -s /sbin/nologin -M # encryption-config.yaml 生成 cd /apps/kubernetes/config export ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64) cat > encryption-config.yaml <<EOF kind: EncryptionConfig apiVersion: v1 resources: - resources: - secrets providers: - aescbc: keys: - name: key1 secret: ${ENCRYPTION_KEY} - identity: {} EOF # Apiserver配置文件生成 cd /apps/kubernetes/conf vi kube-apiserver KUBE_APISERVER_OPTS="--logtostderr=false \ --bind-address=192.168.30.32 \ --advertise-address=192.168.30.32 \ --secure-port=5443 \ --insecure-port=0 \ --service-cluster-ip-range=10.66.0.0/16 \ --service-node-port-range=30000-65000 \ --etcd-cafile=/apps/kubernetes/ssl/etcd/etcd-ca.pem \ --etcd-certfile=/apps/kubernetes/ssl/etcd/etcd_client.pem \ --etcd-keyfile=/apps/kubernetes/ssl/etcd/etcd_client-key.pem \ --etcd-prefix=/registry \ --etcd-servers=https://192.168.30.31:2379 \ --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --tls-cert-file=/apps/kubernetes/ssl/k8s/k8s_server.pem \ --tls-private-key-file=/apps/kubernetes/ssl/k8s/k8s_server-key.pem \ --kubelet-client-certificate=/apps/kubernetes/ssl/k8s/k8s_server.pem \ --kubelet-client-key=/apps/kubernetes/ssl/k8s/k8s_server-key.pem \ --service-account-key-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --requestheader-client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --proxy-client-cert-file=/apps/kubernetes/ssl/k8s/aggregator.pem \ --proxy-client-key-file=/apps/kubernetes/ssl/k8s/aggregator-key.pem \ --requestheader-allowed-names=aggregator \ --requestheader-group-headers=X-Remote-Group \ --requestheader-extra-headers-prefix=X-Remote-Extra- \ --requestheader-username-headers=X-Remote-User \ --enable-aggregator-routing=true \ --anonymous-auth=false \ --allow-privileged=true \ --experimental-encryption-provider-config=/apps/kubernetes/config/encryption-config.yaml \ --enable-admission-plugins=AlwaysPullImages,DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection MutatingAdmissionWebhook ValidatingAdmissionWebhook \ --disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy \ --cors-allowed-origins=.* \ --enable-swagger-ui \ --runtime-config=api/all=true \ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \ --authorization-mode=Node,RBAC \ --apiserver-count=1 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --kubelet-https \ --event-ttl=1h \ --feature-gates=RotateKubeletServerCertificate=true,RotateKubeletClientCertificate=true \ --enable-bootstrap-token-auth=true \ --audit-log-path=/apps/kubernetes/log/api-server-audit.log \ --alsologtostderr=true \ --log-dir=/apps/kubernetes/log \ --v=2 \ --endpoint-reconciler-type=lease \ --max-mutating-requests-inflight=100 \ --max-requests-inflight=500 \ --target-ram-mb=6000" # kube-apiserver systemd文件 vi /usr/lib/systemd/system/kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes [Service] Type=notify LimitNOFILE=1024000 LimitNPROC=1024000 LimitCORE=infinity LimitMEMLOCK=infinity EnvironmentFile=-/apps/kubernetes/conf/kube-apiserver ExecStart=/apps/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS Restart=on-failure RestartSec=5 User=k8s [Install] WantedBy=multi-user.target # /apps/kubernetes 目录k8s 权限 chown -R k8s.k8s /apps/kubernetes # 启动kube-apiserver systemctl start kube-apiserver # 设置开机启动 systemctl enable kube-apiserver #创建admin管理员证书 win on Ubuntu 操作 cat << EOF | tee /apps/work/k8s/cfssl/k8s/k8s_apiserver_admin.json { "CN": "admin", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "system:masters", "OU": "Kubernetes-manual" } ] } EOF cfssl gencert -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/k8s_apiserver_admin.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/k8s_apiserver_admin # 创建admin管理员登录 admin.kubeconfig export KUBE_APISERVER="https://192.168.30.32:5443" kubectl config set-cluster kubernetes \ --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=admin.kubeconfig kubectl config set-credentials admin \ --client-certificate=/apps/work/k8s/cfssl/pki/k8s/k8s_apiserver_admin.pem \ --client-key=/apps/work/k8s/cfssl/pki/k8s/k8s_apiserver_admin-key.pem \ --embed-certs=true \ --kubeconfig=admin.kubeconfig kubectl config set-context kubernetes \ --cluster=kubernetes \ --user=admin \ --namespace=kube-system \ --kubeconfig=admin.kubeconfig kubectl config use-context kubernetes --kubeconfig=admin.kubeconfig # cp 证书到当前用户目录 cp admin.kubeconfig ~/.kube/config # 验证kube-apiserver 是否正常 kubectl cluster-info [root@]~]#kubectl cluster-info Kubernetes master is running at https://192.168.30.32:5443# 生成kube_scheduler访问kube-apiserver 证书win on Ubuntu 操作 cat << EOF | tee /apps/work/k8s/cfssl/k8s/k8s_scheduler.json { "CN": "system:kube-scheduler", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "system:kube-scheduler", "OU": "Kubernetes-manual" } ] } EOF ## 生成 Kubernetes Scheduler 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/k8s_scheduler.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/k8s_scheduler #创建kube_scheduler.kubeconfig kubectl config set-cluster kubernetes \ --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube_scheduler.kubeconfig kubectl config set-credentials system:kube-scheduler \ --client-certificate=/apps/work/k8s/cfssl/pki/k8s/k8s_scheduler.pem \ --embed-certs=true \ --client-key=/apps/work/k8s/cfssl/pki/k8s/k8s_scheduler-key.pem \ --kubeconfig=kube_scheduler.kubeconfig kubectl config set-context kubernetes \ --cluster=kubernetes \ --user=system:kube-scheduler \ --kubeconfig=kube_scheduler.kubeconfig kubectl config use-context kubernetes --kubeconfig=kube_scheduler.kubeconfig # cp kube_scheduler.kubeconfig 到远程服务器 scp kube_scheduler.kubeconfig 192.168.30.32:/apps/kubernetes/config # 远程服务器操作 cd /apps/kubernetes/conf # 创建kube-scheduler 启动配置文件 vi kube-scheduler KUBE_SCHEDULER_OPTS=" \ --logtostderr=false \ --address=0.0.0.0 \ --leader-elect=true \ --kubeconfig=/apps/kubernetes/config/kube_scheduler.kubeconfig \ --authentication-kubeconfig=/apps/kubernetes/config/kube_scheduler.kubeconfig \ --authorization-kubeconfig=/apps/kubernetes/config/kube_scheduler.kubeconfig \ --alsologtostderr=true \ --kube-api-qps=100 \ --kube-api-burst=100 \ --log-dir=/apps/kubernetes/log \ --v=2" # 创建/kube-scheduler 启动文件 vi /usr/lib/systemd/system/kube-scheduler.service [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/kubernetes/kubernetes [Service] LimitNOFILE=1024000 LimitNPROC=1024000 LimitCORE=infinity LimitMEMLOCK=infinity EnvironmentFile=-/apps/kubernetes/conf/kube-scheduler ExecStart=/apps/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS Restart=on-failure RestartSec=5 User=k8s [Install] WantedBy=multi-user.target # 给新创建文件 k8s 用户权限 chown -R k8s.k8s /apps/kubernetes # 启动 kube-scheduler systemctl start kube-scheduler # 设置开启启动 systemctl enable kube-scheduler # 验证状态 kubectl get cs [root@]~]# kubectl get cs NAME STATUS MESSAGE ERROR controller-manager Healthy ok scheduler Healthy ok etcd-0 Healthy {"health":"true"}# 生成kube-controller-manager访问kube-apiserver 证书 win on Ubuntu 操作 cat << EOF | tee /apps/work/k8s/cfssl/k8s/k8s_controller_manager.json { "CN": "system:kube-controller-manager", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "system:kube-controller-manager", "OU": "Kubernetes-manual" } ] } EOF ## 生成 Kubernetes Controller Manager 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/k8s_controller_manager.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/k8s_controller_manager # 创建kube_controller_manager.kubeconfig kubectl config set-cluster kubernetes \ --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube_controller_manager.kubeconfig kubectl config set-credentials system:kube-controller-manager \ --client-certificate=/apps/work/k8s/cfssl/pki/k8s/k8s_controller_manager.pem \ --embed-certs=true \ --client-key=/apps/work/k8s/cfssl/pki/k8s/k8s_controller_manager-key.pem \ --kubeconfig=kube_controller_manager.kubeconfig kubectl config set-context kubernetes \ --cluster=kubernetes \ --user=system:kube-controller-manager \ --kubeconfig=kube_controller_manager.kubeconfig kubectl config use-context kubernetes --kubeconfig=kube_controller_manager.kubeconfig # cp kube_controller_manager.kubeconfig 到远程服务器 scp kube_controller_manager.kubeconfig 192.168.30.32:/apps/kubernetes/config # 远程服务器操作 cd /apps/kubernetes/conf vi kube-controller-manager KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \ --leader-elect=true \ --address=0.0.0.0 \ --service-cluster-ip-range=10.66.0.0/16 \ --cluster-cidr=10.67.0.0/16 \ --node-cidr-mask-size=24 \ --cluster-name=kubernetes \ --allocate-node-cidrs=true \ --kubeconfig=/apps/kubernetes/config/kube_controller_manager.kubeconfig \ --authentication-kubeconfig=/apps/kubernetes/config/kube_controller_manager.kubeconfig \ --authorization-kubeconfig=/apps/kubernetes/config/kube_controller_manager.kubeconfig \ --use-service-account-credentials=true \ --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --requestheader-client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --node-monitor-grace-period=40s \ --node-monitor-period=5s \ --pod-eviction-timeout=5m0s \ --terminated-pod-gc-threshold=50 \ --alsologtostderr=true \ --cluster-signing-cert-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --cluster-signing-key-file=/apps/kubernetes/ssl/k8s/k8s-ca-key.pem \ --deployment-controller-sync-period=10s \ --experimental-cluster-signing-duration=86700h0m0s \ --enable-garbage-collector=true \ --root-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --service-account-private-key-file=/apps/kubernetes/ssl/k8s/k8s-ca-key.pem \ --feature-gates=RotateKubeletServerCertificate=true,RotateKubeletClientCertificate=true \ --controllers=*,bootstrapsigner,tokencleaner \ --horizontal-pod-autoscaler-use-rest-clients=true \ --horizontal-pod-autoscaler-sync-period=10s \ --flex-volume-plugin-dir=/apps/kubernetes/kubelet-plugins/volume \ --tls-cert-file=/apps/kubernetes/ssl/k8s/k8s_controller_manager.pem \ --tls-private-key-file=/apps/kubernetes/ssl/k8s/k8s_controller_manager-key.pem \ --kube-api-qps=100 \ --kube-api-burst=100 \ --log-dir=/apps/kubernetes/log \ --v=2" # 创建启动文件kube-controller-manager vi /usr/lib/systemd/system/kube-controller-manager.service [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/kubernetes/kubernetes [Service] LimitNOFILE=1024000 LimitNPROC=1024000 LimitCORE=infinity LimitMEMLOCK=infinity EnvironmentFile=-/apps/kubernetes/conf/kube-controller-manager ExecStart=/apps/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS Restart=on-failure RestartSec=5 User=k8s [Install] WantedBy=multi-user.target # 给新创建文件 k8s 用户权限 chown -R k8s.k8s /apps/kubernetes # 启动kube-controller-manager systemctl start kube-controller-manager # 设置开机启动 systemctl enable kube-controller-manager # 验证状态 [root@]~]#kubectl get cs NAME STATUS MESSAGE ERROR scheduler Healthy ok controller-manager Healthy ok etcd-0 Healthy {"health":"true"} # 配置 kube-controller-manager，kubelet 、kube-scheduler 访问kube-api 用户授权 授予 kubernetes API 的权限 kubectl create clusterrolebinding controller-node-clusterrolebing --clusterrole=system:kube-controller-manager --user=system:kube-controller-manager kubectl create clusterrolebinding scheduler-node-clusterrolebing --clusterrole=system:kube-scheduler --user=system:kube-scheduler kubectl create clusterrolebinding controller-manager:system:auth-delegator --user system:kube-controller-manager --clusterrole system:auth-delegator 授予 kubernetes 证书访问 kubelet API 的权限 kubectl create clusterrolebinding --user system:serviceaccount:kube-system:default kube-system-cluster-admin --clusterrole cluster-admin kubectl create clusterrolebinding kubelet-node-clusterbinding --clusterrole=system:node --group=system:nodes kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes# 远程服务器节点操作 # 使用阿里源 cat > /etc/yum.repos.d/docker-ce.repo << EOF [docker-ce-stable] name=Docker CE Stable - \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/\$basearch/stable enabled=1 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-stable-debuginfo] name=Docker CE Stable - Debuginfo \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-\$basearch/stable enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-stable-source] name=Docker CE Stable - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/stable enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-edge] name=Docker CE Edge - \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/\$basearch/edge enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-edge-debuginfo] name=Docker CE Edge - Debuginfo \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-\$basearch/edge enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-edge-source] name=Docker CE Edge - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/edge enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-test] name=Docker CE Test - \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/\$basearch/test enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-test-debuginfo] name=Docker CE Test - Debuginfo \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-\$basearch/test enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-test-source] name=Docker CE Test - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/test enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-nightly] name=Docker CE Nightly - \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/\$basearch/nightly enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-nightly-debuginfo] name=Docker CE Nightly - Debuginfo \$basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-\$basearch/nightly enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-nightly-source] name=Docker CE Nightly - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/nightly enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg EOF # 安装docker依赖 yum install -y python-pip python-devel yum-utils device-mapper-persistent-data lvm2 # 安装docker yum install -y docker-ce # 修改docker 启动配置 vi /lib/systemd/system/docker.service ExecStart= 修改成 ExecStart=/usr/bin/dockerd -H fd:// --graph /apps/docker -H unix:///var/run/docker.sock --max-concurrent-downloads=20 --log-opt max-size=200M --log-opt max-file=10 --default-ulimit nofile=1024000 --default-ulimit nproc=1024000 # reload service 配置 systemctl daemon-reload # 重启docker systemctl restart docker # 设置开机启动 systemctl enable docker# 远程服务器节点操作 cni 插件安装 mkdir -p /apps/cni/bin cd /apps/cni/bin wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz tar -xvf cni-plugins-linux-amd64-v0.8.2.tgz rm -f cni-plugins-linux-amd64-v0.8.2.tgz # 创建/etc/cni/net.d 目录 mkdir -p /etc/cni/net.d vi /etc/cni/net.d/10-kuberouter.conflist { "cniVersion":"0.3.0", "name":"mynet", "plugins":[ { "name":"kubernetes", "type":"bridge", "bridge":"kube-bridge", "isDefaultGateway":true, "ipam":{ "type":"host-local" } }, { "type":"portmap", "capabilities":{ "snat":true, "portMappings":true } } ] } # lxcfs 安装 yum install -y git automake libtool fuse-devel git clone git://github.com/lxc/lxcfs cd lxcfs/ ./bootstrap.sh ./configure make make install # 创建目录 mkdir -p /var/lib/lxcfs/ # 配置 lxcfs 启动文件 vi /usr/lib/systemd/system/lxcfs.service [Unit] Description=FUSE filesystem for LXC ConditionVirtualization=!container Before=lxc.service Documentation=man:lxcfs(1) [Service] ExecStart=/usr/local/bin/lxcfs /var/lib/lxcfs/ KillMode=process Restart=on-failure ExecStopPost=-/bin/fusermount -u /var/lib/lxcfs Delegate=yes [Install] WantedBy=multi-user.target # 启动lxcfs systemctl start lxcfs # 设置开机启动 systemctl enable lxcfs # 安装 kubelet 依赖 yum install -y epel-release yum install -y yum-utils ipvsadm telnet wget net-tools conntrack ipset jq iptables curl sysstat libseccomp socat nfs-utils fuse fuse-devel # win on Ubuntu 操作 # 生成 bootstrap Token # Bootstrap Token 生成 echo "$(head -c 6 /dev/urandom | md5sum | head -c 6)"."$(head -c 16 /dev/urandom | md5sum | head -c 16)" 9dad00.2ac445bf1cc5e9c2 vi bootstrap.secret.yaml apiVersion: v1 kind: Secret metadata: # Name MUST be of form "bootstrap-token-<token id>" name: bootstrap-token-9dad00 namespace: kube-system # Type MUST be 'bootstrap.kubernetes.io/token' type: bootstrap.kubernetes.io/token stringData: # Human readable description. Optional. description: "The default bootstrap token generated by 'kubelet '." # Token ID and secret. Required. token-id: 9dad00 token-secret: 2ac445bf1cc5e9c2 # Allowed usages. usage-bootstrap-authentication: "true" usage-bootstrap-signing: "true" # Extra groups to authenticate the token as. Must start with "system:bootstrappers:" auth-extra-groups: system:bootstrappers:worker,system:bootstrappers:ingress ### 创建k8s资源 kubectl create -f bootstrap.secret.yaml ### 创建bootstrap.clusterrole.yaml vi bootstrap.clusterrole.yaml # A ClusterRole which instructs the CSR approver to approve a node requesting a # serving cert matching its client cert. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:certificates.k8s.io:certificatesigningrequests:selfnodeserver rules: - apiGroups: ["certificates.k8s.io"] resources: ["certificatesigningrequests/selfnodeserver"] verbs: ["create"] kubectl create -f bootstrap.clusterrole.yaml ### 创建 apiserver-to-kubelet.yaml vi apiserver-to-kubelet.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: kubernetes.io/bootstrapping: rbac-defaults name: system:kubernetes-to-kubelet rules: - apiGroups: - "" resources: - nodes/proxy - nodes/stats - nodes/log - nodes/spec - nodes/metrics verbs: - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:kubernetes namespace: "" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:kubernetes-to-kubelet subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: kubernetes kubectl create -f apiserver-to-kubelet.yaml ### 查看创建的token kubeadm token list # 允许 system:bootstrappers 组用户创建 CSR 请求 kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:bootstrappers # 自动批准 system:bootstrappers 组用户 TLS bootstrapping 首次申请证书的 CSR 请求 kubectl create clusterrolebinding node-client-auto-approve-csr --clusterrole=system:certificates.k8s.io:certificatesigningrequests:nodeclient --group=system:bootstrappers # 自动批准 system:nodes 组用户更新 kubelet 自身与 apiserver 通讯证书的 CSR 请求 kubectl create clusterrolebinding node-client-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeclient --group=system:nodes # 自动批准 system:nodes 组用户更新 kubelet 10250 api 端口证书的 CSR 请求 kubectl create clusterrolebinding node-server-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeserver --group=system:nodes # 创建bootstrap.kubeconfig # 设置集群参数 kubectl config set-cluster kubernetes \ --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=bootstrap.kubeconfig # 设置客户端认证参数 kubectl config set-credentials system:bootstrap:9dad00 \ --token=9dad00.2ac445bf1cc5e9c2 \ --kubeconfig=bootstrap.kubeconfig # 设置上下文参数 kubectl config set-context default \ --cluster=kubernetes \ --user=system:bootstrap:9dad00 \ --kubeconfig=bootstrap.kubeconfig # 设置默认上下文 kubectl config use-context default --kubeconfig=bootstrap.kubeconfig # 分发 bootstrap.kubeconfig 到远程节点 scp bootstrap.kubeconfig 192.168.30.32:/apps/kubernetes/conf # 创建kubelet 启动配置文件 远程节点 操作 cd /apps/kubernetes/conf vi kubelet KUBELET_OPTS="--bootstrap-kubeconfig=/apps/kubernetes/conf/bootstrap.kubeconfig \ --fail-swap-on=false \ --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/apps/cni/bin \ --kubeconfig=/apps/kubernetes/conf/kubelet.kubeconfig \ --address=192.168.30.32 \ --node-ip=192.168.30.32 \ --hostname-override=master \ --cluster-dns=10.66.0.2 \ --cluster-domain=cluster.local \ --authorization-mode=Webhook \ --authentication-token-webhook=true \ --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --rotate-certificates=true \ --cgroup-driver=cgroupfs \ --healthz-port=10248 \ --healthz-bind-address=192.168.30.32 \ --cert-dir=/apps/kubernetes/ssl \ --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \ --node-labels=node-role.kubernetes.io/k8s-node=true \ --serialize-image-pulls=false \ --enforce-node-allocatable=pods,kube-reserved,system-reserved \ --pod-manifest-path=/apps/work/kubernetes/manifests \ --runtime-cgroups=/systemd/system.slice/kubelet.service \ --kube-reserved-cgroup=/systemd/system.slice/kubelet.service \ --system-reserved-cgroup=/systemd/system.slice \ --root-dir=/apps/work/kubernetes/kubelet \ --log-dir=/apps/kubernetes/log \ --alsologtostderr=true \ --logtostderr=false \ --anonymous-auth=true \ --image-gc-high-threshold=70 \ --image-gc-low-threshold=50 \ --kube-reserved=cpu=500m,memory=512Mi,ephemeral-storage=1Gi \ --system-reserved=cpu=1000m,memory=1024Mi,ephemeral-storage=1Gi \ --eviction-hard=memory.available<500Mi,nodefs.available<10% \ --serialize-image-pulls=false \ --sync-frequency=30s \ --resolv-conf=/etc/resolv.conf \ --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 \ --image-pull-progress-deadline=30s \ --v=2 \ --event-burst=30 \ --event-qps=15 \ --kube-api-burst=30 \ --kube-api-qps=15 \ --max-pods=200 \ --pods-per-core=10 \ --read-only-port=0 \ --allowed-unsafe-sysctls 'kernel.msg*,kernel.shm*,kernel.sem,fs.mqueue.*,net.*' \ --volume-plugin-dir=/apps/kubernetes/kubelet-plugins/volume" # 创建 kubelet 服务文件 vi /usr/lib/systemd/system/kubelet.service [Unit] Description=Kubernetes Kubelet After=docker.service Requires=docker.service [Service] LimitNOFILE=1024000 LimitNPROC=1024000 LimitCORE=infinity LimitMEMLOCK=infinity EnvironmentFile=-/apps/kubernetes/conf/kubelet ExecStart=/apps/kubernetes/bin/kubelet $KUBELET_OPTS Restart=on-failure KillMode=process [Install] WantedBy=multi-user.target # kubernetes 工作目录 mkdir -p /apps/work/kubernetes/{manifests,kubelet} # 启动 kubelet systemctl start kubelet # 设置开机启动 systemctl enable kubelet #查看证书是否签发 cd /apps/kubernetes/ssl/ [root@master ssl]# ll total 12 drwxr-xr-x 2 k8s k8s 75 Jul 4 15:06 etcd drwxr-xr-x 2 k8s k8s 310 Sep 3 11:18 k8s -rw------- 1 k8s k8s 1277 Sep 3 10:25 kubelet-client-2019-09-03-10-25-10.pem lrwxrwxrwx 1 k8s k8s 59 Sep 3 10:25 kubelet-client-current.pem -> /apps/kubernetes/ssl/kubelet-client-2019-09-03-10-25-10.pem -rw-r--r-- 1 k8s k8s 2153 Sep 3 10:25 kubelet.crt -rw------- 1 k8s k8s 1675 Sep 3 10:25 kubelet.key # 查看节点是否正常 kubectl get node [root@]~]#kubectl get node NAME STATUS ROLES AGE VERSION master Ready k8s-node 27h v1.15.3# win on Ubuntu 操作 # 创建kube-router 访问kube-apiserver 证书 cat << EOF | tee /apps/work/k8s/cfssl/k8s/kube-router.json { "CN": "kube-router", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "system:masters", "OU": "Kubernetes-manual" } ] } EOF ## 生成 kube-router 证书和私钥 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/kube-router.json | \ cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/kube-router # 设置集群参数 kubectl config set-cluster kubernetes \ --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kubeconfig.conf # 设置客户端认证参数 kubectl config set-credentials kube-router \ --client-certificate=/apps/work/k8s/cfssl/pki/k8s/kube-router.pem \ --client-key=/apps/work/k8s/cfssl/pki/k8s/kube-router-key.pem \ --embed-certs=true \ --kubeconfig=kubeconfig.conf # 设置上下文参数 kubectl config set-context default \ --cluster=kubernetes \ --user=kube-router \ --kubeconfig=kubeconfig.conf # 设置默认上下文 kubectl config use-context default --kubeconfig=kubeconfig.conf # 创建kube-router configmap kubectl create configmap "kube-proxy" --from-file=kubeconfig.conf # 创建kubeadm-kuberouter-all-features-hostport.yaml vi kubeadm-kuberouter-all-features-hostport.yaml apiVersion: v1 kind: ConfigMap metadata: name: kube-router-cfg namespace: kube-system labels: tier: node k8s-app: kube-router data: cni-conf.json: | { "cniVersion":"0.3.0", "name":"mynet", "plugins":[ { "name":"kubernetes", "type":"bridge", "bridge":"kube-bridge", "isDefaultGateway":true, "ipam":{ "type":"host-local" } }, { "type":"portmap", "capabilities":{ "snat":true, "portMappings":true } } ] } --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: k8s-app: kube-router tier: node name: kube-router namespace: kube-system spec: selector: matchLabels: k8s-app: kube-router template: metadata: labels: k8s-app: kube-router tier: node annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: kube-router serviceAccount: kube-router containers: - name: kube-router image: docker.io/cloudnativelabs/kube-router imagePullPolicy: Always args: - --run-router=true - --run-firewall=true - --run-service-proxy=true - --advertise-cluster-ip=true - --advertise-loadbalancer-ip=true - --advertise-pod-cidr=true - --advertise-external-ip=true - --cluster-asn=64512 - --metrics-path=/metrics - --metrics-port=20241 - --enable-cni=true - --enable-ibgp=true - --enable-overlay=true - --nodeport-bindon-all-ip=true - --nodes-full-mesh=true - --enable-pod-egress=true - --cluster-cidr=10.67.0.0/16 - --v=2 - --kubeconfig=/var/lib/kube-router/kubeconfig env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: KUBE_ROUTER_CNI_CONF_FILE value: /etc/cni/net.d/10-kuberouter.conflist livenessProbe: httpGet: path: /healthz port: 20244 initialDelaySeconds: 10 periodSeconds: 3 resources: requests: cpu: 250m memory: 250Mi securityContext: privileged: true volumeMounts: - name: lib-modules mountPath: /lib/modules readOnly: true - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kubeconfig mountPath: /var/lib/kube-router readOnly: true initContainers: - name: install-cni image: busybox imagePullPolicy: Always command: - /bin/sh - -c - set -e -x; if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then if [ -f /etc/cni/net.d/*.conf ]; then rm -f /etc/cni/net.d/*.conf; fi; TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; fi volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router hostNetwork: true tolerations: - key: CriticalAddonsOnly operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists - effect: NoSchedule key: node.kubernetes.io/not-ready operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/ingress operator: Equal volumes: - name: lib-modules hostPath: path: /lib/modules - name: cni-conf-dir hostPath: path: /etc/cni/net.d - name: kube-router-cfg configMap: name: kube-router-cfg - name: kubeconfig configMap: name: kube-proxy items: - key: kubeconfig.conf path: kubeconfig --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-router namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: kube-router namespace: kube-system rules: - apiGroups: - "" resources: - namespaces - pods - services - nodes - endpoints verbs: - list - get - watch - apiGroups: - "networking.k8s.io" resources: - networkpolicies verbs: - list - get - watch - apiGroups: - extensions resources: - networkpolicies verbs: - get - list - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: kube-router roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: kube-router subjects: - kind: ServiceAccount name: kube-router namespace: kube-system # 创建 kube-router 服务 kubectl apply -f kubeadm-kuberouter-all-features-hostport.yaml # 查看服务是否成功创建 [root@]~]#kubectl get pod -A | grep kube-router kube-system kube-router-5tmgw 1/1 Running 0 21h # 进入192.168.30.32 cat /etc/cni/net.d/10-kuberouter.conflist [root@master ssl]# cat /etc/cni/net.d/10-kuberouter.conflist {"cniVersion":"0.3.0","name":"mynet","plugins":[{"bridge":"kube-bridge","ipam":{"subnet":"10.67.0.0/24","type":"host-local"},"isDefaultGateway":true,"name":"kubernetes","type":"bridge"},{"capabilities":{"portMappings":true,"snat":true},"type":"portmap"}]} # 已经获取到ip段 ip a| grep kube [root@master ssl]# ip a| grep kube 4: kube-bridge: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 inet 10.67.0.1/24 brd 10.67.0.255 scope global kube-bridge 6: kube-dummy-if: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default inet 10.66.0.1/32 brd 10.66.0.1 scope link kube-dummy-if inet 10.66.0.2/32 brd 10.66.0.2 scope link kube-dummy-if inet 10.66.91.125/32 brd 10.66.91.125 scope link kube-dummy-if inet 10.66.86.10/32 brd 10.66.86.10 scope link kube-dummy-if inet 10.66.52.216/32 brd 10.66.52.216 scope link kube-dummy-if # kube-router 部署正常# win on Ubuntu 操作 vi coredns.yaml # __MACHINE_GENERATED_WARNING__ apiVersion: v1 kind: ServiceAccount metadata: name: coredns namespace: kube-system labels: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: kubernetes.io/bootstrapping: rbac-defaults addonmanager.kubernetes.io/mode: Reconcile name: system:coredns rules: - apiGroups: - "" resources: - endpoints - services - pods - namespaces verbs: - list - watch - apiGroups: - "" resources: - nodes verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: kubernetes.io/bootstrapping: rbac-defaults addonmanager.kubernetes.io/mode: EnsureExists name: system:coredns roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:coredns subjects: - kind: ServiceAccount name: coredns namespace: kube-system --- apiVersion: v1 kind: ConfigMap metadata: name: coredns namespace: kube-system labels: addonmanager.kubernetes.io/mode: EnsureExists data: Corefile: | .:53 { errors health kubernetes cluster.local in-addr.arpa ip6.arpa { pods insecure upstream /etc/resolv.conf fallthrough in-addr.arpa ip6.arpa } prometheus :9153 forward . /etc/resolv.conf cache 30 reload loadbalance } --- apiVersion: apps/v1 kind: Deployment metadata: name: coredns namespace: kube-system labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "CoreDNS" spec: # replicas: not specified here: # 1. In order to make Addon Manager do not reconcile this replicas parameter. # 2. Default is 1. # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. strategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 selector: matchLabels: k8s-app: kube-dns template: metadata: labels: k8s-app: kube-dns annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical serviceAccountName: coredns tolerations: - key: "CriticalAddonsOnly" operator: "Exists" nodeSelector: beta.kubernetes.io/os: linux containers: - name: coredns image: coredns/coredns imagePullPolicy: Always resources: limits: memory: 170Mi requests: cpu: 100m memory: 70Mi args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume mountPath: /etc/coredns readOnly: true ports: - containerPort: 53 name: dns protocol: UDP - containerPort: 53 name: dns-tcp protocol: TCP - containerPort: 9153 name: metrics protocol: TCP livenessProbe: httpGet: path: /health port: 8080 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 readinessProbe: httpGet: path: /health port: 8080 scheme: HTTP securityContext: allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - all readOnlyRootFilesystem: true dnsPolicy: Default volumes: - name: config-volume configMap: name: coredns items: - key: Corefile path: Corefile --- apiVersion: v1 kind: Service metadata: name: kube-dns namespace: kube-system annotations: prometheus.io/port: "9153" prometheus.io/scrape: "true" labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "CoreDNS" spec: selector: k8s-app: kube-dns clusterIP: 10.66.0.2 ports: - name: dns port: 53 protocol: UDP - name: dns-tcp port: 53 protocol: TCP - name: metrics port: 9153 protocol: TCP # 创建 CoreDNS dns 服务 kubectl apply -f coredns.yaml # 验证服务 [root@]~]#kubectl get all -A | grep coredns kube-system pod/coredns-597b77445b-fhxvr 1/1 Running 0 27h kube-system deployment.apps/coredns 1/1 1 1 27h kube-system replicaset.apps/coredns-597b77445b 1 1 1 27h dig @10.66.0.2 www.baidu.com [root@master ssl]# dig @10.66.0.2 www.baidu.com ; <<>> DiG 9.9.4-RedHat-9.9.4-74.el7_6.1 <<>> @10.66.0.2 www.baidu.com ; (1 server found) ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 40347 ;; flags: qr rd ra; QUERY: 1, ANSWER: 3, AUTHORITY: 13, ADDITIONAL: 27 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 4096 ;; QUESTION SECTION: ;www.baidu.com. IN A ;; ANSWER SECTION: www.baidu.com. 30 IN CNAME www.a.shifen.com. www.a.shifen.com. 30 IN A 14.215.177.38 www.a.shifen.com. 30 IN A 14.215.177.39 ;; AUTHORITY SECTION: com. 30 IN NS h.gtld-servers.net. com. 30 IN NS m.gtld-servers.net. com. 30 IN NS g.gtld-servers.net. com. 30 IN NS d.gtld-servers.net. com. 30 IN NS a.gtld-servers.net. com. 30 IN NS j.gtld-servers.net. com. 30 IN NS c.gtld-servers.net. com. 30 IN NS l.gtld-servers.net. com. 30 IN NS b.gtld-servers.net. com. 30 IN NS f.gtld-servers.net. com. 30 IN NS k.gtld-servers.net. com. 30 IN NS i.gtld-servers.net. com. 30 IN NS e.gtld-servers.net. ;; ADDITIONAL SECTION: e.gtld-servers.net. 30 IN AAAA 2001:502:1ca1::30 a.gtld-servers.net. 30 IN A 192.5.6.30 i.gtld-servers.net. 30 IN AAAA 2001:503:39c1::30 c.gtld-servers.net. 30 IN A 192.26.92.30 g.gtld-servers.net. 30 IN AAAA 2001:503:eea3::30 m.gtld-servers.net. 30 IN A 192.55.83.30 d.gtld-servers.net. 30 IN A 192.31.80.30 a.gtld-servers.net. 30 IN AAAA 2001:503:a83e::2:30 b.gtld-servers.net. 30 IN A 192.33.14.30 b.gtld-servers.net. 30 IN AAAA 2001:503:231d::2:30 i.gtld-servers.net. 30 IN A 192.43.172.30 d.gtld-servers.net. 30 IN AAAA 2001:500:856e::30 l.gtld-servers.net. 30 IN A 192.41.162.30 h.gtld-servers.net. 30 IN AAAA 2001:502:8cc::30 e.gtld-servers.net. 30 IN A 192.12.94.30 l.gtld-servers.net. 30 IN AAAA 2001:500:d937::30 k.gtld-servers.net. 30 IN AAAA 2001:503:d2d::30 j.gtld-servers.net. 30 IN AAAA 2001:502:7094::30 m.gtld-servers.net. 30 IN AAAA 2001:501:b1f9::30 f.gtld-servers.net. 30 IN A 192.35.51.30 g.gtld-servers.net. 30 IN A 192.42.93.30 h.gtld-servers.net. 30 IN A 192.54.112.30 j.gtld-servers.net. 30 IN A 192.48.79.30 k.gtld-servers.net. 30 IN A 192.52.178.30 c.gtld-servers.net. 30 IN AAAA 2001:503:83eb::30 f.gtld-servers.net. 30 IN AAAA 2001:503:d414::30 ;; Query time: 6 msec ;; SERVER: 10.66.0.2#53(10.66.0.2) ;; WHEN: Wed Sep 04 14:17:05 CST 2019 ;; MSG SIZE rcvd: 897 dig @10.66.0.2 kube-dns.kube-system.svc.cluster.local [root@master ssl]# dig @10.66.0.2 kube-dns.kube-system.svc.cluster.local ; <<>> DiG 9.9.4-RedHat-9.9.4-74.el7_6.1 <<>> @10.66.0.2 kube-dns.kube-system.svc.cluster.local ; (1 server found) ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 40471 ;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; WARNING: recursion requested but not available ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 4096 ;; QUESTION SECTION: ;kube-dns.kube-system.svc.cluster.local. IN A ;; ANSWER SECTION: kube-dns.kube-system.svc.cluster.local. 5 IN A 10.66.0.2 ;; Query time: 1 msec ;; SERVER: 10.66.0.2#53(10.66.0.2) ;; WHEN: Wed Sep 04 14:19:13 CST 2019 ;; MSG SIZE rcvd: 121 能够正常解析# win on Ubuntu 操作 #申请证书 请使用letsencrypt 申请免费 多域名证书 #重命名证书名字 tls.crt tls.key #创建 secret kubectl -n kube-system create secret generic tls-cert --from-file=tls.key --from-file=tls.crt # 创建traefik 配置 vi traefik.toml defaultEntryPoints = ["http","https"] [entryPoints] [entryPoints.http] address = ":80" entryPoint = "https" [entryPoints.https] address = ":443" [entryPoints.https.tls] [[entryPoints.https.tls.certificates]] certFile = "/certs/tls.crt" keyFile = "/certs/tls.key" # 生成 configmap kubectl create configmap traefik-conf --from-file=traefik.toml -n kube-system # 创建traefik-rbac vi traefik-rbac.yaml --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: traefik namespace: kube-system rules: - apiGroups: - "" resources: - services - endpoints - secrets verbs: - get - list - watch - apiGroups: - extensions resources: - ingresses verbs: - get - list - watch - apiGroups: - extensions resources: - ingresses/status verbs: - update --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: traefik roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: traefik subjects: - kind: ServiceAccount name: traefik namespace: kube-system # traefik-deployment-https vi traefik-deployment-https.yaml --- apiVersion: v1 kind: ServiceAccount metadata: name: traefik namespace: kube-system --- kind: Deployment apiVersion: apps/v1 metadata: name: traefik namespace: kube-system labels: k8s-app: traefik spec: replicas: 1 selector: matchLabels: k8s-app: traefik template: metadata: labels: k8s-app: traefik name: traefik spec: serviceAccountName: traefik terminationGracePeriodSeconds: 60 volumes: - name: ssl secret: secretName: tls-cert - name: config configMap: name: traefik-conf defaultMode: 0644 items: - key: traefik.toml path: traefik.toml hostNetwork: true dnsPolicy: ClusterFirstWithHostNet containers: - image: traefik name: traefik imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /certs name: "ssl" - mountPath: /etc/traefik.toml subPath: traefik.toml name: "config" ports: - name: http containerPort: 80 hostPort: 80 - name: https containerPort: 443 hostPort: 443 - name: admin containerPort: 8080 args: - --api - --web - --api.dashboard - --logLevel=INFO - --web.metrics - --metrics.prometheus - --web.metrics.prometheus - --kubernetes - --traefiklog - --traefiklog.format=json - --accesslog - --accesslog.format=json - --accessLog.fields.headers.defaultMode=redact - --insecureskipverify=true - --configFile=/etc/traefik.toml # nodeSelector: # ingress: "yes" # tolerations: # - effect: NoSchedule # key: node-role.kubernetes.io/ingress # operator: Equal --- kind: Service apiVersion: v1 metadata: labels: k8s-app: traefik name: traefik namespace: kube-system spec: selector: k8s-app: traefik clusterIP: None ports: - protocol: TCP port: 80 name: http - protocol: TCP port: 443 name: https - protocol: TCP port: 8080 name: admin type: ClusterIP # 或者traefik-daemonset-https --- apiVersion: v1 kind: ServiceAccount metadata: name: traefik namespace: kube-system --- kind: DaemonSet apiVersion: apps/v1 metadata: name: traefik namespace: kube-system labels: k8s-app: traefik spec: selector: matchLabels: k8s-app: traefik template: metadata: labels: k8s-app: traefik name: traefik spec: serviceAccountName: traefik terminationGracePeriodSeconds: 60 volumes: - name: ssl secret: secretName: tls-cert - name: config configMap: name: traefik-conf defaultMode: 0644 items: - key: traefik.toml path: traefik.toml hostNetwork: true dnsPolicy: ClusterFirstWithHostNet containers: - image: traefik name: traefik imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /certs name: "ssl" - mountPath: /etc/traefik.toml subPath: traefik.toml name: "config" ports: - name: http containerPort: 80 hostPort: 80 - name: https containerPort: 443 hostPort: 443 - name: admin containerPort: 8080 securityContext: capabilities: drop: - ALL add: - NET_BIND_SERVICE args: - --api - --web - --api.dashboard - --logLevel=INFO - --web.metrics - --metrics.prometheus - --web.metrics.prometheus - --kubernetes - --traefiklog - --traefiklog.format=json - --accesslog - --accesslog.format=json - --accessLog.fields.headers.defaultMode=redact - --insecureskipverify=true - --configFile=/etc/traefik.toml nodeSelector: ingress: "yes" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/ingress operator: Equal --- kind: Service apiVersion: v1 metadata: labels: k8s-app: traefik name: traefik namespace: kube-system spec: selector: k8s-app: traefik clusterIP: None ports: - protocol: TCP port: 80 name: http - protocol: TCP port: 443 name: https - protocol: TCP port: 8080 name: admin type: ClusterIP #创建 traefik-dashboard vi traefik-dashboard.yam apiVersion: extensions/v1beta1 kind: Ingress metadata: name: traefik-dashboard namespace: kube-system annotations: kubernetes.io/ingress.class: traefik traefik.ingress.kubernetes.io/frontend-entry-points: http,https spec: rules: - host: trae.xxx.com http: paths: - backend: serviceName: traefik servicePort: 8080 tls: - secretName: tls-cert # 创建 服务器 kubectl apply -f traefik-deployment-https.yaml kubectl apply -f traefik-rbac.yaml kubectl apply -f traefik-dashboard.yaml hosts 绑定 访问 trae.xxx.com 是否正常 能正常打开证明正常# win on Ubuntu 操作 # 创建 kubernetes-dashboard 使用证书 cat << EOF | tee /apps/work/k8s/cfssl/k8s/dashboard.json { "CN": "dashboard", "hosts": [""], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF ##### 生成kubernetes-dashboard 证书，当然如果有外部签发的证书也可以使用 cfssl gencert \ -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes \ /apps/work/k8s/cfssl/k8s/dashboard.json | \ cfssljson -bare ./dashboard # base64 加密 cat dashboard.pem|base64 | tr -d '\n' cat dashboard-key.pem|base64 | tr -d '\n' # 做好记录 # kubernetes-dashboard vi kubernetes-dashboard.yaml # Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: Namespace metadata: name: kubernetes-dashboard --- apiVersion: v1 kind: ServiceAccount metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard --- kind: Service apiVersion: v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard annotations: kubernetes.io/ingress.class: traefik spec: ports: - port: 443 targetPort: 8443 selector: k8s-app: kubernetes-dashboard --- apiVersion: v1 kind: Secret metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard-certs namespace: kubernetes-dashboard type: Opaque data: dashboard.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBeFlFV0MxbGlqcnFzNW5vcHBxTXF0YzZSY0pnSWFJSGhGemZZUWhRQm5pK0Vjam8vCkRTUkYvY3BUOFlkTTg2MVpEV1lSN1FEelFLNmJUTmRLWXJJYmpVWHJpRFVFU01EUW13Y1VteTMzWjFpeXR6K0wKUUVmTVFvWVNReGVIY2RqUHp3bUhFS0todk9vNmxQTHNFWkMwQ3ZCamw2VHlERjhuSDEzby9kRlRVbGJhWUlGaQpPeGVIWkxMMTZKbmNLK3RVaW9ncjdLekFKMUkxTjdwOVQ1blZ5YU9PbWNCVEFnU3RJM0ZwSzdMZG1zaVU0ZEZ0CkpSSFZ0eTh6Y3dCSU9wWnhqV29mM2ROVkRrVUFsYjVtV2psU0RaQ2lhYmFYQi91NmJ0R0k3RlY2cENaUzdDVG4KeWlpUFlFSXRPSGRCT0VycGpKZWQ0bHQ5K2MvNDE3UTRIaiswdndJREFRQUJBb0lCQVFDK1daSWdjQTZRRnhScQpzVlNST1BNQjlFdXlJNlQrN0NZL2xXQUZGM2tUdHlKRVlTVEJpck0yVFprbjBFbjNGSndlVU1CNEZwRmJScTJBCm1vSWpxeHJveG5taGRjOWlPd3NTVHZtcU1kd2ZLNXBiQ0pBeDdNRE5ZS0FiTDRNbjAxazlaaVpaZnhTNG1WcksKa1hHNTRDZlYzeWR0VU5qRDJiVkFBdWQ2TVJQSDV5QWJTVktsMG9ONkRCaFV4MlYyWEo0WnRUVHE0b3R6VGYxZwp3SjNJeVFjSXl3czE2V3dkeHpuYStqVmpOYU5OQ3ZCT1BMbm9TeXZBQXZGRG9UYmUrMG1tcnZLVmlSeDBDT1FzCkUwNjFtNHY2eUExL3locndkT1BDYXN6SkpjWlYzOThJTzFKb2QxUHk3OU9aT1FpY1FEOGhwQmxqb0FSQ2JlY3QKRFFPcG5CR0JBb0dCQVBhYlJSSGpPTkxIQ25JZWlFQU1EYXNwQXo2RGxRNkQvdWNNdzROdkVPRVNVa3dvQ0p4cApwK1hJeVVzT1B1d2swTzVCcHJRcHZjdGYyWXlLZTFtR25iVUpmUVNWNGpLdWpqb0M0OWhOWk9lSE8zd0xMcnNXCkl1SU1Qeko0TjhxSzl0dUpDQ3BVYUZFVzRiN1R2OGsyK1pJWHJwN3hzNklDd01EUnpTaW9wY0hCQW9HQkFNMEgKQVl1bmdzY3hTM2JnZ05idU5sQ3lIOHBLZFVPbi95cU9IQUdYcG9vZmJUbXJiUUlWN0ZOVSszUTlYc2ErVVE0QwpUbVdFbzhabVhrL3lIV2FDVWxpRkN0ckRhTzNUZVhvb2pia1JyaDcxakFXN0pjVDRVZ1ZwcG1RakFVUW8vOWtVCmxHMUNpOTFZZy94dlV5dHlYM1BnZHJ6SnU2aWNsM1pVZ1h4dzNoWi9Bb0dBZENmY2w3bFVLWXZSTXNHSTRjb0wKb2lRMlAvclFlYjdZa05IbFFZSk9EQVdLT0E3ZlIzVkl2U1lmRWpoS2tRWWlWeWNiTTE4NTQ1SnBNUmFGVlR6ZwpDY2JIV1NLVUlkVXdic2l2czFGNUJza2V6cVdoeEVOLytNTlYvUnE5QkswQjY1UVhBWUV5aFlkbW0zQzN0RG90CndZOWdFOE83SGNONE1ScGhMUmFLeE1FQ2dZRUFoS2E5eHorUUM1VEhRSmlzZzJNSVhWbUIyLzRrdEt0akdvTnIKZDFSSStpQ3ZLSnJUSW9CUXNQSFE1em8xc2R5ODBKV0paNEZUL1MrS1lhdENmbXBmSU1xalpUcjlEcksrYTkwRgpKUEpkZDhaaTIrcGoyM2JXaW8zNmk5dGlIRmx5ZjE4alVUVzNESFVTb0NiZTVzTlBJc2ZkeXZPeXFMcjMvQ1ZjCnlaOU1jYjBDZ1lBMVp2RVM3bU42Nm10T2JpSlR3a3hhaTVvS2tHbDdHTDJkZXJFUmxsc1YrNWRCSVY4dG5DTnAKT2tjMFlMbHV2TEg4cG4zd2VCNzg5dUFCQjNXYmNKcHg0L2NIRm9oZDNhdlR0RThRVjJod0tNS2RKQVBvTHNoMgprK2lEUWd1dmFxSzNmL1RYUW43bWU3dWFqSDk3SXZldXJtWWsvVmRJY0dicnd1SVRzd0FEYWc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= dashboard.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ5ekNDQXQrZ0F3SUJBZ0lVUWRIVXdKS1JYc1ZRb2VYS1JDTjd0eVcwWU04d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2JqRUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2R0WkdSbllXMWxNUkF3RGdZRFZRUUxFd2R0WkdSbllXMWxNUk13CkVRWURWUVFERXdwcmRXSmxjbTVsZEdWek1CNFhEVEU1TURjd05ERXhNVE13TUZvWERUSTVNRGN3TVRFeE1UTXcKTUZvd2JURUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2R0WkdSbllXMWxNUkF3RGdZRFZRUUxFd2R0WkdSbllXMWxNUkl3CkVBWURWUVFERXdsa1lYTm9ZbTloY21Rd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFERmdSWUxXV0tPdXF6bWVpbW1veXExenBGd21BaG9nZUVYTjloQ0ZBR2VMNFJ5T2o4TkpFWDl5bFB4aDB6egpyVmtOWmhIdEFQTkFycHRNMTBwaXNodU5SZXVJTlFSSXdOQ2JCeFNiTGZkbldMSzNQNHRBUjh5Q2hoSkRGNGR4CjJNL1BDWWNRb3FHODZqcVU4dXdSa0xRSzhHT1hwUElNWHljZlhlajkwVk5TVnRwZ2dXSTdGNGRrc3ZYb21kd3IKNjFTS2lDdnNyTUFuVWpVM3VuMVBtZFhKbzQ2WndGTUNCSzBqY1drcnN0MmF5SlRoMFcwbEVkVzNMekp6QUVnNgpsbkdOYWgvZDAxVU9SUUNWdm1aYU9WSU5rS0pwdHBjSCs3cHUwWWpzVlhxa0psTHNKT2ZLS0k5Z1FpMDRkMEU0ClN1bU1sNTNpVzMzNXovalh0RGdlUDdTL0FnTUJBQUdqZ1kwd2dZb3dEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CMEcKQTFVZEpRUVdNQlFHQ0NzR0FRVUZCd01CQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CQWY4RUFqQUFNQjBHQTFVZApEZ1FXQkJURTl6cWx4dkErRXMrbE8zWlFEMlhubGFHRFpqQWZCZ05WSFNNRUdEQVdnQlJ4NEtjQVJjYWtSL2J4Cm13b1RCZURzK3hBb2FUQUxCZ05WSFJFRUJEQUNnZ0F3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUJnWHZwTEMKQjIybXlQaURlZnhsWGNZRzAvY0R2RXlYcTlENWtKTnBxKzFZQ0EvMlp2RDIyN1Q5VjY3aHVyTlA3T2FvSG95Tgo0MHpkR3lZTGRNV3pyZTQwVksxdC84N3pDTENzamt1ZXRCRWEwNVRqUTJhbDRhSzJ6TXl5MkJLWEpYbjlvdkhzCjJwNndvL001eklEOXl2OEhyRkZqWHM3NitTUTFzNXpOdUxuaDBET0Z1SktiZUZxSUJyNmZRbXlsb0l1VURtZjYKcGtQYkJyRnJpNHFGS0lDcVZKRCt3Z01zRFBiclVMZXF5NWlBVjNqRzJKMFgxOE4zdklCeUFwdWhZbjNudlV0TwpLREVIWkFJcFpjRWdqQ2ZLVDNyaERLL3JLN0VFZkxLcGlCdGJya3pFbjVWV3FQUFJEK3ZPU2VySldETDl1K0xyCmhEazlvZ084cmNqQzZGdz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== --- apiVersion: v1 kind: Secret metadata: labels: k8s-app: kubernetes-dashboard name: dashboard-tls-cert namespace: kubernetes-dashboard type: Opaque data: tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUdYekNDQlVlZ0F3SUJBZ0lTQkdVcDlSaVAvK2lNMDVYM0FuY0FUeTg1TUEwR0NTcUdTSWIzRFFFQkN3VUEKTUVveEN6QUpCZ05WQkFZVEFsVlRNUll3RkFZRFZRUUtFdzFNWlhRbmN5QkZibU55ZVhCME1TTXdJUVlEVlFRRApFeHBNWlhRbmN5QkZibU55ZVhCMElFRjFkR2h3Y21sMGVTQllNekFlRncweE9UQTNNRGt3T1RJNU1ESmFGdzB4Ck9URXdNRGN3T1RJNU1ESmFNQll4RkRBU0JnTlZCQU1UQzIxa1pHZGhiV1V1WTI5dE1JSUNJakFOQmdrcWhraUcKOXcwQkFRRUZBQU9DQWc4QU1JSUNDZ0tDQWdFQW9mSVdOdTE4YUp1T3Jzd0JjZE9lODN0dWpXZ2dpUXl0VVYxQwpqNVhYbzNjQTM1L2ZxQXNGVHpJRGNwUmxhTGJ6SHd1d1psOWNSKzJuRENaUzI4VlhZaXcrSkQvQXpna3FzTHFJCjZ3YlFhcHNCa1lYUzRuT1UrZzhSMVgwcm52ckpickE1eHFJSWJKM002ajVLTXZ4RktvMEV3YXNBY2NiYlVGOW4KMHQ2RzNreG4zWW1Sek5HeHh2bXZ4V2prNWNkSWMza0MyT1VuRktGOG5XemJab2JiNk9PUnZSaElEWW5YdjkxdgoyMUYwQnZ0Q21GY0FEaDRqZXUrLzNKVDVLcEJkdkFHOHI3aU1wbkhKaFU1alhqTXlPRytMbkcvcnJuRzJGaXpHCmx1UHQwKzRlK0ZRSXFZY1BUM1cyTUF2ZDlzQTNEMThsUW82M00vZlMyYjNIYVNidFY0b1pmNS9zTzJNeEVPVnoKVEd1M0NxYk40TkcrZE8ycXoxYWxMQmlGZlVjNEdmUVpYRmlLaDFzazl3Qm5zeWhqYUZmdUx6bHRxMDg3STJLYQorVlRaUzFQSlJFbGduM3UwY1FmaENjelF5ZTJ3Vjl6RE9lVmUxeTBjLzZ0RWJhNllCeGR2ZGcwOFpKL0QwYTBLCnJvWlVJMW5Rc2RKeE8rQ3N1OURLYjROZzJCYnZkWVpHVWJrSCtSUDU0UUdrS1VnYnVxNVIwbXI0U1I2VUwrRE4KZjNxem81a3ZiMXVRWXFpaDZYUFVDVUVPOTNOU1Y2MTNUSUVOTUpyYjVhbGRLUkhPZlpWL201QThlUy9ibFFYcgpOV3FCRy9OL2RtckZjMmcyNGJEY3d5OXIzL3FkNy9MTWxmMVRVdzJGczR3M2x2VHJFanlwWEZhQ3BRRGxkc0xJCkYwcWVKVnNDQXdFQUFhT0NBbkV3Z2dKdE1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlZIU1VFRmpBVUJnZ3IKQmdFRkJRY0RBUVlJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWRCZ05WSFE0RUZnUVVHUUNXOGNFbgpaNWhVWjBDa004QW03Wjh7NGJNd0h4WURWUjBqQkJnd0ZvQVVxRXBxWXdSOTNicm0wVG0zcGtWbDcvT283S0V3CmJ3WUlLd1lCQlFVSEFRRUVZekJoTUM0R0NDc0dBUVVGQnpBQmhpSm9kSFJ3T2k4dmIyTnpjQzVwYm5RdGVETXUKYkdWMGMyVnVZM0o1Y0hRdWIzSm5NQzhHQ0NzR0FRVUZCekFDaGlOb2RIUndPaTh3WTJWeWRDNXBiblF0ZURNdQpiR1YwYzJWdVkzSjVjSFF1YjNKbkx6QWxCZ05WSFJFRUhqQWNnZzBxTG0xa1pHZGhiV1V1WTI5dGdndHRaR1JuCllXMWxMbU52YlRCTUJnTlZIU0FFUlRCRE1BZ0dCbWVCREFFQ0FUQTNCZ3NyQmdFRUFZTGZFd0VCQVRBb01DWUcKQ0NzR0FRVUZCd0lCRmhwb2RIUndPaTh3WTNCekxteGxkSE5sYm1OeWVYQjBMbTl5WnpDQ0FRWUdDaXNHQVFRQgoxbmtDQkFJRWdmY0VnZlFBOGdCM0FPSnBTNjRtNk9sQUNlaUdHN1k3ZzlRKzUvNTBpUHVranlpVEFaM2Q4ZHYrCkFBQUJhOVpIamZBQUFBUURBRWd3UmdJaEFKNXBWaDFDSEpmcTFhd2NOYmxEU2FwL1prQmVBeXU5ajcrTVhISnMKTEI3TUFpRUFwM2xLVVNCZXpiQWpodkZWSTBGR3ZFWmtzU2lYKyt3SitiZ3VLOXlaS3JBQWR3QXBQRkdXVk1nNQpaYnFxVVB4WUI5UzNiNzlZZWlseTNLVEREUFRsUlVmMGVBQUFBV3ZXUjQzd0FBQUVBd0JJTUVZQ0lRRDI1L1NHClcrWHRDa2VzaHViekZtUnRnaDUrWXMxaXpnSG5CSmtOS1Z0cE9nSWhBT1lteWJCWjV3RjZBeE5UT29WdnkyYVMKNktEdURyWmRzSVYrN251WkhFSDdNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUNjRHFwTzF3OWdNbzJGaW1GTgpwSUlxT3d1N2hsUWVURU44enY1UmFiYWtGelpvZlhURXpRcGNtSlNWRUhET25MVGpjaWpITWxtbGdIbndTM2w0CjAyWFB0akIzUWJUNFRWUHlqUGpBZ1ZvL1ZmclNJT2N5S1pKRDNJMWxLNXV1anRCdGF3Rnh4cjBYeGd1Q2k5TlUKdlQ2R0RxYnlaVVdiL1I0bXVVYzFwRzMySVJiS3BxQnZveitsaGRMNHdOb1M5YXdiUlg3LzBmUytEZUZiZ09vbgpzYnBDYTFQeFdqWHYwNloxNkF0LzBRTlVZLzExdEw4bTRDK3Q2OW5kOUt6eUdRZmdOank2NmM1RmhIODVBQkNFClJ6L3NoVkdyb1lTQkh4M1Q0c0NKZnh5dW5oK0tVZ0dvRFk5VUc5RzI2T200eHgvWFQ5OTZONTNxUytPS21iY0wKajVJMgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCgotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0KTUlJRWtqQ0NBM3FnQXdJQkFnSVFDZ0ZCUWdBQUFWT0ZjMm9MaGV5bkNEQU5CZ2txaGtpRzl3MEJBUXNGQURBLwpNU1F3SWdZRFZRUUtFeHRFYVdkcGRHRnNJRk5wWjI1aGRIVnlaU0JVY25WemRDQkRieTR4RnpBVkJnTlZCQU1UCkRrUlRWQ0JTYjI5MElFTkJJRmd6TUI0WERURTJNRE14TnpFMk5EQTBObG9YRFRJeE1ETXhOekUyTkRBME5sb3cKU2pFTE1Ba0dBMVVFQmhNQ1ZWTXhGakFVQmdOVkJBb1REVXhsZENkeklFVnVZM0o1Y0hReEl6QWhCZ05WQkFNVApHa3hsZENkeklFVnVZM0o1Y0hRZ1FYVjBhRzl5YVhSNUlGZ3pNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFuTk1NOEZybExrZTNjbDAzZzdOb1l6RHExelVtR1NYaHZiNDE4WENTTDdlNFMwRUYKcTZtZU5RaFk3TEVxeEdpSEM2UGpkZVRtODZkaWNicDVnV0FmMTVHYW4vUFFlR2R4eUdrT2xaSFAvdWFaNldBOApTTXgreWsxM0VpU2RSeHRhNjduc0hqY0FISnlzZTZjRjZzNUs2NzFCNVRhWXVjdjliVHlXYU44aktrS1FESVowClo4aC9wWnE0VW1FVUV6OWw2WUtIeTl2NkRsYjJob256aFQrWGhxK3czQnJ2YXcyVkZuM0VLNkJsc3BrRU5uV0EKYTZ4Szh5dVFTWGd2b3BaUEtpQWxLUVRHZE1EUU1jMlBNVGlWRnJxb003aEQ4YkVmd3pCL29ua3hFejB0TnZqagovUEl6YXJrNU1jV3Z4STBOSFdRV002cjZoQ20yMUF2QTJIM0Rrd0lEQVFBQm80SUJmVENDQVhrd0VnWURWUjBUCkFRSC9CQWd3QmdFQi93SUJBREFPQmdOVkhROEJBZjhFQkFNQ0FZWXdmd1lJS3dZQkJRVUhBUUVFY3pCeE1ESUcKQ0NzR0FRVUZCekFCaGlab2RIUndPaTh3YVhOeVp5NTBjblZ6ZEdsa0xtOWpjM0F1YVdSbGJuUnlkWE4wTG1OdgpiVEE3QmdnckJnRUZCUWN3QW9ZdmFIUjBjRG92TDJGd2NITXVhV1JsYm5SeWRYTjBMbU52YlM5eWIyOTBjeTlrCmMzUnliMjkwWTJGNE15NXdOMk13SHdZRFZSMGpCQmd3Rm9BVXhLZXhwSHNzY2ZyYjRVdVFkZi9FRldDRmlSQXcKVkFZRFZSMGdCRTB3U3pBSUJnWm5nUXdCQWdFd1B3WUxLd1lCQkFHQzN4TUJBUUV3TURBdUJnZ3JCZ0VGQlFjQwpBUllpYUhSMGNEb3ZMMk53Y3k1eWIyOTBMWGd4TG14bGRITmxibU55ZVhCMExtOXlaekE4QmdOVkhSOEVOVEF6Ck1ER2dMNkF0aGl0b2RIUndPaTh3WTNKc0xtbGtaVzUwY25WemRDNWpiMjB2UkZOVVVrOVBWRU5CV0RORFVrd3UKWTNKc01CMEdBMVVkRGdRV0JCU29TbXBqQkgzZHV1YlJPYmVtUldYdjg2anNvVEFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQTNUUFhFZk5qV0RqZEdCWDdDVlcrZGxhNWNFaWxhVWNuZThJa0NKTHhXaDlLRWlrM0pIUlJIR0pvCnVNMlZjR2ZsOTZTOFRpaFJ6WnZvcm9lZDZ0aTZXcUVCbXR6dzNXb2RhdGcrVnlPZXBoNEVZcHIvMXdYS3R4OC8Kd0FwSXZKU3d0bVZpNE1GVTVhTXFyU0RFNmVhNzNNajJ0Y015bzVqTWQ2am1lV1VISzhzby9qb1dVb0hPVWd3dQpYNFBvMVFZeiszZHN6a0RxTXA0ZmtseEJ3WFJzVzEwS1h7UE1UWitzT1BBdmV5eGluZG1qa1c4bEd5K1FzUmxHClBmWitHNlo2aDdtamVtMFkraVdsa1ljVjRQSVdMMWl3Qmk4c2FDYkdTNWpOMnA4TStYK1E3VU5LRWtST2IzTjYKS09xa3FtNTdUSDJIM2VESkFrU25oNi9ETkZ1MFFnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBb2ZJV051MThhSnVPcnN3QmNkT2U4M3R1aldnZ2lReXRVVjFDajVYWG8zY0EzNS9mCnFBc0ZUeklEY3BSbGFMYnpId3V3Wmw5Y1IrMm5EQ1pTMjhWWFlpdytKRC9BemdrcXNMcUk2d2JRYXBzQmtZWFMKNG5PVStnOFIxWDBybnZySmJyQTV4cUlJYkozTTZqNUtNdnhGS28wRXdhc0FjY2JiVUY5bjB0Nkcza3huM1ltUgp6Tkd4eHVtdnhXams1Y2RJYzNrQzJPVW5GS0Y4bld6YlpvYmI2T09SdlJoSURZblh3OTF2MjFGMEJ2dENtRmNBCkRoNGpldSsvM0pUNUtwQmR2QUc4cjdpTXBuSEpoVTVqWGpNeU9HK0xuRy9ycm5HMkZpekdsdVB0MCs0ZStGUUkKcVljUFQzVzJNQXZkOXNBM0QxOGxRbzYzTS9mUzJiM0hhU2J0VjRvWmY1L3NPMk14RU9WelRHdTNDcWJONE5HKwpkTzJxejFhbExCaUZmVWM0R2ZRWlhGaUtoMXNrOXdCbnN5aGphRmZ1THpsdHEwODdJMkthK1ZUWlMxUEpSRWxnCm4zdTBjUWZoQ2N6UXllMndWOXpET2VWZTF5MGMvNnRFYmE2WUJ4ZHZkZzA4WkovRDBhMEtyb1pVSTFuUXNkSngKTytDc3U5REtiNE5nMkJidmRZWkdVYmtIK1JQNTRRR2tLVWdidXE1UjBtcjRTUjZVTCtETmYzcXpvNWt2YjF1UQpZcWloNlhQVUNVRU85M05TVjYxM1RJRU5NSnJiNWFsZEtSSE9mWlYvbTVBOGVTL2JsUVhyTldxQkcvTi9kbXJGCmMyZzI0YkRjd3k5cjMvcWQ3L0xNbGYxVFV3MkZzNHczbHZUckVqeXBYRmFDcFFEbGRzTElGMHFlSlZzQ0F3RUEKQVFLQ0FnQXY5Zk13UnpzTisrdlF4cWd5M3JwM1gzbkpOU3BWakVTVUVTdVNQSTFGWXd3R0xtSGRjWTRiK3pMYwpMeWl0VDJsSEszNE5nM1pmOHZrQzl5S1k1YVBRZGt2ZERtaDZYR3FoTmswd1ZhOUpzeWhPd2JSSHpuVXpiVjBaCnZkMDZVd2x1MTQvMHpLMzBCUFBYOTZTZjN1aFpCclIrNnJiUisxT2VSUE1KbDArWDdFYmliRWlhd1F1R1hsVHAKQVB5eE5FaTNzZ0h2M0VhcnJIdXNYNzNHYW5BY1U3RW9zRlUrZFRGSktEcGxXSVVsUUNwajFYZzF0aVZKMWxFYQo4Wit0UkY0T1BQRjFsUkZLaGU1cHBXSjJWbkVzRjVUZ09xRXc0NHBLbk80Zlo5ZGFhVzRRbTBxSmNtOU5XQTRoCndwSDA3czRmcGt6eG5qU1JsbmFDZDlyandGeVBsSkJzUXNhVlFFNzlpQzJZMTRnTk9KQ0xyMXRKSEQ2ODN3bW4KS3ZNOHZpOTdHTmIybXZHeWNtZnloNVpzTFBpTWNqOFFER3VWZU53dlNESXpybnhqVkZlc0liTWt5UlZRem9IVApTTHRQbXdVR3lwRHVrMDhaZytsT0lYOC85K3lqMER3MDRqenllTVptYlFVdkd2N2lNWjFUaHdaRHF1YkJXV3J4CmtYTmJwTG9BMGxrcHh5bjdGam9Ya20zM2ZKQURjd2xWSS82WFNrSm1FaFVlZmZnaFFSMGNyVGphQVd1Qkx2Qk0KT0s5aEEzT3RTN2F0S2FDb1lvSmRrYkpHQTdWdytNNzA4NEJOTGhxM1Fyckg4S3M3Z05pdC9NN3lxSnU1alBaZgo2SE1seHNyWU9NVUhuVlk4VDkwN0Q3cS9ORUNnRThzODhnZzAyQ3JNWTFqanE4UnBpUUtDQVFFQTE2UHJaMUEwClNISS83akdmS3BETkJzQ0xrVUFxRERKSzQ0dFdJYmJBUXFhRTN1eDh4bkFlU2NjSHozbS9ScEpPSGtteHZTZlgKbTJ1Wk8veGtNTWhYK2lwOHdFOHZibzR1enVNYitTSXE3bWpialJkK1JJczJ5NHJsZVQ2NGVjRWc4R2pZckExZgpiSEI0MmhQclVTcXpxUVIwOTZocm1Lb1diU0RDZDZwOUVNeWVzT3IwTjdtQmJYVVZPazJxZGtYRlZWbHBlUDdpClFxWGdRUUI0bHgzLzJJdlpBMlhJUXlQdGJ0RWVRbmgyQ3FNM2NDMzR0VEVjZ244K0VwNG9SWmkwTTBHaUY3bXgKOTEvZHY2THZlNTR5K1pON1lXd1NFQ09ubzd5bDlvTlBZVnVGMGRiMjh0elppMThCeHJTQ2JESE1XbExvUzhWNgpXTEo0OGlSODJDYkc1d0tDQVFFQXdFRjM4KzYyeDhDU2x0blZZNlJaN0J0NEdiNEJqVWhWYXZ0NFkxUGFlbXFNCjFidFVnR2JyUnBoNHFUSEFTckUwUUZLeVZKYnlCUkJyRHIxWHU4WWRSVXQzZC92VzlIR1dPd1BKdTN2M3pLbHMKQ2xsZnpFY3J5L1l2aHAzSzlEcGR6OE1icHdueW5xcGV6b0xMNlJpL3JnK0hyTzBueXd1RSt0T2xYVFo2eUtadApHWVdTSVBWaG00NUJkc2ZxUzhnYjVvbjA0bHh4bnhxVnJvN0c0TUR6cmVEYlFhaGdyS3VuRWxwajZ4eW1PVWpBCkdCZDR3QUVrUExxNUUrRWcreDY4TkRLVTYwK29ybFhLWVhDQm5HSFZOQ3BVcmswVXkrcHFZZmFEN3VuR2VzaHMKSEwra3lXbXl5a3ErTmNKbnRXMFNSNy9sU1IvZUFhVEZyVzZVaXV0RGJRS0NBUUVBemhRYU9PNmVPSW51N016QgpScVdCT3EyeDg4cjFKQmpBRnZzbkFpc3JTOGJsZmtGVTdXREdvVTB5K3FWb0ZhSm1RMjI4RFlCUS9YZnp4aTdxCjlPL1JuQU1VbTVoUlJQOWVYbHNPZGFXZ2o1em9ETXRoNFZHRnVUbHhHZERGN1oyU3hBMysyMVlnVm5xYUZCY3IKTUxOMVpOWWNqajJITGl1R0tSNUFtcW4wd2FRN0YrcENJQ3NKTkxqSzQ2QXJnc0lrMXU4TzdCSHgyeTI0eFlZVQp1SjV6emRmQU9nNEFONkhURzY5L2twaWFmb29DeGhNNDlyZ0xmZTdxUEZLbk8vTzJhckdUbmNiWi9BWEMzb3h4Ci81dHRMYlF6R2lSMGtyWHdWSHRKdys4elltQmIzL0RtcWF4RHZueTZMdEo5UGJiTmk1aGw1VnZCRTVqa0dzeWgKL3RQNEN3S0NBUUJ2R1dZb0lKcWZkRGxCMHovdEJOeXlCRzJ5OG9vVEN1blJtT0JKQmZ3TEllZWcyMUJKb3kveQo2OGxPZk9HU1NEVFp0dkEyMGNPcUNZTFVVYmFSWERzdUFCNVp4NzdBSTZPZEZ1Tk01S2FlTG9td3NWVWF4MFlYCjUzd3ZYcUFaNG1DejN4dnJ1MlBwTEtyOHk3anFTdEw1MHgra1hxZlFQaWZxaXNQVXlkYktmT0l2RFhFVWVyaWQKRytmWXJFNUkzS3JDM3BZVStUWmJ1eEVrZm4yUEEvSE5XVk5hN2VKdjVnSDJLU1gwaCtuRzBMT3hPRjhmRlluTApUbHdGa09OdU9xU254Vk1wYUM4aUQ1R1VIVi9JN3dBMTFRQjZlVEM3Wmd0ejhQRHM3MHN6U1A2dzNrNXIxaGpyCnJhV2RpMnBDL1hUQzRiR3VRQ3dhNXcwVTNBSWJCVGxCQW9JQkFEc1RONGhvclVHNWw3MXhLZk5ibVBTbDZ6RlIKYTJ4d2U2VVZPOVZzMFpHeEdLWWJSN1VuVDBDL1FqUiswS2JsbE9leDdFY3cyMklCcmFFVzBGbXpuVnoyUW9FNwpMUE5COXhyTTFEeE56UjZEbFBUeERMcEFGWVlUcm40SWY1cjFVdVdpc2lMdmd6T2xGTlVITnN5UFJIZWNGblhUCnNhTk9JWkgrQTJ5KzF3QWdpSFZIS2JPRGRHeVFQVlQ0TXFFWkJaY2pQcmRBekNKcnloSHlYdHBqRjFSdlFEYTMKTVM3U3JVTGM4djJGQWJ1VG1QZ2R1ZHBKd1Q4dENCa2VRKzZ4YmJWN3YrZzBEMG5EWFNIZFVwNXFyUzcrTnhtVwp4NWV4UHo1VENhYXcxSnkzWjRmT1MzMTV6eHJGdmRHTmhWRXhMMzRlUVlzOHRYN0N0VWxuWkNray9zYz0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0= --- apiVersion: v1 kind: Secret metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard-csrf namespace: kubernetes-dashboard type: Opaque data: csrf: "" --- apiVersion: v1 kind: Secret metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard-key-holder namespace: kubernetes-dashboard type: Opaque --- kind: ConfigMap apiVersion: v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard-settings namespace: kubernetes-dashboard --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard rules: # Allow Dashboard to get, update and delete Dashboard exclusive secrets. - apiGroups: [""] resources: ["secrets"] resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] verbs: ["get", "update", "delete"] # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. - apiGroups: [""] resources: ["configmaps"] resourceNames: ["kubernetes-dashboard-settings"] verbs: ["get", "update"] # Allow Dashboard to get metrics. - apiGroups: [""] resources: ["services"] resourceNames: ["heapster", "dashboard-metrics-scraper"] verbs: ["proxy"] - apiGroups: [""] resources: ["services/proxy"] resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] verbs: ["get"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard rules: # Allow Metrics Scraper to get metrics from the Metrics server - apiGroups: ["metrics.k8s.io"] resources: ["pods", "nodes"] verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: kubernetes-dashboard subjects: - kind: ServiceAccount name: kubernetes-dashboard namespace: kubernetes-dashboard --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kubernetes-dashboard namespace: kubernetes-dashboard roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: kubernetes-dashboard subjects: - kind: ServiceAccount name: kubernetes-dashboard namespace: kubernetes-dashboard --- kind: Deployment apiVersion: apps/v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard spec: replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: k8s-app: kubernetes-dashboard template: metadata: labels: k8s-app: kubernetes-dashboard spec: containers: - name: kubernetes-dashboard image: kubernetesui/dashboard:v2.0.0-beta4 imagePullPolicy: Always ports: - containerPort: 8443 protocol: TCP args: - --auto-generate-certificates - --namespace=kubernetes-dashboard - --token-ttl=43200 # Uncomment the following line to manually specify Kubernetes API server Host # If not specified, Dashboard will attempt to auto discover the API server and connect # to it. Uncomment only if the default does not work. # - --apiserver-host=http://my-address:port volumeMounts: - name: kubernetes-dashboard-certs mountPath: /certs # Create on-disk volume to store exec logs - mountPath: /tmp name: tmp-volume livenessProbe: httpGet: scheme: HTTPS path: / port: 8443 initialDelaySeconds: 30 timeoutSeconds: 30 volumes: - name: kubernetes-dashboard-certs secret: secretName: kubernetes-dashboard-certs - name: tmp-volume emptyDir: {} serviceAccountName: kubernetes-dashboard # Comment the following tolerations if Dashboard must not be deployed on master tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule --- kind: Service apiVersion: v1 metadata: labels: k8s-app: dashboard-metrics-scraper name: dashboard-metrics-scraper namespace: kubernetes-dashboard spec: ports: - port: 8000 targetPort: 8000 selector: k8s-app: dashboard-metrics-scraper --- kind: Deployment apiVersion: apps/v1 metadata: labels: k8s-app: dashboard-metrics-scraper name: dashboard-metrics-scraper namespace: kubernetes-dashboard spec: replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: k8s-app: dashboard-metrics-scraper template: metadata: labels: k8s-app: dashboard-metrics-scraper spec: containers: - name: dashboard-metrics-scraper image: kubernetesui/metrics-scraper:v1.0.1 ports: - containerPort: 8000 protocol: TCP livenessProbe: httpGet: scheme: HTTP path: / port: 8000 initialDelaySeconds: 30 timeoutSeconds: 30 volumeMounts: - mountPath: /tmp name: tmp-volume serviceAccountName: kubernetes-dashboard # Comment the following tolerations if Dashboard must not be deployed on master tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule volumes: - name: tmp-volume emptyDir: {} --- apiVersion: extensions/v1beta1 kind: Ingress metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: kubernetes-dashboard annotations: kubernetes.io/ingress.class: traefik traefik.ingress.kubernetes.io/frontend-entry-points: http,https traefik.ingress.kubernetes.io/redirect-entry-point: https spec: rules: - host: csdd.xxxx.com http: paths: - backend: serviceName: kubernetes-dashboard servicePort: 443 tls: - secretName: dashboard-tls-cert # 创建kubernetes-dashboard token 登录 # 生成token kubectl create sa dashboard-admin -n kube-system # 授权token 访问权限 kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin # 获取token ADMIN_SECRET=$(kubectl get secrets -n kube-system | grep dashboard-admin | awk '{print $1}') # 获取dashboard.kubeconfig 使用token 值 DASHBOARD_LOGIN_TOKEN=$(kubectl describe secret -n kube-system ${ADMIN_SECRET} | grep -E '^token' | awk '{print $2}') echo ${DASHBOARD_LOGIN_TOKEN} # 设置集群参数 kubectl config set-cluster kubernetes \ --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=dashboard.kubeconfig # 设置客户端认证参数，使用上面创建的 Token kubectl config set-credentials dashboard_user \ --token=${DASHBOARD_LOGIN_TOKEN} \ --kubeconfig=dashboard.kubeconfig # 设置上下文参数 kubectl config set-context default \ --cluster=kubernetes \ --user=dashboard_user \ --kubeconfig=dashboard.kubeconfig # 设置默认上下文 kubectl config use-context default --kubeconfig=dashboard.kubeconfig # 绑定hosts https://csdd.xxxx.com/#/overview?namespace=default # kubernetes-dashboard 使用metrics 显示cpu内存资源 所有要部署metrics-server# win on Ubuntu 操作 #创建metrics-server 证书 cat << EOF | tee /apps/work/k8s/cfssl/k8s/metrics-server.json { "CN": "metrics-server", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "GuangDong", "L": "GuangZhou", "O": "cluster", "OU": "cluster" } ] } EOF ### 生成证书 cfssl gencert -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \ -config=/apps/work/k8s/cfssl/ca-config.json \ -profile=kubernetes /apps/work/k8s/cfssl/k8s/metrics-server.json | cfssljson -bare ./metrics-server # 创建metrics-server-secrets.yaml # base64 加密 cat metrics-server.pem|base64 | tr -d '\n' cat metrics-server-key.pem|base64 | tr -d '\n' vi metrics-server-secrets.yaml apiVersion: v1 kind: Secret metadata: labels: k8s-app: metrics-server name: metrics-server-certs namespace: kube-system type: Opaque data: metrics-server.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ3VENDQXRXZ0F3SUJBZ0lVYkloOTQ3Z3NET2gxdVplVnBoMU9GVmhYeHA0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2JqRUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2R0WkdSbllXMWxNUkF3RGdZRFZRUUxFd2R0WkdSbllXMWxNUk13CkVRWURWUVFERXdwcmRXSmxjbTVsZEdWek1CNFhEVEU1TURjd05UQXpOVGd3TUZvWERUSTVNRGN3TWpBek5UZ3cKTUZvd2NqRUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2R0WkdSbllXMWxNUkF3RGdZRFZRUUxFd2R0WkdSbllXMWxNUmN3CkZRWURWUVFERXc1dFpYUnlhV056TFhObGNuWmxjakNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0MKQVFvQ2dnRUJBTUlvdHhsakpLcCtyb1hGcGJJWXRuNW1mVXVyREM1bUlkb2Z0RTNSVGhNU1pPSCt0aVVoMDdQRQpnb0xPOG1lSkxaS21ob1BUYzhJWTJYVjdiZzJWWFpRVUd6MFJuMExmNWdWam5UN29yMFFnZzErYnJnZU5wWUtpCjVoNm5ISVE3ZGlKYk10cUFndm16RGR6bWdoUXd2SHBKTzh4bEQwKzRwT0VHT2VtQkNPU3BsaFhrenR3UWQ3ZHYKY2x1QUljQUdiUGF6dzI4VkJJU2F4bCtrZnZwNzIyeEkvVy9DL3pRS1JnN053UG9IaVpFWm9QcGxPY001cGpvUwpJeEdnWVZEYjB6OGlqZWR3RjZmcE9RZkFOcitvQnVONnZnMXAzd2Jud2tKTWtEQUV2MzBXZG1BUzB5STJicC9RCkJZYjU2VWxGTXI4anNoWHJ5dlVsZ3F3S0hscFh0WkVDQXdFQUFhTi9NSDB3RGdZRFZSMFBBUUgvQkFRREFnV2cKTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQU1CZ05WSFJNQkFmOEVBakFBTUIwRwpBMVVkRGdRV0JCVEp6cVJBMWdIN***d3B3TG01ZEtWUHFvdG43VEFmQmdOVkhTTUVHREFXZ0JSeDRLY0FSY2FrClIvYnhtd29UQmVEcyt4QW9hVEFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBcTgyS0JRMWJBM1pjeG9OZGFJVVUKNjVvSkZhM3paM2k5SDk1QWZvU0dXQWQvbFU3L3B2Qkt2K0EwbS9qQ0tMS1FpQ0xXRGtHYzFUNS8xTzhPYTg0YgpDV3I5M3pOWldRb2N6UWJEaHFCRnZNUVp0azRYVVkrMjR3Yjd3cmJCRHc2QVY2R0l2bVMrYm91eFRVd29sbmRMCk5FS2EvcHNvQUtaRUFJZkJUaCtMNVpMQ09GOXFUWEMyOGtnN1czak4vMzBiYlk5UE5ObVpLcGNaNEpEVjA5aGYKU3RaTjZuOVFXK3ZDcFFoZXVISWVORlR2RnQ5bGtSMVBFYUtHUjFiWEdyeUNHOHNTeXVDc0xER1lnVlhmYVZtYgp3dTlnSG1JS2E2aDZWVmVIWitMbVFmZmxqcEdPRStKV1l1TWRPamtHYUYxdEtLUWZFelZGN3BxT0VTQXkrV3hpCnVBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= metrics-server-key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBd2lpM0dXTWtxbjZ1aGNXbHNoaTJmbVo5UzZzTUxtWWgyaCswVGRGT0V4Sms0ZjYyCkpTSFRzOFNDZ3M3eVo0a3RrcWFHZzlOendoalpkWHR1RFpWZGxCUWJQUkdmUXQvbUJXT2RQdWl2UkNDRFg1dXUKQjQybGdxTG1IcWNjaER0Mklsc3kyb0NDK2JNTjNPYUNGREM4ZWtrN3pDVVBUN2lrNFFZNTZZRUk1S21XRmVUTwozQkIzdDI5eVc0QWh4QVpzOXJQRGJ4VUVoSnJHWDZSKytudmJiRWo5YjhML05BcEdEczNBK2dlSmtSbWcrbVU1Cnd6bW1PaElqRWFCaFVOdlRQeUtONTNBWHArazVCOEEydjZnRzQzcStEV25mQnVmQ1FreVFNQVMvZlJaMllCTFQKSWpadW45QUZodm5wU1VVeXZ5T3lGZXZLOVNXQ3JBb2VXbGUxa1FJREFRQUJBb0lCQUQzaEtoSGVSalpNYjZUVQp6QzFKc1FmeDlDYmttZHBEMUxBbkFKajRoekNPNFRZaHJyMkEzVzdpeDFHWFVTeHNUT2o3UjgzRjI1UFZ4YUNxCnVQVjlVRGk4ZTczbjJ1RSthSm41R0ltUE1TUytWQUJwcG5wank0Y3FFYnRkT1RwdmxRUDRHdW9Hb1RlaHVGNVoKM01WQWp5Rk9LOCt4VkFMdGJ5Y0VpL3ArbEc0RGkvOThIcUlDQngwSlhCUnJoV05lWUdZL0c3eGNWT2pCNUl5SQpPNVpoZ1I0Sk9yODloNVZ1RHdIY3E2UVlLQ2sxTktQZzc0Y3BOY2J5ZVZEM0FhYVRHd25QU3BMd0hGaElzTGpNCkllaEJqZzkrZDdyRU8xMHU0azhKWW5qYUdNMzRMM0RlTVVrck95NUMzRjY3RFNwaTJVZUhjOVh7YzViVUFwb0gKTE1zRUxuMENnWUVBenV3STRSN0FrYmZ2VnVFR0hOa0FFNjgzdTJoWk1iUFdwSHVVTGQ5UzJnUzYzODVWUFdJSApiQXp6WGZuTjh2U2dHNVVQdzUxa1VLZVppY0pORTVBWWtOVDRwMWxoTFdKSkwxSWRSdEV3VU5oblVLNlczRWlMCmJLeDhhalk3dkZDV0ZKUmRTUHJYLzViTWU4TVBCWWNTT0FkZEErZFhlaGd0K2x0WEU3LzE1cGNDZ1lFQThEVzcKOEIyZGNYLzZqMVF4UkhEc3ZGUDYxaWxTREJhMEtKUGlobnpSZWVwVHRFc0hvRitBODk2SXRDZFFMSTJZcEZyRApBU1dSSU1VQVVzVE1uMStvZFprOGovd21QRkxzUmRpSVJWZC80ZHdCTmlDNHJxdnkwQTFxUVNJUXF1MC9CcFV2ClRpMjhZeURRdHh0Wmg1d0NDQUx2a0Rqb2N4cXJzbHEwRDViTnNoY0NnWUFFNjB1U3ZuNXBlR3Z5THN0TWFFUSsKTTQ0cG9HTnVjUTRMUHZ6WTBsUlA1b2ppeFM3MWxKMEVqcEpDaTZ1L3VIdXQ3QzlCd1Q0ZlJMeTdyNmpSYkQySgpRK2JkWTV5UnphSmJ3Nkg2aXdLUkNYUDdVUXM1Rldockh3YWVOOGZYeERxdEpwSEpLRjEyTUFtUWI2U3R4dlpCCjZycmxXdHlUaEh2alZnU043YVJVNVFLQmdHbVFJN3lkTnpERy9sVDR1Z0lLNG03Tk5VSGl2TlRsTVYxWHlFZzAKR0ZiTW5PWnh5ck02NVUvRzd5ckUwQjRVU0EyS2VZSktnU0gya1hMT1crSjZSbTBQMzZhak9DWndocmNYTnFQSwpsVCtyMExoNTNzK2NiMFB4Y1UyWWE5ekNFRjJUT0V2U0c2VXdxYWllazFUZVFhSkZzQVFnamo3dmJKOGY3MXVlCmVWMFhBb0dCQUlQWDN1OTJtU3o5clpqOTN3NFNKbUk2K0tkY3BtTGg0TE5IcUdBOVJSNi80T0NWWDBHNHVtZ1YKMkxUOU1MY05CdUVFZE5nTHRHeGIzdTI5cUlLZHZpcDhBZzlTbHQvMlBIcnlsLzIzWWU0bURTWDVUOXNrVXhaRgpjVGxvN3QxSnRZN3NaRU00Vng4cWxQbDcwWXMvSWRuWmhWaFU2d1F2ZGp0TGk1UlU4L2ttCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== # resource-reader.yaml vi resource-reader.yaml --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:metrics-server rules: - apiGroups: - "" resources: - pods - nodes - nodes/stats verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:metrics-server subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system # metrics-server-service.yaml vi metrics-server-service.yaml --- apiVersion: v1 kind: Service metadata: name: metrics-server namespace: kube-system labels: kubernetes.io/name: "Metrics-server" spec: selector: k8s-app: metrics-server ports: - port: 443 protocol: TCP targetPort: 443 # metrics-apiservice.yaml vi metrics-apiservice.yaml --- apiVersion: apiregistration.k8s.io/v1beta1 kind: APIService metadata: name: v1beta1.metrics.k8s.io spec: service: name: metrics-server namespace: kube-system group: metrics.k8s.io version: v1beta1 insecureSkipTLSVerify: true groupPriorityMinimum: 100 versionPriority: 100 # auth-reader.yaml vi auth-reader.yaml --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: metrics-server-auth-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system # auth-delegator.yaml vi auth-delegator.yaml --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: metrics-server:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system # aggregated-metrics-reader.yaml vi aggregated-metrics-reader.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:aggregated-metrics-reader labels: rbac.authorization.k8s.io/aggregate-to-view: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-admin: "true" rules: - apiGroups: ["metrics.k8s.io"] resources: ["pods"] verbs: ["get", "list", "watch"] # metrics-server-deployment.yaml vi metrics-server-deployment.yaml --- apiVersion: v1 kind: ServiceAccount metadata: name: metrics-server namespace: kube-system --- apiVersion: apps/v1 kind: Deployment metadata: name: metrics-server namespace: kube-system labels: k8s-app: metrics-server spec: selector: matchLabels: k8s-app: metrics-server template: metadata: name: metrics-server labels: k8s-app: metrics-server spec: serviceAccountName: metrics-server tolerations: - effect: NoSchedule key: node.kubernetes.io/unschedulable operator: Exists - key: NoSchedule operator: Exists effect: NoSchedule volumes: # mount in tmp so we can safely use from-scratch images and/or read-only containers - name: tmp-dir emptyDir: {} - name: metrics-server-certs secret: secretName: metrics-server-certs containers: - name: metrics-server image: juestnow/metrics-server-amd64:v0.3.3 imagePullPolicy: Always command: - /metrics-server - --tls-cert-file=/certs/metrics-server.pem - --tls-private-key-file=/certs/metrics-server-key.pem - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP - --kubelet-insecure-tls volumeMounts: - name: tmp-dir mountPath: /tmp - name: metrics-server-certs mountPath: /certs # 创建metrics-server 服务 kubectl apply -f . # 验证metrics-server kubectl top node [root@]~]#kubectl top node NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% master 177m 7% 2057Mi 35% [root@]~]#kubectl top pods -A NAMESPACE NAME CPU(cores) MEMORY(bytes) clusterstorage nfs-client-provisioner-5f6bc44cd7-fjr7f 3m 15Mi kube-system coredns-597b77445b-fhxvr 4m 23Mi kube-system kube-router-5tmgw 9m 16Mi kube-system metrics-server-66d78c47-zn679 1m 14Mi kube-system traefik-578574dfdb-dzl22 6m 41Mi kubernetes-dashboard dashboard-metrics-scraper-fb986f88d-rc6zs 1m 25Mi kubernetes-dashboard kubernetes-dashboard-668c4f84bc-w6vw6 2m 40Mi # 能够正常获取CPU 内存值
免责声明：本站发布的内容（图片、视频和文字）以原创、转载和分享为主，文章观点不代表本网站立场，如果涉及侵权请联系站长邮箱：is@yisu.com进行举报，并提供相关证据，一经查实，将立刻删除涉嫌侵权内容。
