Launch A Multi-node Kubernetes Cluster Using Vagrant And KVM

# -*- mode: ruby -*- # vi: set ft=ruby : VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provider :libvirt do |libvirt| libvirt.driver = "kvm" end config.vm.synced_folder './shared', '/vagrant', type: '9p', disabled: false, accessmode: "mapped" config.vm.box = "generic/ubuntu2004" config.vm.box_version = "3.0.6" config.vm.network "forwarded_port", guest: 5432, host: 5432 config.vm.provision:shell, inline: <<-SHELL echo "root:rootroot" | sudo chpasswd sudo timedatectl set-timezone Asia/Ho_Chi_Minh SHELL config.vm.define "kube1" do |kube1| kube1.vm.hostname = "kube1" kube1.vm.network "private_network", ip: "192.168.121.211" kube1.vm.provision:shell, inline: <<-SHELL sudo echo -e "192.168.121.211\tkube1" >> /etc/hosts SHELL kube1.vm.provision:shell, path: "kube-bootstrap.sh" end config.vm.define "kube2" do |kube2| kube2.vm.hostname = "kube2" kube2.vm.network "private_network", ip: "192.168.121.212" kube2.vm.provision:shell, inline: <<-SHELL sudo echo -e "192.168.121.212\tkube2" >> /etc/hosts SHELL kube2.vm.provision:shell, path: "kube-bootstrap.sh" end config.vm.define "master" do |master| master.vm.hostname = "master" master.vm.network "private_network", ip: "192.168.121.210" master.vm.provision:shell, inline: <<-SHELL sudo sed -i 's/127\.0\.1\.1.*master//' /etc/hosts sudo echo -e "192.168.121.210\tmaster" >> /etc/hosts SHELL master.vm.provision:shell, path: "kube-bootstrap.sh" master.vm.provision:shell, path: "kubeadm_reinstall.sh" end end 

# master kube1.vm.network "private_network", ip: "192.168.121.210" # kube1 kube1.vm.network "private_network", ip: "192.168.121.211" # kube2 kube2.vm.network "private_network", ip: "192.168.121.212" 

#!/bin/bash # Bootstrap machine ensure_netplan_apply() { # First node up assign dhcp IP for eth1, not base on netplan yml sleep 5 sudo netplan apply } step=1 step() { echo "Step $step $1" step=$((step+1)) } resolve_dns() { step "===== Create symlink to /run/systemd/resolve/resolv.conf =====" sudo rm /etc/resolv.conf sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf } install_docker() { step "===== Installing docker =====" sudo apt update sudo apt install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" sudo apt update sudo apt install -y docker-ce docker-ce-cli containerd.io sudo groupadd docker sudo gpasswd -a $USER docker sudo chmod 777 /var/run/docker.sock } install_kube() { step "===== Installing kube =====" sudo apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main" sudo apt update sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add sudo apt install -y kubeadm sudo swapoff -a sudo echo "source <(kubectl completion bash)" >> /home/vagrant/.bashrc sudo echo "source <(kubeadm completion bash)" >> /home/vagrant/.bashrc } setup_root_login() { step "===== Setup root login =====" sudo sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config sudo systemctl restart ssh sudo echo "root:rootroot" | chpasswd } setup_welcome_msg() { step "===== Bash =====" sudo apt -y install cowsay sudo echo -e "\necho \"Welcome to Vagrant Kube Ubuntu 18.04\" | cowsay\n" >> /home/vagrant/.bashrc sudo ln -s /usr/games/cowsay /usr/local/bin/cowsay sudo apt install -y tcl expect net-tools } main() { ensure_netplan_apply resolve_dns install_docker install_kube setup_root_login setup_welcome_msg } main 

#!/bin/bash # Reset and start kubeadm echo "Enter password:" read -s PASSWORD sudo_expect() { command="sudo $1" cat > /tmp/tmp.exp <<EOF #!/usr/bin/expect set password [lindex \$argv 0] set cmd [lrange \$argv 1 end] spawn {*}\$cmd expect { "password" { send "\$password\r" exp_continue } "Are you sure you want to proceed?" { send "y\r" exp_continue } eof { exit } } interact EOF expect /tmp/tmp.exp $PASSWORD $command } resestKubeadm() { sudo_expect "kubeadm reset -f" sudo_expect "iptables -F" } initKubeadm() { sudo_expect "swapoff -a" if [ "$(hostname)" != "master" ]; then sudo_expect "hostname master" fi sudo_expect "rm -rf /var/lib/etcd/member" sudo_expect "kubeadm init --node-name master --apiserver-advertise-address $(hostname -i)" mkdir -p $HOME/.kube rm -f $HOME/.kube/config sudo_expect "cp -i /etc/kubernetes/admin.conf $HOME/.kube/config" sudo_expect "chown $(id -u):$(id -g) $HOME/.kube/config" sudo_expect "chmod +wr /etc/kubernetes/admin.conf" } kubeAddOn() { kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" } resestKubeadm initKubeadm kubeAddOn 

#!/bin/bash # Join remote node to kubernetes cluster ip=$1 PASSWORD="rootroot" ssh_expect() { command="$1" cat > /tmp/tmp.exp <<EOF #!/usr/bin/expect set password [lindex \$argv 0] set cmd [lrange \$argv 1 end] spawn {*}\$cmd expect { "password" { send "\$password\r" exp_continue } "Are you sure you want to proceed?" { send "y\r" exp_continue } eof { exit } } interact EOF expect /tmp/tmp.exp $PASSWORD $command } # Get token-ca-cert-hash sha=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | \ openssl dgst -sha256 -hex | sed 's/^.* //') # Get token of master node by creating new one token=$(kubeadm token create) # Join the remote node cluster_ip=$(kubectl get node master -o jsonpath="{.status.addresses[0].address}") ssh_expect "ssh root@$ip kubeadm reset" ssh_expect "ssh root@$ip swapoff -a" ssh_expect "ssh root@$ip kubeadm join ${cluster_ip}:6443 --token ${token} --discovery-token-ca-cert-hash sha256:${sha}" 

vagrant up 

vagrant@master:/vagrant$ kubectl get node NAME STATUS ROLES AGE VERSION kube1 Ready <none> 2d1h v1.21.0 kube2 Ready <none> 2d1h v1.21.0 master Ready control-plane,master 2d1h v1.21.0