From 21154f56a59023e86a408c9b73adb83a9a4308b9 Mon Sep 17 00:00:00 2001
From: Alexander Trost <galexrt@googlemail.com>
Date: Tue, 24 Mar 2020 00:42:10 +0100
Subject: [PATCH 1/2] *: install docker from official docker repos

Install Docker from official Docker repos instead of OS repositories.
This will get us a more uptodate Docker version.

Signed-off-by: Alexander Trost <galexrt@googlemail.com>
---
 vagrantfiles/Vagrantfile   |  1 +
 vagrantfiles/centos/common | 18 ++++++++++++++++++
 vagrantfiles/fedora/common | 15 +++++++++++++++
 vagrantfiles/fedora/master |  2 +-
 vagrantfiles/ubuntu/common | 18 ++++++++++++++++++
 vagrantfiles/ubuntu/master | 19 ++++---------------
 vagrantfiles/ubuntu/pre    |  8 ++++++++
 7 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/vagrantfiles/Vagrantfile b/vagrantfiles/Vagrantfile
index a93958c..8e8d2b6 100644
--- a/vagrantfiles/Vagrantfile
+++ b/vagrantfiles/Vagrantfile
@@ -94,6 +94,7 @@ Vagrant.configure('2') do |config|
             subconfig.vm.provision :shell, inline: $osPrepareScript
         end
         subconfig.vm.provision :shell, inline: $prepareScript
+        subconfig.vm.provision :shell, inline: $containerRuntimeInstallScript
         subconfig.vm.provision :shell, inline: $baseInstallScript
         subconfig.vm.provision :shell, inline: $verifyNodeScript
         # Run diskandreboot plugin to add VirtualBox disks to VMs
diff --git a/vagrantfiles/centos/common b/vagrantfiles/centos/common
index ea7b1eb..f999c47 100644
--- a/vagrantfiles/centos/common
+++ b/vagrantfiles/centos/common
@@ -1,5 +1,6 @@
 $box_image = ENV['BOX_IMAGE'] || 'centos/7'.freeze
 
+# Stop and disable firewalld service
 $osPrepareScript = <<SCRIPT
 set -x
 
@@ -7,3 +8,20 @@ systemctl stop firewalld || true
 systemctl disable firewalld || true
 systemctl mask firewalld || true
 SCRIPT
+
+$containerRuntimeInstallScript = <<SCRIPT
+set -x
+
+retries=5
+for ((i=0; i<retries; i++)); do
+    yum install -y yum-utils \
+        device-mapper-persistent-data \
+        lvm2
+    yum-config-manager \
+        --add-repo \
+        https://download.docker.com/linux/centos/docker-ce.repo
+    yum install -y docker-ce docker-ce-cli containerd.io && \
+        break
+done
+[[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
+SCRIPT
diff --git a/vagrantfiles/fedora/common b/vagrantfiles/fedora/common
index 91991aa..1d296d8 100644
--- a/vagrantfiles/fedora/common
+++ b/vagrantfiles/fedora/common
@@ -7,3 +7,18 @@ if [ ! -f /usr/sbin/iptables-legacy ]; then
 fi
 update-alternatives --set iptables /usr/sbin/iptables-legacy
 SCRIPT
+
+$containerRuntimeInstallScript = <<SCRIPT
+set -x
+
+retries=5
+for ((i=0; i<retries; i++)); do
+    dnf -y install dnf-plugins-core
+    dnf config-manager \
+        --add-repo \
+        https://download.docker.com/linux/fedora/docker-ce.repo
+    dnf install -y docker-ce docker-ce-cli containerd.io && \
+        break
+done
+[[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
+SCRIPT
diff --git a/vagrantfiles/fedora/master b/vagrantfiles/fedora/master
index 210311f..3bd5bf9 100644
--- a/vagrantfiles/fedora/master
+++ b/vagrantfiles/fedora/master
@@ -23,7 +23,7 @@ sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
 
 # Use yum so we can use this for CentOS too
 yum clean expire-cache
-yum install --nogpgcheck -y net-tools screen tree telnet conntrack socat docker rsync ${KUBERNETES_PACKAGES}
+yum install --nogpgcheck -y net-tools screen tree telnet conntrack socat rsync ${KUBERNETES_PACKAGES}
 
 systemctl enable docker && systemctl start docker
 SCRIPT
diff --git a/vagrantfiles/ubuntu/common b/vagrantfiles/ubuntu/common
index b9564d3..f085c31 100644
--- a/vagrantfiles/ubuntu/common
+++ b/vagrantfiles/ubuntu/common
@@ -5,3 +5,21 @@ if $kubernetes_pkg_version_suffix == ''
 end
 
 $docker_version = '5:18.09.1~3-0~ubuntu-bionic'
+
+$containerRuntimeInstallScript = <<SCRIPT
+set -x
+
+curl --retry 5 -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable"
+
+retries=5
+for ((i=0; i<retries; i++)); do
+    apt-get update
+    apt-get -y install docker-ce=#{$docker_version} && \
+        break
+done
+[[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
+SCRIPT
diff --git a/vagrantfiles/ubuntu/master b/vagrantfiles/ubuntu/master
index aae98e9..4461f13 100644
--- a/vagrantfiles/ubuntu/master
+++ b/vagrantfiles/ubuntu/master
@@ -1,19 +1,6 @@
 $baseInstallScript = <<SCRIPT
 set -x
 
-retries=5
-for ((i=0; i<retries; i++)); do
-    apt-get update
-    apt-get -y install apt-transport-https curl software-properties-common ca-certificates && \
-        break
-done
-[[ $retries -eq i ]] && { echo "Failed to install apt-transport-https and other dependenices after 5 tries"; exit 1; }
-
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-add-apt-repository \
-   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
-   $(lsb_release -cs) \
-   stable"
 curl curl --retry 5 --fail -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 cat <<EOF > /etc/apt/sources.list.d/kubernetes.list
 deb https://apt.kubernetes.io/ kubernetes-xenial main
@@ -28,13 +15,15 @@ fi
 retries=5
 for ((i=0; i<retries; i++)); do
     apt-get update
-    apt-get -y install screen telnet conntrack socat docker-ce=#{$docker_version} ${KUBERNETES_PACKAGES} && \
+    apt-get -y install screen telnet conntrack socat ${KUBERNETES_PACKAGES} && \
         break
 done
-[[ $retries -eq i ]] && { echo "Failed to install docker, kubelet, kubeadm and other dependenices after 5 tries"; exit 1; }
+[[ $retries -eq i ]] && { echo "Failed to install kubelet, kubeadm and other dependenices after 5 tries"; exit 1; }
 
 apt-mark hold kubelet kubeadm kubectl
 
 # Increase /run dir as it is/was a problem on Ubuntu 18.04 (at least on bento box image)
 echo 'tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=614460k,mode=755' >> /etc/fstab
+
+systemctl enable docker && systemctl start docker
 SCRIPT
diff --git a/vagrantfiles/ubuntu/pre b/vagrantfiles/ubuntu/pre
index 177eca0..1742695 100644
--- a/vagrantfiles/ubuntu/pre
+++ b/vagrantfiles/ubuntu/pre
@@ -8,4 +8,12 @@ sudo update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
 sudo update-alternatives --set arptables /usr/sbin/arptables-legacy
 sudo update-alternatives --set ebtables /usr/sbin/ebtables-legacy
 
+retries=5
+for ((i=0; i<retries; i++)); do
+    apt-get update
+    apt-get -y install apt-transport-https curl software-properties-common ca-certificates && \
+        break
+done
+[[ $retries -eq i ]] && { echo "Failed to install apt-transport-https and other dependenices after 5 tries"; exit 1; }
+
 SCRIPT

From 1a01b8f84e89dec8dd7ff3e6fca3fe202d5e609f Mon Sep 17 00:00:00 2001
From: Alexander Trost <galexrt@googlemail.com>
Date: Sun, 29 Mar 2020 19:50:02 +0200
Subject: [PATCH 2/2] vagrantfile: fixed retry loops

Signed-off-by: Alexander Trost <galexrt@googlemail.com>
---
 Makefile                      | 13 ++++---
 vagrantfiles/Vagrantfile_vars | 66 +++++++++++++++++++++++------------
 vagrantfiles/centos/common    |  5 +--
 vagrantfiles/fedora/common    | 15 ++++++--
 vagrantfiles/ubuntu/common    |  5 +--
 vagrantfiles/ubuntu/master    |  5 +--
 vagrantfiles/ubuntu/pre       | 10 ++++--
 7 files changed, 81 insertions(+), 38 deletions(-)

diff --git a/Makefile b/Makefile
index 88283a1..792a465 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ MFILECWD = $(shell pwd)
 REVERSE_LINES=sed -e '1!G;h;$$!d'
 
 # === BEGIN USER OPTIONS ===
+KUBECTL ?= kubectl
 # Vagrantfile set to use.
 BOX_OS ?= fedora
 # Vagrant Provider
@@ -95,12 +96,16 @@ endif
 
 up: preflight ## Start Kubernetes Vagrant multi-node cluster. Creates, starts and bootsup the master and node VMs.
 	@$(MAKE) start
+	@echo
+	$(KUBECTL) get nodes
+	@echo
+	@echo "Your k8s-vagrant-multi-node Kuberenetes cluster should be ready now."
 
 start: preflight pull
 ifeq ($(VAGRANT_DEFAULT_PROVIDER), "virtualbox")
 	@$(MAKE) start-master start-nodes
 else
-	# Need to start master and nodes separately due to some weird IP assignment side effects
+	# Need to start master and nodes separately due to some weird IP assignment side effects (at least on my machine)
 	@$(MAKE) start-master
 	@$(MAKE) start-nodes
 endif
@@ -128,14 +133,14 @@ kubectl: ## Configure kubeconfig context for the cluster using `kubectl config`
 		> $(CLUSTERCERTSDIR)/client-key.key
 
 	# kubectl create cluster
-	kubectl \
+	$(KUBECTL) \
 		config set-cluster \
 			$(CLUSTER_NAME) \
 			--embed-certs=true \
 			--server=https://$(MASTER_IP):6443 \
 			--certificate-authority=$(CLUSTERCERTSDIR)/ca.crt
 	# kubectl create user
-	kubectl \
+	$(KUBECTL) \
 		config set-credentials \
 			$(CLUSTER_NAME)-kubernetes-admin \
 			--embed-certs=true \
@@ -144,7 +149,7 @@ kubectl: ## Configure kubeconfig context for the cluster using `kubectl config`
 			--client-key=$(CLUSTERCERTSDIR)/client-key.key
 	@rm -rf $(CLUSTERCERTSDIR)
 	# kubectl create context
-	kubectl \
+	$(KUBECTL) \
 		config set-context \
 			$(CLUSTER_NAME) \
 			--cluster=$(CLUSTER_NAME) \
diff --git a/vagrantfiles/Vagrantfile_vars b/vagrantfiles/Vagrantfile_vars
index 5495b61..568c6d6 100644
--- a/vagrantfiles/Vagrantfile_vars
+++ b/vagrantfiles/Vagrantfile_vars
@@ -50,6 +50,10 @@ KUBE_PROXY_IPVS = ENV['KUBE_PROXY_IPVS'] || false
 # Generate new using steps in README
 KUBETOKEN = ENV['KUBETOKEN'] || 'b029ee.968a33e8d8e6bb0d'
 
+if $kubeletExtraArgsFile.nil? || $kubeletExtraArgsFile == ""
+    $kubeletExtraArgsFile = '/etc/sysconfig/kubelet'
+end
+
 # The controller name is different per OS it seems (this is the case for at least Ubuntu right now).
 # For more information, see https://github.com/kusnier/vagrant-persistent-storage/issues/33.
 $storagecontroller = 'k8svmSATAController'
@@ -111,6 +115,22 @@ kubelet --version || { echo "Something wrong with kubelet"; exit 1; }
 
 systemctl enable kubelet
 
+mkdir -p /etc/docker
+cat > /etc/docker/daemon.json <<EOF
+{
+  "exec-opts": ["native.cgroupdriver=systemd"],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2",
+  "storage-opts": [
+    "overlay2.override_kernel_check=true"
+  ]
+}
+EOF
+
+mkdir -p /etc/systemd/system/docker.service.d
 SCRIPT
 
 $kubeMasterScript = <<SCRIPT
@@ -119,8 +139,8 @@ set -x
 kubeadm reset -f
 
 retries=5
-for ((i=0; i<retries; i++)); do
-    # For now ignore SystemVerification error
+for ((i=1; i<=retries; i++)); do
+    # For now ignore preflight SystemVerification error
     kubeadm init \
         #{$kubeadm_init_flags} \
         --ignore-preflight-errors=SystemVerification \
@@ -128,23 +148,23 @@ for ((i=0; i<retries; i++)); do
         --pod-network-cidr=#{POD_NW_CIDR} \
         --token "#{KUBETOKEN}" \
         --token-ttl 0 && \
-        break
+            break
+
+    [[ $retries -eq i ]] && { echo "Failed to run kubeadm init after 5 tries"; exit 1; }
     echo "kubeadm join failed, trying again in 3 seconds (try ${i}/${retries})..."
     sleep 3
 done
-[[ $retries -eq i ]] && { echo "Failed to run kubeadm init after 5 tries"; exit 1; }
 
-KUBELET_EXTRA_ARGS_FILE=/etc/sysconfig/kubelet
-if [ -f /etc/default/kubelet ]; then
-    KUBELET_EXTRA_ARGS_FILE=/etc/default/kubelet
-else
-    KUBELET_EXTRA_ARGS_FILE=/etc/sysconfig/kubelet
-fi
+KUBELET_EXTRA_ARGS_FILE="#{$kubeletExtraArgsFile}"
 
-if grep -q -- '--node-ip=' "$KUBELET_EXTRA_ARGS_FILE"; then
-    sed -ri -e 's|KUBELET_EXTRA_ARGS=--node-ip=[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|KUBELET_EXTRA_ARGS=|' "$KUBELET_EXTRA_ARGS_FILE"
+if [ -f "$KUBELET_EXTRA_ARGS_FILE" ]; then
+    if grep -q -- '--node-ip=' "$KUBELET_EXTRA_ARGS_FILE"; then
+        sed -ri -e 's|KUBELET_EXTRA_ARGS=--node-ip=[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|KUBELET_EXTRA_ARGS=|' "$KUBELET_EXTRA_ARGS_FILE"
+    fi
+    sed -i 's|KUBELET_EXTRA_ARGS=|KUBELET_EXTRA_ARGS=--node-ip=#{NODE_IP} --cni-bin-dir=/opt/cni/bin,/usr/libexec/cni |' "$KUBELET_EXTRA_ARGS_FILE"
+else
+    echo "KUBELET_EXTRA_ARGS=--node-ip=#{NODE_IP} " > "$KUBELET_EXTRA_ARGS_FILE"
 fi
-sed -i 's|KUBELET_EXTRA_ARGS=|KUBELET_EXTRA_ARGS=--node-ip=#{NODE_IP} --cni-bin-dir=/opt/cni/bin,/usr/libexec/cni |' "$KUBELET_EXTRA_ARGS_FILE"
 
 systemctl daemon-reload
 systemctl restart kubelet.service
@@ -185,7 +205,7 @@ set -x
 kubeadm reset -f
 
 retries=5
-for ((i=0; i<retries; i++)); do
+for ((i=1; i<=retries; i++)); do
     # For now ignore SystemVerification error
     kubeadm join \
         #{$kubeadm_join_flags} \
@@ -194,20 +214,22 @@ for ((i=0; i<retries; i++)); do
         --token "#{KUBETOKEN}" \
         "#{MASTER_IP}:6443" && \
         break
+
+    [[ $retries -eq i ]] && { echo "Failed to run kubeadm join after 5 tries"; exit 1; }
     echo "kubeadm join failed, trying again in 3 seconds (try ${i}/${retries})..."
     sleep 3
 done
-[[ $retries -eq i ]] && { echo "Failed to run kubeadm join after 5 tries"; exit 1; }
 
-KUBELET_EXTRA_ARGS_FILE=/etc/sysconfig/kubelet
-if [ ! -f /etc/sysconfig/kubelet ]; then
-    KUBELET_EXTRA_ARGS_FILE=/etc/default/kubelet
-fi
+KUBELET_EXTRA_ARGS_FILE="#{$kubeletExtraArgsFile}"
 
-if grep -q -- '--node-ip=' "$KUBELET_EXTRA_ARGS_FILE"; then
-    sed -ri -e 's/KUBELET_EXTRA_ARGS=--node-ip=[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/KUBELET_EXTRA_ARGS=/' "$KUBELET_EXTRA_ARGS_FILE"
+if [ -f "$KUBELET_EXTRA_ARGS_FILE" ]; then
+    if grep -q -- '--node-ip=' "$KUBELET_EXTRA_ARGS_FILE"; then
+        sed -ri -e 's|KUBELET_EXTRA_ARGS=--node-ip=[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|KUBELET_EXTRA_ARGS=|' "$KUBELET_EXTRA_ARGS_FILE"
+    fi
+    sed -i 's|KUBELET_EXTRA_ARGS=|KUBELET_EXTRA_ARGS=--node-ip=#{NODE_IP} --cni-bin-dir=/opt/cni/bin,/usr/libexec/cni |' "$KUBELET_EXTRA_ARGS_FILE"
+else
+    echo "KUBELET_EXTRA_ARGS=--node-ip=#{NODE_IP} " > "$KUBELET_EXTRA_ARGS_FILE"
 fi
-sed -i 's|KUBELET_EXTRA_ARGS=|KUBELET_EXTRA_ARGS=--node-ip=#{NODE_IP} --cni-bin-dir=/opt/cni/bin,/usr/libexec/cni |' "$KUBELET_EXTRA_ARGS_FILE"
 
 systemctl daemon-reload
 systemctl restart kubelet.service
diff --git a/vagrantfiles/centos/common b/vagrantfiles/centos/common
index f999c47..2779d41 100644
--- a/vagrantfiles/centos/common
+++ b/vagrantfiles/centos/common
@@ -13,7 +13,7 @@ $containerRuntimeInstallScript = <<SCRIPT
 set -x
 
 retries=5
-for ((i=0; i<retries; i++)); do
+for ((i=1; i<=retries; i++)); do
     yum install -y yum-utils \
         device-mapper-persistent-data \
         lvm2
@@ -22,6 +22,7 @@ for ((i=0; i<retries; i++)); do
         https://download.docker.com/linux/centos/docker-ce.repo
     yum install -y docker-ce docker-ce-cli containerd.io && \
         break
+
+    [[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
 done
-[[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
 SCRIPT
diff --git a/vagrantfiles/fedora/common b/vagrantfiles/fedora/common
index 1d296d8..aaa72c1 100644
--- a/vagrantfiles/fedora/common
+++ b/vagrantfiles/fedora/common
@@ -2,8 +2,16 @@ $box_image = ENV['BOX_IMAGE'] || 'fedora/30-cloud-base'.freeze
 
 # Use iptables-legacy for iptables
 $osPrepareScript = <<SCRIPT
+set -x
+
 if [ ! -f /usr/sbin/iptables-legacy ]; then
-    dnf install -y iptables
+    retries=5
+    for ((i=1; i<=retries; i++)); do
+        dnf install -y iptables && \
+            break
+
+        [[ $retries -eq i ]] && { echo "Failed to install iptables after 5 tries"; exit 1; }
+    done
 fi
 update-alternatives --set iptables /usr/sbin/iptables-legacy
 SCRIPT
@@ -12,13 +20,14 @@ $containerRuntimeInstallScript = <<SCRIPT
 set -x
 
 retries=5
-for ((i=0; i<retries; i++)); do
+for ((i=1; i<retries; i++)); do
     dnf -y install dnf-plugins-core
     dnf config-manager \
         --add-repo \
         https://download.docker.com/linux/fedora/docker-ce.repo
     dnf install -y docker-ce docker-ce-cli containerd.io && \
         break
+
+    [[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
 done
-[[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
 SCRIPT
diff --git a/vagrantfiles/ubuntu/common b/vagrantfiles/ubuntu/common
index f085c31..5c9f014 100644
--- a/vagrantfiles/ubuntu/common
+++ b/vagrantfiles/ubuntu/common
@@ -16,10 +16,11 @@ add-apt-repository \
    stable"
 
 retries=5
-for ((i=0; i<retries; i++)); do
+for ((i=1; i<=retries; i++)); do
     apt-get update
     apt-get -y install docker-ce=#{$docker_version} && \
         break
+
+    [[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
 done
-[[ $retries -eq i ]] && { echo "Failed to install docker-ce after 5 tries"; exit 1; }
 SCRIPT
diff --git a/vagrantfiles/ubuntu/master b/vagrantfiles/ubuntu/master
index 4461f13..d998f07 100644
--- a/vagrantfiles/ubuntu/master
+++ b/vagrantfiles/ubuntu/master
@@ -13,12 +13,13 @@ else
 fi
 
 retries=5
-for ((i=0; i<retries; i++)); do
+for ((i=1; i<=retries; i++)); do
     apt-get update
     apt-get -y install screen telnet conntrack socat ${KUBERNETES_PACKAGES} && \
         break
+
+    [[ $retries -eq i ]] && { echo "Failed to install kubelet, kubeadm and other dependenices after 5 tries"; exit 1; }
 done
-[[ $retries -eq i ]] && { echo "Failed to install kubelet, kubeadm and other dependenices after 5 tries"; exit 1; }
 
 apt-mark hold kubelet kubeadm kubectl
 
diff --git a/vagrantfiles/ubuntu/pre b/vagrantfiles/ubuntu/pre
index 1742695..270963c 100644
--- a/vagrantfiles/ubuntu/pre
+++ b/vagrantfiles/ubuntu/pre
@@ -1,6 +1,8 @@
 $kube_network = 'canal'
 
 $osPrepareScript = <<SCRIPT
+set -x
+
 sed -i 's/^DNS=.*/DNS=8.8.8.8 8.8.4.4/' /etc/systemd/resolved.conf
 
 sudo update-alternatives --set iptables /usr/sbin/iptables-legacy
@@ -9,11 +11,13 @@ sudo update-alternatives --set arptables /usr/sbin/arptables-legacy
 sudo update-alternatives --set ebtables /usr/sbin/ebtables-legacy
 
 retries=5
-for ((i=0; i<retries; i++)); do
+for ((i=1; i<=retries; i++)); do
     apt-get update
     apt-get -y install apt-transport-https curl software-properties-common ca-certificates && \
         break
-done
-[[ $retries -eq i ]] && { echo "Failed to install apt-transport-https and other dependenices after 5 tries"; exit 1; }
 
+    [[ $retries -eq i ]] && { echo "Failed to install apt-transport-https and other dependenices after 5 tries"; exit 1; }
+done
 SCRIPT
+
+$kubeletExtraArgsFile = '/etc/default/kubelet'