[GitHub] [cloudstack] poussa commented on a change in pull request #3680: [WIP: DO NOT MERGE] CloudStack Kubernetes Service
poussa commented on a change in pull request #3680: [WIP: DO NOT MERGE]
CloudStack Kubernetes Service
URL: https://github.com/apache/cloudstack/pull/3680#discussion_r366173663
##
File path:
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml
##
@@ -0,0 +1,265 @@
+#cloud-config
+
+---
+ssh_authorized_keys:
+ {{ k8s.ssh.pub.key }}
+
+write-files:
+ - path: /etc/conf.d/nfs
+permissions: '0644'
+content: |
+ OPTS_RPC_MOUNTD=""
+
+ - path: /etc/kubernetes/pki/cloudstack/ca.crt
+permissions: '0644'
+content: |
+ {{ k8s_master.ca.crt }}
+
+ - path: /etc/kubernetes/pki/cloudstack/apiserver.crt
+permissions: '0644'
+content: |
+ {{ k8s_master.apiserver.crt }}
+
+ - path: /etc/kubernetes/pki/cloudstack/apiserver.key
+permissions: '0600'
+content: |
+ {{ k8s_master.apiserver.key }}
+
+ - path: /opt/bin/setup-kube-system
+permissions: 0700
+owner: root:root
+content: |
+ #!/bin/bash -e
+
+ if [[ -f "/home/core/success" ]]; then
+ echo "Already provisioned!"
+ exit 0
+ fi
+
+ export PATH=$PATH:/opt/bin
+
+ ISO_MOUNT_DIR=/mnt/k8sdisk
+ BINARIES_DIR=${ISO_MOUNT_DIR}/
+ ATTEMPT_ONLINE_INSTALL=false
+ setup_complete=false
+
+ OFFLINE_INSTALL_ATTEMPT_SLEEP=5
+ MAX_OFFLINE_INSTALL_ATTEMPTS=36
+ offline_attempts=1
+ MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
+ crucial_cmd_attempts=1
+ while true; do
+if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then
+ echo "Warning: Offline install timed out!"
+ break
+fi
+set +e
+output=`blkid -o device -t TYPE=iso9660`
+set -e
+if [ "$output" != "" ]; then
+ while read -r line; do
+if [ ! -d "${ISO_MOUNT_DIR}" ]; then
+ mkdir "${ISO_MOUNT_DIR}"
+fi
+retval=0
+set +e
+mount -o ro "${line}" "${ISO_MOUNT_DIR}"
+retval=$?
+set -e
+if [ $retval -eq 0 ]; then
+ if [ -d "$BINARIES_DIR" ]; then
+break
+ else
+umount "${line}" && rmdir "${ISO_MOUNT_DIR}"
+ fi
+fi
+ done <<< "$output"
+fi
+if [ -d "$BINARIES_DIR" ]; then
+ break
+fi
+echo "Waiting for Binaries directory $BINARIES_DIR to be available,
sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts"
+sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP
+offline_attempts=$[$offline_attempts + 1]
+ done
+
+ if [ -d "$BINARIES_DIR" ]; then
+### Binaries available offline ###
+echo "Installing binaries from ${BINARIES_DIR}"
+mkdir -p /opt/cni/bin
+tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz
+
+mkdir -p /opt/bin
+tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C
/opt/bin -xz
+
+mkdir -p /opt/bin
+cd /opt/bin
+cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin
+chmod +x {kubeadm,kubelet,kubectl}
+
+sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service >
/etc/systemd/system/kubelet.service
+mkdir -p /etc/systemd/system/kubelet.service.d
+sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf >
/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+output=`ls ${BINARIES_DIR}/docker/`
+if [ "$output" != "" ]; then
+ while read -r line; do
+crucial_cmd_attempts=1
+while true; do
+ if (( "$crucial_cmd_attempts" >
"$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
+echo "Loading docker image ${BINARIES_DIR}/docker/$line
failed!"
+break;
+ fi
+ retval=0
+ set +e
+ docker load < "${BINARIES_DIR}/docker/$line"
+ retval=$?
+ set -e
+ if [ $retval -eq 0 ]; then
+break;
+ fi
+ crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
+done
+ done <<< "$output"
+ setup_complete=true
+fi
+umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+ fi
+ if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true
]; then
+### Binaries not available offline ###
+RELEASE="v1.16.3"
+CNI_VERSION="v0.7.5"
+CRICTL_VERSION="v1.16.0"
+echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker
images from Internet."
+mkdir -p /opt/cni/bin
+curl -L
"https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz";
| tar -C /opt/cni/bin -xz
+
+mkdir -p /opt/bin
+curl -L
"https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRI
[GitHub] [cloudstack] poussa commented on a change in pull request #3680: [WIP: DO NOT MERGE] CloudStack Kubernetes Service
poussa commented on a change in pull request #3680: [WIP: DO NOT MERGE]
CloudStack Kubernetes Service
URL: https://github.com/apache/cloudstack/pull/3680#discussion_r366208602
##
File path:
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml
##
@@ -0,0 +1,265 @@
+#cloud-config
+
+---
+ssh_authorized_keys:
+ {{ k8s.ssh.pub.key }}
+
+write-files:
+ - path: /etc/conf.d/nfs
+permissions: '0644'
+content: |
+ OPTS_RPC_MOUNTD=""
+
+ - path: /etc/kubernetes/pki/cloudstack/ca.crt
+permissions: '0644'
+content: |
+ {{ k8s_master.ca.crt }}
+
+ - path: /etc/kubernetes/pki/cloudstack/apiserver.crt
+permissions: '0644'
+content: |
+ {{ k8s_master.apiserver.crt }}
+
+ - path: /etc/kubernetes/pki/cloudstack/apiserver.key
+permissions: '0600'
+content: |
+ {{ k8s_master.apiserver.key }}
+
+ - path: /opt/bin/setup-kube-system
+permissions: 0700
+owner: root:root
+content: |
+ #!/bin/bash -e
+
+ if [[ -f "/home/core/success" ]]; then
+ echo "Already provisioned!"
+ exit 0
+ fi
+
+ export PATH=$PATH:/opt/bin
+
+ ISO_MOUNT_DIR=/mnt/k8sdisk
+ BINARIES_DIR=${ISO_MOUNT_DIR}/
+ ATTEMPT_ONLINE_INSTALL=false
+ setup_complete=false
+
+ OFFLINE_INSTALL_ATTEMPT_SLEEP=5
+ MAX_OFFLINE_INSTALL_ATTEMPTS=36
+ offline_attempts=1
+ MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
+ crucial_cmd_attempts=1
+ while true; do
+if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then
+ echo "Warning: Offline install timed out!"
+ break
+fi
+set +e
+output=`blkid -o device -t TYPE=iso9660`
+set -e
+if [ "$output" != "" ]; then
+ while read -r line; do
+if [ ! -d "${ISO_MOUNT_DIR}" ]; then
+ mkdir "${ISO_MOUNT_DIR}"
+fi
+retval=0
+set +e
+mount -o ro "${line}" "${ISO_MOUNT_DIR}"
+retval=$?
+set -e
+if [ $retval -eq 0 ]; then
+ if [ -d "$BINARIES_DIR" ]; then
+break
+ else
+umount "${line}" && rmdir "${ISO_MOUNT_DIR}"
+ fi
+fi
+ done <<< "$output"
+fi
+if [ -d "$BINARIES_DIR" ]; then
+ break
+fi
+echo "Waiting for Binaries directory $BINARIES_DIR to be available,
sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts"
+sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP
+offline_attempts=$[$offline_attempts + 1]
+ done
+
+ if [ -d "$BINARIES_DIR" ]; then
+### Binaries available offline ###
+echo "Installing binaries from ${BINARIES_DIR}"
+mkdir -p /opt/cni/bin
+tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz
+
+mkdir -p /opt/bin
+tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C
/opt/bin -xz
+
+mkdir -p /opt/bin
+cd /opt/bin
+cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin
+chmod +x {kubeadm,kubelet,kubectl}
+
+sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service >
/etc/systemd/system/kubelet.service
+mkdir -p /etc/systemd/system/kubelet.service.d
+sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf >
/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+output=`ls ${BINARIES_DIR}/docker/`
+if [ "$output" != "" ]; then
+ while read -r line; do
+crucial_cmd_attempts=1
+while true; do
+ if (( "$crucial_cmd_attempts" >
"$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
+echo "Loading docker image ${BINARIES_DIR}/docker/$line
failed!"
+break;
+ fi
+ retval=0
+ set +e
+ docker load < "${BINARIES_DIR}/docker/$line"
+ retval=$?
+ set -e
+ if [ $retval -eq 0 ]; then
+break;
+ fi
+ crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
+done
+ done <<< "$output"
+ setup_complete=true
+fi
+umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+ fi
+ if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true
]; then
+### Binaries not available offline ###
+RELEASE="v1.16.3"
+CNI_VERSION="v0.7.5"
+CRICTL_VERSION="v1.16.0"
+echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker
images from Internet."
+mkdir -p /opt/cni/bin
+curl -L
"https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz";
| tar -C /opt/cni/bin -xz
+
+mkdir -p /opt/bin
+curl -L
"https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRI
[GitHub] [cloudstack] poussa commented on a change in pull request #3680: [WIP: DO NOT MERGE] CloudStack Kubernetes Service
poussa commented on a change in pull request #3680: [WIP: DO NOT MERGE]
CloudStack Kubernetes Service
URL: https://github.com/apache/cloudstack/pull/3680#discussion_r367256989
##
File path:
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml
##
@@ -0,0 +1,265 @@
+#cloud-config
+
+---
+ssh_authorized_keys:
+ {{ k8s.ssh.pub.key }}
+
+write-files:
+ - path: /etc/conf.d/nfs
+permissions: '0644'
+content: |
+ OPTS_RPC_MOUNTD=""
+
+ - path: /etc/kubernetes/pki/cloudstack/ca.crt
+permissions: '0644'
+content: |
+ {{ k8s_master.ca.crt }}
+
+ - path: /etc/kubernetes/pki/cloudstack/apiserver.crt
+permissions: '0644'
+content: |
+ {{ k8s_master.apiserver.crt }}
+
+ - path: /etc/kubernetes/pki/cloudstack/apiserver.key
+permissions: '0600'
+content: |
+ {{ k8s_master.apiserver.key }}
+
+ - path: /opt/bin/setup-kube-system
+permissions: 0700
+owner: root:root
+content: |
+ #!/bin/bash -e
+
+ if [[ -f "/home/core/success" ]]; then
+ echo "Already provisioned!"
+ exit 0
+ fi
+
+ export PATH=$PATH:/opt/bin
+
+ ISO_MOUNT_DIR=/mnt/k8sdisk
+ BINARIES_DIR=${ISO_MOUNT_DIR}/
+ ATTEMPT_ONLINE_INSTALL=false
+ setup_complete=false
+
+ OFFLINE_INSTALL_ATTEMPT_SLEEP=5
+ MAX_OFFLINE_INSTALL_ATTEMPTS=36
+ offline_attempts=1
+ MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
+ crucial_cmd_attempts=1
+ while true; do
+if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then
+ echo "Warning: Offline install timed out!"
+ break
+fi
+set +e
+output=`blkid -o device -t TYPE=iso9660`
+set -e
+if [ "$output" != "" ]; then
+ while read -r line; do
+if [ ! -d "${ISO_MOUNT_DIR}" ]; then
+ mkdir "${ISO_MOUNT_DIR}"
+fi
+retval=0
+set +e
+mount -o ro "${line}" "${ISO_MOUNT_DIR}"
+retval=$?
+set -e
+if [ $retval -eq 0 ]; then
+ if [ -d "$BINARIES_DIR" ]; then
+break
+ else
+umount "${line}" && rmdir "${ISO_MOUNT_DIR}"
+ fi
+fi
+ done <<< "$output"
+fi
+if [ -d "$BINARIES_DIR" ]; then
+ break
+fi
+echo "Waiting for Binaries directory $BINARIES_DIR to be available,
sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts"
+sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP
+offline_attempts=$[$offline_attempts + 1]
+ done
+
+ if [ -d "$BINARIES_DIR" ]; then
+### Binaries available offline ###
+echo "Installing binaries from ${BINARIES_DIR}"
+mkdir -p /opt/cni/bin
+tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz
+
+mkdir -p /opt/bin
+tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C
/opt/bin -xz
+
+mkdir -p /opt/bin
+cd /opt/bin
+cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin
+chmod +x {kubeadm,kubelet,kubectl}
+
+sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service >
/etc/systemd/system/kubelet.service
+mkdir -p /etc/systemd/system/kubelet.service.d
+sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf >
/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+output=`ls ${BINARIES_DIR}/docker/`
+if [ "$output" != "" ]; then
+ while read -r line; do
+crucial_cmd_attempts=1
+while true; do
+ if (( "$crucial_cmd_attempts" >
"$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
+echo "Loading docker image ${BINARIES_DIR}/docker/$line
failed!"
+break;
+ fi
+ retval=0
+ set +e
+ docker load < "${BINARIES_DIR}/docker/$line"
+ retval=$?
+ set -e
+ if [ $retval -eq 0 ]; then
+break;
+ fi
+ crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
+done
+ done <<< "$output"
+ setup_complete=true
+fi
+umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+ fi
+ if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true
]; then
+### Binaries not available offline ###
+RELEASE="v1.16.3"
+CNI_VERSION="v0.7.5"
+CRICTL_VERSION="v1.16.0"
+echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker
images from Internet."
+mkdir -p /opt/cni/bin
+curl -L
"https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz";
| tar -C /opt/cni/bin -xz
+
+mkdir -p /opt/bin
+curl -L
"https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRI
