| #cloud-config |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| --- |
| users: |
| - name: cloud |
| sudo: ALL=(ALL) NOPASSWD:ALL |
| shell: /bin/bash |
| ssh_authorized_keys: |
| {{ k8s.ssh.pub.key }} |
| |
| write_files: |
| - path: /etc/conf.d/nfs |
| permissions: '0644' |
| content: | |
| OPTS_RPC_MOUNTD="" |
| |
| - path: /etc/kubernetes/pki/cloudstack/ca.crt |
| permissions: '0644' |
| content: | |
| {{ k8s_control_node.ca.crt }} |
| |
| - path: /etc/kubernetes/pki/cloudstack/apiserver.crt |
| permissions: '0644' |
| content: | |
| {{ k8s_control_node.apiserver.crt }} |
| |
| - path: /etc/kubernetes/pki/cloudstack/apiserver.key |
| permissions: '0600' |
| content: | |
| {{ k8s_control_node.apiserver.key }} |
| |
| - path: /opt/bin/setup-kube-system |
| permissions: '0700' |
| owner: root:root |
| content: | |
| #!/bin/bash -e |
| |
| if [[ -f "/home/cloud/success" ]]; then |
| echo "Already provisioned!" |
| exit 0 |
| fi |
| |
| sysctl net.ipv4.conf.default.arp_announce=0 |
| sysctl net.ipv4.conf.default.arp_ignore=0 |
| sysctl net.ipv4.conf.all.arp_announce=0 |
| sysctl net.ipv4.conf.all.arp_ignore=0 |
| sysctl net.ipv4.conf.eth0.arp_announce=0 |
| sysctl net.ipv4.conf.eth0.arp_ignore=0 |
| sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf |
| sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf |
| sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf |
| sed -i "s/net.ipv4.conf.all.arp_ignore =.*$/net.ipv4.conf.all.arp_ignore = 0/" /etc/sysctl.conf |
| |
| ISO_MOUNT_DIR=/mnt/k8sdisk |
| BINARIES_DIR=${ISO_MOUNT_DIR}/ |
| K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ |
| ATTEMPT_ONLINE_INSTALL=false |
| setup_complete=false |
| |
| OFFLINE_INSTALL_ATTEMPT_SLEEP=15 |
| MAX_OFFLINE_INSTALL_ATTEMPTS=100 |
| offline_attempts=1 |
| MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 |
| EJECT_ISO_FROM_OS={{ k8s.eject.iso }} |
| crucial_cmd_attempts=1 |
| iso_drive_path="" |
| while true; do |
| if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then |
| echo "Warning: Offline install timed out!" |
| break |
| fi |
| set +e |
| output=`blkid -o device -t TYPE=iso9660` |
| set -e |
| if [ "$output" != "" ]; then |
| while read -r line; do |
| if [ ! -d "${ISO_MOUNT_DIR}" ]; then |
| mkdir "${ISO_MOUNT_DIR}" |
| fi |
| retval=0 |
| set +e |
| mount -o ro "${line}" "${ISO_MOUNT_DIR}" |
| retval=$? |
| set -e |
| if [ $retval -eq 0 ]; then |
| if [ -d "$BINARIES_DIR" ]; then |
| iso_drive_path="${line}" |
| break |
| else |
| umount "${line}" && rmdir "${ISO_MOUNT_DIR}" |
| fi |
| fi |
| done <<< "$output" |
| fi |
| if [ -d "$BINARIES_DIR" ]; then |
| break |
| fi |
| echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" |
| sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP |
| offline_attempts=$[$offline_attempts + 1] |
| done |
| |
| if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then |
| export PATH=$PATH:/opt/bin |
| fi |
| |
| if [ -d "$BINARIES_DIR" ]; then |
| ### Binaries available offline ### |
| echo "Installing binaries from ${BINARIES_DIR}" |
| mkdir -p /opt/cni/bin |
| tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz |
| |
| mkdir -p /opt/bin |
| tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz |
| |
| mkdir -p /opt/bin |
| cd /opt/bin |
| cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . |
| chmod +x {kubeadm,kubelet,kubectl} |
| |
| sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service |
| mkdir -p /etc/systemd/system/kubelet.service.d |
| sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf |
| |
| echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet |
| |
| output=`ls ${BINARIES_DIR}/docker/` |
| if [ "$output" != "" ]; then |
| while read -r line; do |
| crucial_cmd_attempts=1 |
| while true; do |
| if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then |
| echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" |
| break; |
| fi |
| retval=0 |
| set +e |
| ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" |
| retval=$? |
| set -e |
| if [ $retval -eq 0 ]; then |
| break; |
| fi |
| crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] |
| done |
| done <<< "$output" |
| setup_complete=true |
| fi |
| mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" |
| cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" |
| if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then |
| mkdir -p /opt/autoscaler |
| cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml |
| fi |
| if [ -e "${BINARIES_DIR}/provider.yaml" ]; then |
| mkdir -p /opt/provider |
| cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml |
| fi |
| |
| PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` |
| echo $PAUSE_IMAGE |
| if [ -n "$PAUSE_IMAGE" ]; then |
| sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml |
| fi |
| systemctl daemon-reload |
| systemctl restart containerd |
| |
| umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" |
| if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then |
| eject "${iso_drive_path}" |
| fi |
| fi |
| if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then |
| ### Binaries not available offline ### |
| RELEASE="v1.16.3" |
| CNI_VERSION="v0.7.5" |
| CRICTL_VERSION="v1.16.0" |
| echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." |
| mkdir -p /opt/cni/bin |
| curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz |
| |
| mkdir -p /opt/bin |
| curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz |
| |
| mkdir -p /opt/bin |
| cd /opt/bin |
| curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} |
| chmod +x {kubeadm,kubelet,kubectl} |
| |
| curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service |
| mkdir -p /etc/systemd/system/kubelet.service.d |
| curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf |
| fi |
| |
| systemctl enable kubelet && systemctl start kubelet |
| modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 |
| |
| if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then |
| crucial_cmd_attempts=1 |
| while true; do |
| if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then |
| echo "Warning: kubeadm pull images failed after multiple tries!" |
| break; |
| fi |
| retval=0 |
| set +e |
| kubeadm config images pull --cri-socket /run/containerd/containerd.sock |
| retval=$? |
| set -e |
| if [ $retval -eq 0 ]; then |
| break; |
| fi |
| crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] |
| done |
| fi |
| |
| - path: /opt/bin/deploy-kube-system |
| permissions: '0700' |
| owner: root:root |
| content: | |
| #!/bin/bash -e |
| |
| if [[ -f "/home/cloud/success" ]]; then |
| echo "Already provisioned!" |
| exit 0 |
| fi |
| |
| if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then |
| export PATH=$PATH:/opt/bin |
| fi |
| |
| MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 |
| crucial_cmd_attempts=1 |
| while true; do |
| if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then |
| echo "Error: kubeadm init failed!" |
| exit 1 |
| fi |
| retval=0 |
| set +e |
| kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock |
| retval=$? |
| set -e |
| if [ $retval -eq 0 ]; then |
| break; |
| fi |
| crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] |
| done |
| |
| K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ |
| |
| if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then |
| echo "setup-kube-system is running!" |
| exit 1 |
| fi |
| if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then |
| export PATH=$PATH:/opt/bin |
| fi |
| export KUBECONFIG=/etc/kubernetes/admin.conf |
| |
| mkdir -p /root/.kube |
| cp -i /etc/kubernetes/admin.conf /root/.kube/config |
| chown $(id -u):$(id -g) /root/.kube/config |
| echo export PATH=\$PATH:/opt/bin >> /root/.bashrc |
| |
| if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then |
| ### Network, dashboard configs available offline ### |
| echo "Offline configs are available!" |
| /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml |
| /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml |
| rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" |
| else |
| /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" |
| /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml |
| fi |
| |
| /opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true |
| /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true |
| /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true |
| |
| sudo touch /home/cloud/success |
| echo "true" > /home/cloud/success |
| |
| - path: /opt/bin/setup-containerd |
| permissions: '0755' |
| owner: root:root |
| content: | |
| #!/bin/bash -e |
| |
| export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" |
| export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" |
| |
| echo "creating config file for containerd" |
| containerd config default > /etc/containerd/config.toml |
| sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml |
| sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml |
| |
| echo "Restarting containerd service" |
| systemctl daemon-reload |
| systemctl restart containerd |
| |
| - path: /etc/systemd/system/deploy-kube-system.service |
| permissions: '0755' |
| owner: root:root |
| content: | |
| [Unit] |
| Requires=containerd.service |
| After=containerd.service |
| |
| [Service] |
| Type=simple |
| StartLimitInterval=0 |
| Restart=on-failure |
| ExecStart=/opt/bin/deploy-kube-system |
| |
| runcmd: |
| - chown -R cloud:cloud /home/cloud/.ssh |
| - containerd config default > /etc/containerd/config.toml |
| - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml |
| - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done |
| - /opt/bin/setup-kube-system |
| - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done |
| - systemctl start deploy-kube-system |
