| #!/usr/bin/env bash |
| |
| # Copyright 2015 The Kubernetes Authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts. |
| |
| # exit on any error |
| set -e |
| |
| SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR -C" |
| |
| # Use the config file specified in $KUBE_CONFIG_FILE, or default to |
| # config-default.sh. |
| KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. |
| readonly ROOT=$(dirname "${BASH_SOURCE}") |
| source "${ROOT}/${KUBE_CONFIG_FILE:-"config-default.sh"}" |
| source "$KUBE_ROOT/cluster/common.sh" |
| |
| |
| KUBECTL_PATH=${KUBE_ROOT}/cluster/centos/binaries/kubectl |
| |
| # Directory to be used for master and node provisioning. |
| KUBE_TEMP="~/kube_temp" |
| |
| |
| # Get master IP addresses and store in KUBE_MASTER_IP_ADDRESSES[] |
| # Must ensure that the following ENV vars are set: |
| # MASTERS |
| function detect-masters() { |
| KUBE_MASTER_IP_ADDRESSES=() |
| for master in ${MASTERS}; do |
| KUBE_MASTER_IP_ADDRESSES+=("${master#*@}") |
| done |
| echo "KUBE_MASTERS: ${MASTERS}" 1>&2 |
| echo "KUBE_MASTER_IP_ADDRESSES: [${KUBE_MASTER_IP_ADDRESSES[*]}]" 1>&2 |
| } |
| |
| # Get node IP addresses and store in KUBE_NODE_IP_ADDRESSES[] |
| function detect-nodes() { |
| KUBE_NODE_IP_ADDRESSES=() |
| for node in ${NODES}; do |
| KUBE_NODE_IP_ADDRESSES+=("${node#*@}") |
| done |
| echo "KUBE_NODE_IP_ADDRESSES: [${KUBE_NODE_IP_ADDRESSES[*]}]" 1>&2 |
| } |
| |
| # Verify prereqs on host machine |
| function verify-prereqs() { |
| local rc |
| rc=0 |
| ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" |
| # "Could not open a connection to your authentication agent." |
| if [[ "${rc}" -eq 2 ]]; then |
| eval "$(ssh-agent)" > /dev/null |
| trap-add "kill ${SSH_AGENT_PID}" EXIT |
| fi |
| rc=0 |
| ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" |
| # "The agent has no identities." |
| if [[ "${rc}" -eq 1 ]]; then |
| # Try adding one of the default identities, with or without passphrase. |
| ssh-add || true |
| fi |
| rc=0 |
| # Expect at least one identity to be available. |
| if ! ssh-add -L 1> /dev/null 2> /dev/null; then |
| echo "Could not find or add an SSH identity." |
| echo "Please start ssh-agent, add your identity, and retry." |
| exit 1 |
| fi |
| } |
| |
| # Install handler for signal trap |
| function trap-add { |
| local handler="$1" |
| local signal="${2-EXIT}" |
| local cur |
| |
| cur="$(eval "sh -c 'echo \$3' -- $(trap -p ${signal})")" |
| if [[ -n "${cur}" ]]; then |
| handler="${cur}; ${handler}" |
| fi |
| |
| trap "${handler}" ${signal} |
| } |
| |
| # Validate a kubernetes cluster |
| function validate-cluster() { |
| # by default call the generic validate-cluster.sh script, customizable by |
| # any cluster provider if this does not fit. |
| set +e |
| "${KUBE_ROOT}/cluster/validate-cluster.sh" |
| if [[ "$?" -ne "0" ]]; then |
| for master in ${MASTERS}; do |
| troubleshoot-master ${master} |
| done |
| for node in ${NODES}; do |
| troubleshoot-node ${node} |
| done |
| exit 1 |
| fi |
| set -e |
| } |
| |
| # Instantiate a kubernetes cluster |
| function kube-up() { |
| make-ca-cert |
| |
| local num_infra=0 |
| for master in ${MASTERS}; do |
| provision-master "${master}" "infra${num_infra}" |
| let ++num_infra |
| done |
| |
| for master in ${MASTERS}; do |
| post-provision-master "${master}" |
| done |
| |
| for node in ${NODES}; do |
| provision-node "${node}" |
| done |
| |
| detect-masters |
| |
| # set CONTEXT and KUBE_SERVER values for create-kubeconfig() and get-password() |
| export CONTEXT="centos" |
| export KUBE_SERVER="http://${MASTER_ADVERTISE_ADDRESS}:8080" |
| source "${KUBE_ROOT}/cluster/common.sh" |
| |
| # set kubernetes user and password |
| get-password |
| create-kubeconfig |
| } |
| |
| # Delete a kubernetes cluster |
| function kube-down() { |
| for master in ${MASTERS}; do |
| tear-down-master ${master} |
| done |
| |
| for node in ${NODES}; do |
| tear-down-node ${node} |
| done |
| } |
| |
| function troubleshoot-master() { |
| # Troubleshooting on master if all required daemons are active. |
| echo "[INFO] Troubleshooting on master $1" |
| local -a required_daemon=("kube-apiserver" "kube-controller-manager" "kube-scheduler") |
| local daemon |
| local daemon_status |
| printf "%-24s %-10s \n" "PROCESS" "STATUS" |
| for daemon in "${required_daemon[@]}"; do |
| local rc=0 |
| kube-ssh "${1}" "sudo systemctl is-active ${daemon}" >/dev/null 2>&1 || rc="$?" |
| if [[ "${rc}" -ne "0" ]]; then |
| daemon_status="inactive" |
| else |
| daemon_status="active" |
| fi |
| printf "%-24s %s\n" ${daemon} ${daemon_status} |
| done |
| printf "\n" |
| } |
| |
| function troubleshoot-node() { |
| # Troubleshooting on node if all required daemons are active. |
| echo "[INFO] Troubleshooting on node ${1}" |
| local -a required_daemon=("kube-proxy" "kubelet" "docker" "flannel") |
| local daemon |
| local daemon_status |
| printf "%-24s %-10s \n" "PROCESS" "STATUS" |
| for daemon in "${required_daemon[@]}"; do |
| local rc=0 |
| kube-ssh "${1}" "sudo systemctl is-active ${daemon}" >/dev/null 2>&1 || rc="$?" |
| if [[ "${rc}" -ne "0" ]]; then |
| daemon_status="inactive" |
| else |
| daemon_status="active" |
| fi |
| printf "%-24s %s\n" ${daemon} ${daemon_status} |
| done |
| printf "\n" |
| } |
| |
| # Clean up on master |
| function tear-down-master() { |
| echo "[INFO] tear-down-master on $1" |
| for service_name in etcd kube-apiserver kube-controller-manager kube-scheduler ; do |
| service_file="/usr/lib/systemd/system/${service_name}.service" |
| kube-ssh "$1" " \ |
| if [[ -f $service_file ]]; then \ |
| sudo systemctl stop $service_name; \ |
| sudo systemctl disable $service_name; \ |
| sudo rm -f $service_file; \ |
| fi" |
| done |
| kube-ssh "${1}" "sudo rm -rf /opt/kubernetes" |
| kube-ssh "${1}" "sudo rm -rf /srv/kubernetes" |
| kube-ssh "${1}" "sudo rm -rf ${KUBE_TEMP}" |
| kube-ssh "${1}" "sudo rm -rf /var/lib/etcd" |
| } |
| |
| # Clean up on node |
| function tear-down-node() { |
| echo "[INFO] tear-down-node on $1" |
| for service_name in kube-proxy kubelet docker flannel ; do |
| service_file="/usr/lib/systemd/system/${service_name}.service" |
| kube-ssh "$1" " \ |
| if [[ -f $service_file ]]; then \ |
| sudo systemctl stop $service_name; \ |
| sudo systemctl disable $service_name; \ |
| sudo rm -f $service_file; \ |
| fi" |
| done |
| kube-ssh "$1" "sudo rm -rf /run/flannel" |
| kube-ssh "$1" "sudo rm -rf /opt/kubernetes" |
| kube-ssh "$1" "sudo rm -rf /srv/kubernetes" |
| kube-ssh "$1" "sudo rm -rf ${KUBE_TEMP}" |
| } |
| |
| # Generate the CA certificates for k8s components |
| function make-ca-cert() { |
| echo "[INFO] make-ca-cert" |
| bash "${ROOT}/make-ca-cert.sh" "${MASTER_ADVERTISE_IP}" "IP:${MASTER_ADVERTISE_IP},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" |
| } |
| |
| # Provision master |
| # |
| # Assumed vars: |
| # $1 (master) |
| # $2 (etcd_name) |
| # KUBE_TEMP |
| # ETCD_SERVERS |
| # ETCD_INITIAL_CLUSTER |
| # SERVICE_CLUSTER_IP_RANGE |
| # MASTER_ADVERTISE_ADDRESS |
| function provision-master() { |
| echo "[INFO] Provision master on $1" |
| local master="$1" |
| local master_ip="${master#*@}" |
| local etcd_name="$2" |
| ensure-setup-dir "${master}" |
| ensure-etcd-cert "${etcd_name}" "${master_ip}" |
| |
| kube-scp "${master}" "${ROOT}/ca-cert ${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" |
| kube-scp "${master}" "${ROOT}/etcd-cert/ca.pem \ |
| ${ROOT}/etcd-cert/client.pem \ |
| ${ROOT}/etcd-cert/client-key.pem \ |
| ${ROOT}/etcd-cert/server-${etcd_name}.pem \ |
| ${ROOT}/etcd-cert/server-${etcd_name}-key.pem \ |
| ${ROOT}/etcd-cert/peer-${etcd_name}.pem \ |
| ${ROOT}/etcd-cert/peer-${etcd_name}-key.pem" "${KUBE_TEMP}/etcd-cert" |
| kube-ssh "${master}" " \ |
| sudo rm -rf /opt/kubernetes/bin; \ |
| sudo cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes; \ |
| sudo mkdir -p /srv/kubernetes/; sudo cp -f ${KUBE_TEMP}/ca-cert/* /srv/kubernetes/; \ |
| sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \ |
| sudo chmod -R +x /opt/kubernetes/bin; \ |
| sudo ln -sf /opt/kubernetes/bin/* /usr/local/bin/; \ |
| sudo bash ${KUBE_TEMP}/master/scripts/etcd.sh ${etcd_name} ${master_ip} ${ETCD_INITIAL_CLUSTER}; \ |
| sudo bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE} ${ADMISSION_CONTROL}; \ |
| sudo bash ${KUBE_TEMP}/master/scripts/controller-manager.sh ${MASTER_ADVERTISE_ADDRESS}; \ |
| sudo bash ${KUBE_TEMP}/master/scripts/scheduler.sh ${MASTER_ADVERTISE_ADDRESS}" |
| } |
| |
| # Post-provision master, run after all masters were provisioned |
| # |
| # Assumed vars: |
| # $1 (master) |
| # KUBE_TEMP |
| # ETCD_SERVERS |
| # FLANNEL_NET |
| function post-provision-master() { |
| echo "[INFO] Post provision master on $1" |
| local master=$1 |
| kube-ssh "${master}" " \ |
| sudo bash ${KUBE_TEMP}/master/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ |
| sudo bash ${KUBE_TEMP}/master/scripts/post-etcd.sh" |
| } |
| |
| # Provision node |
| # |
| # Assumed vars: |
| # $1 (node) |
| # KUBE_TEMP |
| # ETCD_SERVERS |
| # FLANNEL_NET |
| # MASTER_ADVERTISE_ADDRESS |
| # DOCKER_OPTS |
| # DNS_SERVER_IP |
| # DNS_DOMAIN |
| function provision-node() { |
| echo "[INFO] Provision node on $1" |
| local node=$1 |
| local node_ip=${node#*@} |
| local dns_ip=${DNS_SERVER_IP#*@} |
| local dns_domain=${DNS_DOMAIN#*@} |
| ensure-setup-dir ${node} |
| |
| kube-scp "${node}" "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" |
| kube-scp "${node}" "${ROOT}/etcd-cert/ca.pem \ |
| ${ROOT}/etcd-cert/client.pem \ |
| ${ROOT}/etcd-cert/client-key.pem" "${KUBE_TEMP}/etcd-cert" |
| kube-ssh "${node}" " \ |
| rm -rf /opt/kubernetes/bin; \ |
| sudo cp -r ${KUBE_TEMP}/node/bin /opt/kubernetes; \ |
| sudo chmod -R +x /opt/kubernetes/bin; \ |
| sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \ |
| sudo ln -s /opt/kubernetes/bin/* /usr/local/bin/; \ |
| sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \ |
| sudo bash ${KUBE_TEMP}/node/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ |
| sudo bash ${KUBE_TEMP}/node/scripts/docker.sh \"${DOCKER_OPTS}\"; \ |
| sudo bash ${KUBE_TEMP}/node/scripts/kubelet.sh ${MASTER_ADVERTISE_ADDRESS} ${node_ip} ${dns_ip} ${dns_domain}; \ |
| sudo bash ${KUBE_TEMP}/node/scripts/proxy.sh ${MASTER_ADVERTISE_ADDRESS}" |
| } |
| |
| # Create dirs that'll be used during setup on target machine. |
| # |
| # Assumed vars: |
| # KUBE_TEMP |
| function ensure-setup-dir() { |
| kube-ssh "${1}" "mkdir -p ${KUBE_TEMP}; \ |
| mkdir -p ${KUBE_TEMP}/etcd-cert; \ |
| sudo mkdir -p /opt/kubernetes/bin; \ |
| sudo mkdir -p /opt/kubernetes/cfg" |
| } |
| |
| # Generate certificates for etcd cluster |
| # |
| # Assumed vars: |
| # $1 (etcd member name) |
| # $2 (master ip) |
| function ensure-etcd-cert() { |
| local etcd_name="$1" |
| local master_ip="$2" |
| local cert_dir="${ROOT}/etcd-cert" |
| |
| if [[ ! -r "${cert_dir}/client.pem" || ! -r "${cert_dir}/client-key.pem" ]]; then |
| generate-etcd-cert "${cert_dir}" "${master_ip}" "client" "client" |
| fi |
| |
| generate-etcd-cert "${cert_dir}" "${master_ip}" "server" "server-${etcd_name}" |
| generate-etcd-cert "${cert_dir}" "${master_ip}" "peer" "peer-${etcd_name}" |
| } |
| |
| # Run command over ssh |
| function kube-ssh() { |
| local host="$1" |
| shift |
| ssh ${SSH_OPTS} -t "${host}" "$@" >/dev/null 2>&1 |
| } |
| |
| # Copy file recursively over ssh |
| function kube-scp() { |
| local host="$1" |
| local src=($2) |
| local dst="$3" |
| scp -r ${SSH_OPTS} ${src[*]} "${host}:${dst}" |
| } |
| |
| # Ensure that we have a password created for validating to the master. Will |
| # read from kubeconfig if available. |
| # |
| # Vars set: |
| # KUBE_USER |
| # KUBE_PASSWORD |
| function get-password { |
| load-or-gen-kube-basicauth |
| if [[ -z "${KUBE_USER}" || -z "${KUBE_PASSWORD}" ]]; then |
| KUBE_USER=admin |
| KUBE_PASSWORD=$(python -c 'import string,random; \ |
| print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))') |
| fi |
| } |