| #!/usr/bin/env bash |
| |
| # Copyright 2015 The Kubernetes Authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # Script that starts kubelet on kubemark-master as a supervisord process |
| # and then runs the master components as pods using kubelet. |
| |
| set -o errexit |
| set -o nounset |
| set -o pipefail |
| |
| # Define key path variables. |
| KUBE_ROOT="/home/kubernetes" |
| KUBE_BINDIR="${KUBE_ROOT}/kubernetes/server/bin" |
| |
| function config-ip-firewall { |
| echo "Configuring IP firewall rules" |
| # The GCI image has host firewall which drop most inbound/forwarded packets. |
| # We need to add rules to accept all TCP/UDP/ICMP packets. |
| if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then |
| echo "Add rules to accept all inbound TCP/UDP/ICMP packets" |
| iptables -A INPUT -w -p TCP -j ACCEPT |
| iptables -A INPUT -w -p UDP -j ACCEPT |
| iptables -A INPUT -w -p ICMP -j ACCEPT |
| fi |
| if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then |
| echo "Add rules to accept all forwarded TCP/UDP/ICMP packets" |
| iptables -A FORWARD -w -p TCP -j ACCEPT |
| iptables -A FORWARD -w -p UDP -j ACCEPT |
| iptables -A FORWARD -w -p ICMP -j ACCEPT |
| fi |
| } |
| |
| function create-dirs { |
| echo "Creating required directories" |
| mkdir -p /var/lib/kubelet |
| mkdir -p /etc/kubernetes/manifests |
| mkdir -p /etc/kubernetes/addons |
| } |
| |
| # Setup working directory for kubelet. |
| function setup-kubelet-dir { |
| echo "Making /var/lib/kubelet executable for kubelet" |
| mount -B /var/lib/kubelet /var/lib/kubelet/ |
| mount -B -o remount,exec,suid,dev /var/lib/kubelet |
| } |
| |
| # Remove any default etcd config dirs/files. |
| function delete-default-etcd-configs { |
| if [[ -d /etc/etcd ]]; then |
| rm -rf /etc/etcd |
| fi |
| if [[ -e /etc/default/etcd ]]; then |
| rm -f /etc/default/etcd |
| fi |
| if [[ -e /etc/systemd/system/etcd.service ]]; then |
| rm -f /etc/systemd/system/etcd.service |
| fi |
| if [[ -e /etc/init.d/etcd ]]; then |
| rm -f /etc/init.d/etcd |
| fi |
| } |
| |
| # Compute etcd related variables. |
| function compute-etcd-variables { |
| ETCD_IMAGE="${ETCD_IMAGE:-}" |
| ETCD_QUOTA_BYTES="" |
| if [ "${ETCD_VERSION:0:2}" == "3." ]; then |
| # TODO: Set larger quota to see if that helps with |
| # 'mvcc: database space exceeded' errors. If so, pipe |
| # though our setup scripts. |
| ETCD_QUOTA_BYTES=" --quota-backend-bytes=4294967296 " |
| fi |
| } |
| |
| # Formats the given device ($1) if needed and mounts it at given mount point |
| # ($2). |
| function safe-format-and-mount() { |
| device=$1 |
| mountpoint=$2 |
| |
| # Format only if the disk is not already formatted. |
| if ! tune2fs -l "${device}" ; then |
| echo "Formatting '${device}'" |
| mkfs.ext4 -F "${device}" |
| fi |
| |
| echo "Mounting '${device}' at '${mountpoint}'" |
| mount -o discard,defaults "${device}" "${mountpoint}" |
| } |
| |
| # Finds a PD device with name '$1' attached to the master. |
| function find-attached-pd() { |
| local -r pd_name=$1 |
| if [[ ! -e /dev/disk/by-id/${pd_name} ]]; then |
| echo "" |
| fi |
| device_info=$(ls -l /dev/disk/by-id/${pd_name}) |
| relative_path=${device_info##* } |
| echo "/dev/disk/by-id/${relative_path}" |
| } |
| |
| # Mounts a persistent disk (formatting if needed) to store the persistent data |
| # on the master. safe-format-and-mount only formats an unformatted disk, and |
| # mkdir -p will leave a directory be if it already exists. |
| function mount-pd() { |
| local -r pd_name=$1 |
| local -r mount_point=$2 |
| |
| if [[ -z "${find-attached-pd ${pd_name}}" ]]; then |
| echo "Can't find ${pd_name}. Skipping mount." |
| return |
| fi |
| |
| local -r pd_path="/dev/disk/by-id/${pd_name}" |
| echo "Mounting PD '${pd_path}' at '${mount_point}'" |
| # Format and mount the disk, create directories on it for all of the master's |
| # persistent data, and link them to where they're used. |
| mkdir -p "${mount_point}" |
| safe-format-and-mount "${pd_path}" "${mount_point}" |
| echo "Mounted PD '${pd_path}' at '${mount_point}'" |
| |
| # NOTE: These locations on the PD store persistent data, so to maintain |
| # upgradeability, these locations should not change. If they do, take care |
| # to maintain a migration path from these locations to whatever new |
| # locations. |
| } |
| |
| # Create kubeconfig for controller-manager's service account authentication. |
| function create-kubecontrollermanager-kubeconfig { |
| echo "Creating kube-controller-manager kubeconfig file" |
| mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager" |
| cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" |
| apiVersion: v1 |
| kind: Config |
| users: |
| - name: kube-controller-manager |
| user: |
| token: ${KUBE_CONTROLLER_MANAGER_TOKEN} |
| clusters: |
| - name: local |
| cluster: |
| insecure-skip-tls-verify: true |
| server: https://localhost:443 |
| contexts: |
| - context: |
| cluster: local |
| user: kube-controller-manager |
| name: service-account-context |
| current-context: service-account-context |
| EOF |
| } |
| |
| function create-kubescheduler-kubeconfig { |
| echo "Creating kube-scheduler kubeconfig file" |
| mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-scheduler" |
| cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" |
| apiVersion: v1 |
| kind: Config |
| users: |
| - name: kube-scheduler |
| user: |
| token: ${KUBE_SCHEDULER_TOKEN} |
| clusters: |
| - name: local |
| cluster: |
| insecure-skip-tls-verify: true |
| server: https://localhost:443 |
| contexts: |
| - context: |
| cluster: local |
| user: kube-scheduler |
| name: kube-scheduler |
| current-context: kube-scheduler |
| EOF |
| } |
| |
| function assemble-docker-flags { |
| echo "Assemble docker command line flags" |
| local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" |
| docker_opts+=" --log-level=debug" # Since it's a test cluster |
| # TODO(shyamjvs): Incorporate network plugin options, etc later. |
| echo "DOCKER_OPTS=\"${docker_opts}\"" > /etc/default/docker |
| echo "DOCKER_NOFILE=65536" >> /etc/default/docker # For setting ulimit -n |
| systemctl restart docker |
| } |
| |
| # A helper function for loading a docker image. It keeps trying up to 5 times. |
| # |
| # $1: Full path of the docker image |
| function try-load-docker-image { |
| local -r img=$1 |
| echo "Try to load docker image file ${img}" |
| # Temporarily turn off errexit, because we don't want to exit on first failure. |
| set +e |
| local -r max_attempts=5 |
| local -i attempt_num=1 |
| until timeout 30 docker load -i "${img}"; do |
| if [[ "${attempt_num}" == "${max_attempts}" ]]; then |
| echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!" |
| exit 1 |
| else |
| attempt_num=$((attempt_num+1)) |
| sleep 5 |
| fi |
| done |
| # Re-enable errexit. |
| set -e |
| } |
| |
| # Loads kube-system docker images. It is better to do it before starting kubelet, |
| # as kubelet will restart docker daemon, which may interfere with loading images. |
| function load-docker-images { |
| echo "Start loading kube-system docker images" |
| local -r img_dir="${KUBE_BINDIR}" |
| try-load-docker-image "${img_dir}/kube-apiserver.tar" |
| try-load-docker-image "${img_dir}/kube-controller-manager.tar" |
| try-load-docker-image "${img_dir}/kube-scheduler.tar" |
| } |
| |
| # Computes command line arguments to be passed to kubelet. |
| function compute-kubelet-params { |
| local params="${KUBELET_TEST_ARGS:-}" |
| params+=" --allow-privileged=true" |
| params+=" --cgroup-root=/" |
| params+=" --cloud-provider=gce" |
| params+=" --pod-manifest-path=/etc/kubernetes/manifests" |
| if [[ -n "${KUBELET_PORT:-}" ]]; then |
| params+=" --port=${KUBELET_PORT}" |
| fi |
| params+=" --enable-debugging-handlers=false" |
| params+=" --hairpin-mode=none" |
| echo "${params}" |
| } |
| |
| # Creates the systemd config file for kubelet.service. |
| function create-kubelet-conf() { |
| local -r kubelet_bin="$1" |
| local -r kubelet_env_file="/etc/default/kubelet" |
| local -r flags=$(compute-kubelet-params) |
| echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}" |
| |
| # Write the systemd service file for kubelet. |
| cat <<EOF >/etc/systemd/system/kubelet.service |
| [Unit] |
| Description=Kubermark kubelet |
| Requires=network-online.target |
| After=network-online.target |
| |
| [Service] |
| Restart=always |
| RestartSec=10 |
| EnvironmentFile=${kubelet_env_file} |
| ExecStart=${kubelet_bin} \$KUBELET_OPTS |
| |
| [Install] |
| WantedBy=multi-user.target |
| EOF |
| } |
| |
| # This function assembles the kubelet systemd service file and starts it using |
| # systemctl, on the kubemark master. |
| function start-kubelet { |
| # Create systemd config. |
| local -r kubelet_bin="/usr/bin/kubelet" |
| create-kubelet-conf "${kubelet_bin}" |
| |
| # Flush iptables nat table |
| iptables -t nat -F || true |
| |
| # Start the kubelet service. |
| systemctl start kubelet.service |
| } |
| |
| # Create the log file and set its properties. |
| # |
| # $1 is the file to create. |
| function prepare-log-file { |
| touch $1 |
| chmod 644 $1 |
| chown root:root $1 |
| } |
| |
| # A helper function for copying addon manifests and set dir/files |
| # permissions. |
| # |
| # $1: addon category under /etc/kubernetes |
| # $2: manifest source dir |
| function setup-addon-manifests { |
| local -r src_dir="${KUBE_ROOT}/$2" |
| local -r dst_dir="/etc/kubernetes/$1/$2" |
| if [[ ! -d "${dst_dir}" ]]; then |
| mkdir -p "${dst_dir}" |
| fi |
| local files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml") |
| if [[ -n "${files}" ]]; then |
| cp "${src_dir}/"*.yaml "${dst_dir}" |
| fi |
| chown -R root:root "${dst_dir}" |
| chmod 755 "${dst_dir}" |
| chmod 644 "${dst_dir}"/* |
| } |
| |
| # Write the config for the audit policy. |
| # Note: This duplicates the function in cluster/gce/gci/configure-helper.sh. |
| # TODO: Get rid of this function when #53321 is fixed. |
| function create-master-audit-policy { |
| local -r path="${1}" |
| local -r policy="${2:-}" |
| |
| if [[ -n "${policy}" ]]; then |
| echo "${policy}" > "${path}" |
| return |
| fi |
| |
| # Known api groups |
| local -r known_apis=' |
| - group: "" # core |
| - group: "admissionregistration.k8s.io" |
| - group: "apiextensions.k8s.io" |
| - group: "apiregistration.k8s.io" |
| - group: "apps" |
| - group: "authentication.k8s.io" |
| - group: "authorization.k8s.io" |
| - group: "autoscaling" |
| - group: "batch" |
| - group: "certificates.k8s.io" |
| - group: "extensions" |
| - group: "metrics" |
| - group: "networking.k8s.io" |
| - group: "policy" |
| - group: "rbac.authorization.k8s.io" |
| - group: "settings.k8s.io" |
| - group: "storage.k8s.io"' |
| |
| cat <<EOF >"${path}" |
| apiVersion: audit.k8s.io/v1 |
| kind: Policy |
| rules: |
| # The following requests were manually identified as high-volume and low-risk, |
| # so drop them. |
| - level: None |
| users: ["system:kube-proxy"] |
| verbs: ["watch"] |
| resources: |
| - group: "" # core |
| resources: ["endpoints", "services", "services/status"] |
| - level: None |
| # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. |
| # TODO(#46983): Change this to the ingress controller service account. |
| users: ["system:unsecured"] |
| namespaces: ["kube-system"] |
| verbs: ["get"] |
| resources: |
| - group: "" # core |
| resources: ["configmaps"] |
| - level: None |
| users: ["kubelet"] # legacy kubelet identity |
| verbs: ["get"] |
| resources: |
| - group: "" # core |
| resources: ["nodes", "nodes/status"] |
| - level: None |
| userGroups: ["system:nodes"] |
| verbs: ["get"] |
| resources: |
| - group: "" # core |
| resources: ["nodes", "nodes/status"] |
| - level: None |
| users: |
| - system:kube-controller-manager |
| - system:kube-scheduler |
| - system:serviceaccount:kube-system:endpoint-controller |
| verbs: ["get", "update"] |
| namespaces: ["kube-system"] |
| resources: |
| - group: "" # core |
| resources: ["endpoints"] |
| - level: None |
| users: ["system:apiserver"] |
| verbs: ["get"] |
| resources: |
| - group: "" # core |
| resources: ["namespaces", "namespaces/status", "namespaces/finalize"] |
| # Don't log HPA fetching metrics. |
| - level: None |
| users: |
| - system:kube-controller-manager |
| verbs: ["get", "list"] |
| resources: |
| - group: "metrics" |
| # Don't log these read-only URLs. |
| - level: None |
| nonResourceURLs: |
| - /healthz* |
| - /version |
| - /swagger* |
| # Don't log events requests. |
| - level: None |
| resources: |
| - group: "" # core |
| resources: ["events"] |
| # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes |
| - level: Request |
| users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] |
| verbs: ["update","patch"] |
| resources: |
| - group: "" # core |
| resources: ["nodes/status", "pods/status"] |
| omitStages: |
| - "RequestReceived" |
| - level: Request |
| userGroups: ["system:nodes"] |
| verbs: ["update","patch"] |
| resources: |
| - group: "" # core |
| resources: ["nodes/status", "pods/status"] |
| omitStages: |
| - "RequestReceived" |
| # deletecollection calls can be large, don't log responses for expected namespace deletions |
| - level: Request |
| users: ["system:serviceaccount:kube-system:namespace-controller"] |
| verbs: ["deletecollection"] |
| omitStages: |
| - "RequestReceived" |
| # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, |
| # so only log at the Metadata level. |
| - level: Metadata |
| resources: |
| - group: "" # core |
| resources: ["secrets", "configmaps"] |
| - group: authentication.k8s.io |
| resources: ["tokenreviews"] |
| omitStages: |
| - "RequestReceived" |
| # Get repsonses can be large; skip them. |
| - level: Request |
| verbs: ["get", "list", "watch"] |
| resources: ${known_apis} |
| omitStages: |
| - "RequestReceived" |
| # Default level for known APIs |
| - level: RequestResponse |
| resources: ${known_apis} |
| omitStages: |
| - "RequestReceived" |
| # Default level for all other requests. |
| - level: Metadata |
| omitStages: |
| - "RequestReceived" |
| EOF |
| } |
| |
| # Computes command line arguments to be passed to etcd. |
| function compute-etcd-params { |
| local params="${ETCD_TEST_ARGS:-}" |
| params+=" --listen-peer-urls=http://127.0.0.1:2380" |
| params+=" --advertise-client-urls=http://127.0.0.1:2379" |
| params+=" --listen-client-urls=http://0.0.0.0:2379" |
| params+=" --data-dir=/var/etcd/data" |
| params+=" ${ETCD_QUOTA_BYTES}" |
| echo "${params}" |
| } |
| |
| # Computes command line arguments to be passed to etcd-events. |
| function compute-etcd-events-params { |
| local params="${ETCD_TEST_ARGS:-}" |
| params+=" --listen-peer-urls=http://127.0.0.1:2381" |
| params+=" --advertise-client-urls=http://127.0.0.1:4002" |
| params+=" --listen-client-urls=http://0.0.0.0:4002" |
| params+=" --data-dir=/var/etcd/data-events" |
| params+=" ${ETCD_QUOTA_BYTES}" |
| echo "${params}" |
| } |
| |
| # Computes command line arguments to be passed to apiserver. |
| function compute-kube-apiserver-params { |
| local params="${APISERVER_TEST_ARGS:-}" |
| params+=" --insecure-bind-address=0.0.0.0" |
| if [[ -z "${ETCD_SERVERS:-}" ]]; then |
| params+=" --etcd-servers=http://127.0.0.1:2379" |
| params+=" --etcd-servers-overrides=/events#${EVENT_STORE_URL}" |
| else |
| params+=" --etcd-servers=${ETCD_SERVERS}" |
| fi |
| params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" |
| params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" |
| params+=" --requestheader-client-ca-file=/etc/srv/kubernetes/aggr_ca.crt" |
| params+=" --requestheader-allowed-names=aggregator" |
| params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-" |
| params+=" --requestheader-group-headers=X-Remote-Group" |
| params+=" --requestheader-username-headers=X-Remote-User" |
| params+=" --proxy-client-cert-file=/etc/srv/kubernetes/proxy_client.crt" |
| params+=" --proxy-client-key-file=/etc/srv/kubernetes/proxy_client.key" |
| params+=" --enable-aggregator-routing=true" |
| params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt" |
| params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" |
| params+=" --secure-port=443" |
| params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" |
| params+=" --target-ram-mb=$((${NUM_NODES} * 60))" |
| params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" |
| params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}" |
| params+=" --authorization-mode=Node,RBAC" |
| params+=" --allow-privileged=true" |
| if [[ -n "${STORAGE_BACKEND:-}" ]]; then |
| params+=" --storage-backend=${STORAGE_BACKEND}" |
| fi |
| if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then |
| params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" |
| fi |
| if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then |
| params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" |
| fi |
| if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then |
| params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" |
| fi |
| if [[ "${NUM_NODES}" -ge 3000 ]]; then |
| params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000" |
| elif [[ "${NUM_NODES}" -ge 1000 ]]; then |
| params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500" |
| fi |
| if [[ -n "${RUNTIME_CONFIG:-}" ]]; then |
| params+=" --runtime-config=${RUNTIME_CONFIG}" |
| fi |
| if [[ -n "${FEATURE_GATES:-}" ]]; then |
| params+=" --feature-gates=${FEATURE_GATES}" |
| fi |
| if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then |
| # Create the audit policy file, and mount it into the apiserver pod. |
| create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}" |
| |
| # The config below matches the one in cluster/gce/gci/configure-helper.sh. |
| # TODO: Currently supporting just log backend. Support webhook if needed. |
| params+=" --audit-policy-file=${audit_policy_file}" |
| params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" |
| params+=" --audit-log-maxage=0" |
| params+=" --audit-log-maxbackup=0" |
| params+=" --audit-log-maxsize=2000000000" |
| fi |
| echo "${params}" |
| } |
| |
| # Computes command line arguments to be passed to controller-manager. |
| function compute-kube-controller-manager-params { |
| local params="${CONTROLLER_MANAGER_TEST_ARGS:-}" |
| params+=" --use-service-account-credentials" |
| params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig" |
| params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key" |
| params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt" |
| params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" |
| params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" |
| params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" |
| params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" |
| echo "${params}" |
| } |
| |
| # Computes command line arguments to be passed to scheduler. |
| function compute-kube-scheduler-params { |
| local params="${SCHEDULER_TEST_ARGS:-}" |
| params+=" --kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig" |
| echo "${params}" |
| } |
| |
| # Computes command line arguments to be passed to addon-manager. |
| function compute-kube-addon-manager-params { |
| echo "" |
| } |
| |
| # Start a kubernetes master component '$1' which can be any of the following: |
| # 1. etcd |
| # 2. etcd-events |
| # 3. kube-apiserver |
| # 4. kube-controller-manager |
| # 5. kube-scheduler |
| # 6. kube-addon-manager |
| # |
| # It prepares the log file, loads the docker tag, calculates variables, sets them |
| # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. |
| # |
| # Assumed vars: |
| # DOCKER_REGISTRY |
| function start-kubemaster-component() { |
| echo "Start master component $1" |
| local -r component=$1 |
| prepare-log-file /var/log/"${component}".log |
| local -r src_file="${KUBE_ROOT}/${component}.yaml" |
| local -r params=$(compute-${component}-params) |
| |
| # Evaluate variables. |
| sed -i -e "s@{{params}}@${params}@g" "${src_file}" |
| sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" "${src_file}" |
| sed -i -e "s@{{instance_prefix}}@${INSTANCE_PREFIX}@g" "${src_file}" |
| if [ "${component:0:4}" == "etcd" ]; then |
| sed -i -e "s@{{etcd_image}}@${ETCD_IMAGE}@g" "${src_file}" |
| elif [ "${component}" == "kube-addon-manager" ]; then |
| setup-addon-manifests "addons" "kubemark-rbac-bindings" |
| else |
| local -r component_docker_tag=$(cat ${KUBE_BINDIR}/${component}.docker_tag) |
| sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}" |
| if [ "${component}" == "kube-apiserver" ]; then |
| local audit_policy_config_mount="" |
| local audit_policy_config_volume="" |
| if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then |
| read -d '' audit_policy_config_mount << EOF |
| - name: auditpolicyconfigmount |
| mountPath: ${audit_policy_file} |
| readOnly: true |
| EOF |
| read -d '' audit_policy_config_volume << EOF |
| - name: auditpolicyconfigmount |
| hostPath: |
| path: ${audit_policy_file} |
| type: FileOrCreate |
| EOF |
| fi |
| sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" |
| sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" |
| fi |
| fi |
| cp "${src_file}" /etc/kubernetes/manifests |
| } |
| |
| ############################### Main Function ######################################## |
| echo "Start to configure master instance for kubemark" |
| |
| # Extract files from the server tar and setup master env variables. |
| cd "${KUBE_ROOT}" |
| if [[ ! -d "${KUBE_ROOT}/kubernetes" ]]; then |
| tar xzf kubernetes-server-linux-amd64.tar.gz |
| fi |
| source "${KUBE_ROOT}/kubemark-master-env.sh" |
| |
| # Setup IP firewall rules, required directory structure and etcd config. |
| config-ip-firewall |
| create-dirs |
| setup-kubelet-dir |
| delete-default-etcd-configs |
| compute-etcd-variables |
| |
| # Setup authentication tokens and kubeconfigs for kube-controller-manager and kube-scheduler, |
| # only if their kubeconfigs don't already exist as this script could be running on reboot. |
| if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" ]]; then |
| KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) |
| echo "${KUBE_CONTROLLER_MANAGER_TOKEN},system:kube-controller-manager,uid:system:kube-controller-manager" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" |
| create-kubecontrollermanager-kubeconfig |
| fi |
| if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" ]]; then |
| KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) |
| echo "${KUBE_SCHEDULER_TOKEN},system:kube-scheduler,uid:system:kube-scheduler" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" |
| create-kubescheduler-kubeconfig |
| fi |
| |
| # Mount master PD for etcd and create symbolic links to it. |
| { |
| main_etcd_mount_point="/mnt/disks/master-pd" |
| mount-pd "google-master-pd" "${main_etcd_mount_point}" |
| # Contains all the data stored in etcd. |
| mkdir -m 700 -p "${main_etcd_mount_point}/var/etcd" |
| ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd |
| mkdir -p /etc/srv |
| # Setup the dynamically generated apiserver auth certs and keys to pd. |
| mkdir -p "${main_etcd_mount_point}/srv/kubernetes" |
| ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes |
| # Copy the files to the PD only if they don't exist (so we do it only the first time). |
| if [[ "$(ls -A ${main_etcd_mount_point}/srv/kubernetes/)" == "" ]]; then |
| cp -r "${KUBE_ROOT}"/k8s_auth_data/* "${main_etcd_mount_point}/srv/kubernetes/" |
| fi |
| # Directory for kube-apiserver to store SSH key (if necessary). |
| mkdir -p "${main_etcd_mount_point}/srv/sshproxy" |
| ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy |
| } |
| |
| # Mount master PD for event-etcd (if required) and create symbolic links to it. |
| { |
| EVENT_STORE_IP="${EVENT_STORE_IP:-127.0.0.1}" |
| EVENT_STORE_URL="${EVENT_STORE_URL:-http://${EVENT_STORE_IP}:4002}" |
| if [ "${EVENT_PD:-}" == "true" ]; then |
| event_etcd_mount_point="/mnt/disks/master-event-pd" |
| mount-pd "google-master-event-pd" "${event_etcd_mount_point}" |
| # Contains all the data stored in event etcd. |
| mkdir -m 700 -p "${event_etcd_mount_point}/var/etcd/events" |
| ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events |
| fi |
| } |
| |
| # Setup docker flags and load images of the master components. |
| assemble-docker-flags |
| DOCKER_REGISTRY="k8s.gcr.io" |
| load-docker-images |
| |
| readonly audit_policy_file="/etc/audit_policy.config" |
| |
| # Start kubelet as a supervisord process and master components as pods. |
| start-kubelet |
| start-kubemaster-component "etcd" |
| if [ "${EVENT_STORE_IP:-}" == "127.0.0.1" ]; then |
| start-kubemaster-component "etcd-events" |
| fi |
| start-kubemaster-component "kube-apiserver" |
| start-kubemaster-component "kube-controller-manager" |
| start-kubemaster-component "kube-scheduler" |
| start-kubemaster-component "kube-addon-manager" |
| |
| # Wait till apiserver is working fine or timeout. |
| echo -n "Waiting for apiserver to be healthy" |
| start=$(date +%s) |
| until [ "$(curl 127.0.0.1:8080/healthz 2> /dev/null)" == "ok" ]; do |
| echo -n "." |
| sleep 1 |
| now=$(date +%s) |
| if [ $((now - start)) -gt 300 ]; then |
| echo "Timeout!" |
| exit 1 |
| fi |
| done |
| |
| echo "Done for the configuration for kubermark master" |