| #!/usr/bin/env bash |
| |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| #limitations under the License. |
| |
| function check_cmd() { |
| CMD=$1 |
| if ! command -v "${CMD}" &> /dev/null |
| then |
| echo "command ${CMD} could not be found" |
| exit 1 |
| fi |
| } |
| |
| function exit_on_error() { |
| CMD_CODE=$? |
| ERR_MSG=$1 |
| if [[ ${CMD_CODE} -ne 0 ]]; then |
| echo "command execution failed: ${ERR_MSG}" |
| exit ${CMD_CODE} |
| fi |
| } |
| |
| # check options that must have values |
| function check_opt() { |
| OPTION=$1 |
| VALUE=$2 |
| if [[ "${VALUE}" == "" ]]; then |
| echo "ERROR: option ${OPTION} cannot be empty" |
| echo |
| print_usage |
| exit 1 |
| fi |
| } |
| |
| # only support linux and darwin to run e2e tests |
| function check_os() { |
| if [ "${OS}" != "linux" ] && [ "${OS}" != "darwin" ]; then |
| echo "unsupported OS: ${OS}" |
| exit 1 |
| fi |
| } |
| |
| # check docker available and up |
| function check_docker() { |
| check_cmd "docker" |
| DOCKER_UP=$(docker version | grep "^Server:") |
| if [ -z "${DOCKER_UP}" ]; then |
| echo "docker daemon must be running" |
| return 1 |
| fi |
| } |
| |
| function install_kubectl() { |
| if ! command -v kubectl &> /dev/null |
| then |
| check_cmd "curl" |
| STABLE_RELEASE=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) |
| exit_on_error "unable to retrieve latest stable version of kubectl" |
| curl -LO https://storage.googleapis.com/kubernetes-release/release/"${STABLE_RELEASE}"/bin/"${OS}"/"${EXEC_ARCH}"/kubectl \ |
| && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ |
| exit_on_error "install Kubectl failed" |
| fi |
| check_cmd "kubectl" |
| } |
| |
| function install_kind() { |
| KIND="$(go env GOPATH)/bin/kind" |
| if [ ! -x "${KIND}" ]; then |
| FORCE_KIND_INSTALL=true |
| else |
| # check kind version: v0.15.0 or later required for 1.25 tests |
| KIND_MINOR=$(${KIND} version | cut -f2 -d" " | cut -f2 -d".") |
| if [ "${KIND_MINOR}" -lt 15 ]; then |
| FORCE_KIND_INSTALL=true |
| echo "kind version found is too old: force new install" |
| fi |
| fi |
| |
| if [ "${FORCE_KIND_INSTALL}" == "true" ] |
| then |
| check_cmd "curl" |
| curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.15.0/kind-${OS}-${EXEC_ARCH}" \ |
| && chmod +x ./kind && mv ./kind "$(go env GOPATH)"/bin |
| exit_on_error "install KIND failed" |
| fi |
| check_cmd "${KIND}" |
| } |
| |
| function install_helm() { |
| FORCE_HELM_INSTALL=false |
| if ! command -v helm &> /dev/null |
| then |
| FORCE_HELM_INSTALL=true |
| else |
| # get the major helm version, must be v3 |
| HELM_VERSION=$(helm version --short | cut -f1 -d".") |
| if [ "${HELM_VERSION}" != "v3" ]; then |
| FORCE_HELM_INSTALL=true |
| echo "helm version found is too old: force new install" |
| fi |
| fi |
| |
| if [ "${FORCE_HELM_INSTALL}" == "true" ] |
| then |
| check_cmd "curl" |
| curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash |
| exit_on_error "install helm-v3 failed" |
| fi |
| check_cmd "helm" |
| } |
| |
| function install_cluster() { |
| echo "step 1/9: checking required configuration" |
| if [ ! -r "${KIND_CONFIG}" ]; then |
| exit_on_error "kind config not found: ${KIND_CONFIG}" |
| fi |
| # use latest helm charts from the release repo to install yunikorn unless path is provided |
| if [ "${GIT_CLONE}" = "true" ]; then |
| check_cmd "git" |
| git clone --depth 1 https://github.com/apache/yunikorn-release.git ./yunikorn-release |
| fi |
| if [ ! -d "${CHART_PATH}" ]; then |
| exit_on_error "helm charts not found in path: ${CHART_PATH}" |
| fi |
| |
| # build docker images from latest code, so that we can install yunikorn with these latest images |
| echo "step 2/9: building docker images from latest code" |
| check_docker |
| QUIET="--quiet" REGISTRY=local VERSION=latest make image |
| exit_on_error "build docker images failed" |
| QUIET="--quiet" REGISTRY=local VERSION=latest make webtest_image |
| exit_on_error "build test web images failed" |
| |
| # install ginkgo and gomega for e2e tests. |
| echo "step 3/9: installing Ginkgo & Gomega at $(go env GOPATH)/bin" |
| go install github.com/onsi/ginkgo/ginkgo |
| go install github.com/onsi/gomega |
| check_cmd "ginkgo" |
| |
| echo "step 4/9: installing helm-v3" |
| install_helm |
| |
| # install kubectl |
| echo "step 5/9: installing kubectl" |
| install_kubectl |
| |
| # install KIND |
| echo "step 6/9: installing kind" |
| install_kind |
| |
| # create K8s cluster |
| echo "step 7/9: installing K8s cluster using kind" |
| "${KIND}" create cluster --name "${CLUSTER_NAME}" --image "${CLUSTER_VERSION}" --config="${KIND_CONFIG}" |
| exit_on_error "install K8s cluster failed" |
| kubectl cluster-info --context kind-"${CLUSTER_NAME}" |
| exit_on_error "set K8s cluster context failed" |
| kubectl create namespace yunikorn |
| exit_on_error "failed to create yunikorn namespace" |
| echo "cluster node definitions:" |
| kubectl describe nodes |
| |
| # pre-load yunikorn docker images to kind |
| echo "step 8/9: pre-load yunikorn images" |
| "${KIND}" load docker-image "local/yunikorn:${SCHEDULER_IMAGE}" --name "${CLUSTER_NAME}" |
| exit_on_error "pre-load scheduler image failed: ${SCHEDULER_IMAGE}" |
| "${KIND}" load docker-image "local/yunikorn:${ADMISSION_IMAGE}" --name "${CLUSTER_NAME}" |
| exit_on_error "pre-load admission controller image failed: ${ADMISSION_IMAGE}" |
| "${KIND}" load docker-image "local/yunikorn:${WEBTEST_IMAGE}" --name "${CLUSTER_NAME}" |
| exit_on_error "pre-load web image failed: ${WEBTEST_IMAGE}" |
| |
| echo "step 9/9: installing yunikorn" |
| helm install yunikorn "${CHART_PATH}" --namespace yunikorn \ |
| --set image.repository=local/yunikorn \ |
| --set image.tag="${SCHEDULER_IMAGE}" \ |
| --set image.pullPolicy=IfNotPresent \ |
| --set admissionController.image.repository=local/yunikorn \ |
| --set admissionController.image.tag="${ADMISSION_IMAGE}" \ |
| --set admissionController.image.pullPolicy=IfNotPresent \ |
| --set web.image.repository=local/yunikorn \ |
| --set web.image.tag="${WEBTEST_IMAGE}" \ |
| --set web.image.pullPolicy=IfNotPresent |
| exit_on_error "failed to install yunikorn" |
| kubectl wait --for=condition=available --timeout=300s deployment/yunikorn-scheduler -n yunikorn |
| exit_on_error "failed to wait for yunikorn scheduler deployment being deployed" |
| kubectl wait --for=condition=ready --timeout=300s pod -l app=yunikorn -n yunikorn |
| exit_on_error "failed to wait for yunikorn scheduler pods being deployed" |
| } |
| |
| function delete_cluster() { |
| echo "deleting K8s cluster: ${CLUSTER_NAME}" |
| install_kind |
| "${KIND}" delete cluster --name "${CLUSTER_NAME}" |
| exit_on_error "failed to delete the cluster" |
| } |
| |
| function print_usage() { |
| NAME=$(basename "$0") |
| cat <<EOF |
| Usage: ${NAME} -a <action> -n <kind-cluster-name> -v <kind-node-image-version> [-p <chart-path>] [--plugin] [--force-kind-install] |
| <action> the action to be executed, must be either "test" or "cleanup". |
| <kind-cluster-name> the name of the K8s cluster to be created by kind |
| <kind-node-image-version> the kind node image used to provision the K8s cluster, required for "test" action |
| <chart-path> local path to helm charts path (default is to pull from GitHub master) |
| --force-kind-install force Kind to be installed even if already present |
| --plugin use scheduler plugin image instead of default mode image |
| |
| Examples: |
| ${NAME} -a test -n yk8s -v kindest/node:v1.20.15 |
| ${NAME} -a test -n yk8s -v kindest/node:v1.21.14 |
| ${NAME} -a test -n yk8s -v kindest/node:v1.22.13 |
| ${NAME} -a test -n yk8s -v kindest/node:v1.23.10 |
| ${NAME} -a test -n yk8s -v kindest/node:v1.24.4 |
| ${NAME} -a test -n yk8s -v kindest/node:v1.25.0 |
| |
| Use a local helm chart path: |
| ${NAME} -a test -n yk8s -v kindest/node:v1.25.0 -p ./yunikorn-release/helm-charts/yunikorn |
| EOF |
| } |
| |
| # setup architectures and OS type |
| check_cmd "make" |
| eval "$(make arch)" |
| OS=$(uname -s | tr '[:upper:]' '[:lower:]') |
| check_os |
| |
| KIND_CONFIG=./scripts/kind.yaml |
| CHART_PATH="./yunikorn-release/helm-charts/yunikorn" |
| GIT_CLONE=true |
| SCHEDULER_IMAGE="scheduler-${DOCKER_ARCH}-latest" |
| ADMISSION_IMAGE="admission-${DOCKER_ARCH}-latest" |
| WEBTEST_IMAGE="webtest-${DOCKER_ARCH}-latest" |
| FORCE_KIND_INSTALL=false |
| |
| while [[ $# -gt 0 ]]; do |
| key="$1" |
| case ${key} in |
| -a|--action) |
| ACTION="$2" |
| shift |
| shift |
| ;; |
| -n|--cluster-name) |
| CLUSTER_NAME="$2" |
| shift |
| shift |
| ;; |
| -v|--cluster-version) |
| CLUSTER_VERSION="$2" |
| shift |
| shift |
| ;; |
| -p|--charts-path) |
| CHART_PATH="$2" |
| GIT_CLONE=false |
| shift |
| shift |
| ;; |
| --force-kind-install) |
| FORCE_KIND_INSTALL=true |
| shift |
| ;; |
| --plugin) |
| SCHEDULER_IMAGE="scheduler-plugin-${DOCKER_ARCH}-latest" |
| shift |
| ;; |
| -h|--help) |
| print_usage |
| exit 0 |
| ;; |
| *) |
| echo "unknown option: ${key}" |
| print_usage |
| exit 1 |
| ;; |
| esac |
| done |
| |
| echo "e2e test run details" |
| echo " action : ${ACTION}" |
| echo " force kind install : ${FORCE_KIND_INSTALL}" |
| echo " kind cluster name : ${CLUSTER_NAME}" |
| echo " kind node version : ${CLUSTER_VERSION}" |
| echo " kind config : ${KIND_CONFIG}" |
| echo " git clone release : ${GIT_CLONE}" |
| echo " chart path : ${CHART_PATH}" |
| echo " operating system : ${OS}" |
| echo " processor arch : ${EXEC_ARCH}" |
| echo " docker arch : ${DOCKER_ARCH}" |
| echo " scheduler image : ${SCHEDULER_IMAGE}" |
| echo " admission image : ${ADMISSION_IMAGE}" |
| echo " web image : ${WEBTEST_IMAGE}" |
| check_opt "action" "${ACTION}" |
| check_opt "kind-cluster-name" "${CLUSTER_NAME}" |
| |
| # this script only supports 2 actions |
| # 1) test |
| # - install a K8s cluster with kind |
| # - install latest yunikorn |
| # - run e2e tests |
| # 2) cleanup |
| # - delete k8s cluster |
| if [ "${ACTION}" == "test" ]; then |
| # make will fail without go installed but we call it before that... |
| check_cmd "go" |
| check_opt "kind-node-image-version" "${CLUSTER_VERSION}" |
| check_opt "chart-path" "${CHART_PATH}" |
| install_cluster |
| echo "starting e2e tests" |
| # Noticed regular unexplained failures in the tests when run directly after |
| # the install. Running the test, via make, on the installed kind cluster |
| # following the failed run passes. A short sleep seems to settle things down |
| # and prevent the unexplained failures. |
| if [ "${OS}" == "darwin" ]; then |
| sleep 5 |
| fi |
| make e2e_test |
| exit_on_error "e2e tests failed" |
| elif [ "${ACTION}" == "cleanup" ]; then |
| echo "cleaning up the environment" |
| delete_cluster |
| else |
| echo "unknown action: ${ACTION}" |
| print_usage |
| exit 1 |
| fi |