| #!/usr/bin/env bash |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # exit immediately when a command fails |
| set -e |
| # only exit with zero if all commands of the pipeline exit successfully |
| set -o pipefail |
| # error on unset variables |
| set -u |
| |
| show_help() { |
| cat << EOF |
| Usage: ./hack/release/smoke_test/test_cluster.sh [-h] [-i IMAGE] [-k KUBERNETES_VERSION] [-t SOLR_IMAGE] [-g GPG_KEY] -v VERSION -l LOCATION |
| |
| Test the release candidate in a Kind cluster |
| |
| -h Display this help and exit |
| -v Version of the Solr Operator |
| -l Base location of the staged artifacts. Can be a URL or relative or absolute file path. |
| -i Solr Operator docker image to use (Optional, defaults to apache/solr-operator:<version>) |
| -g GPG Key (fingerprint) used to sign the artifacts (Optional, if not provided then the helm chart will not be verified) |
| -k Kubernetes Version to test with (full tag, e.g. v1.26.6) (Optional, defaults to a compatible version) |
| -t Full solr image, or image tag (for the official Solr image), to test with (e.g. apache/solr-nightly:9.0.0, 8.11). (Optional, defaults to a compatible version) |
| EOF |
| } |
| |
| OPTIND=1 |
| while getopts hv:i:l:g:k:t: opt; do |
| case $opt in |
| h) |
| show_help |
| exit 0 |
| ;; |
| v) VERSION=$OPTARG |
| ;; |
| i) IMAGE=$OPTARG |
| ;; |
| l) LOCATION=$OPTARG |
| ;; |
| g) GPG_KEY=$OPTARG |
| ;; |
| k) KUBERNETES_VERSION=$OPTARG |
| ;; |
| t) SOLR_IMAGE=$OPTARG |
| ;; |
| *) |
| show_help >&2 |
| exit 1 |
| ;; |
| esac |
| done |
| shift "$((OPTIND-1))" # Discard the options and sentinel -- |
| |
| if [[ -z "${VERSION:-}" ]]; then |
| echo "Specify a project version through -v, or through the VERSION env var" >&2 && exit 1 |
| fi |
| if [[ -z "${IMAGE:-}" ]]; then |
| IMAGE="apache/solr-operator:${VERSION}" |
| fi |
| if [[ -z "${LOCATION:-}" ]]; then |
| echo "Specify an base artifact location -l, or through the LOCATION env var" >&2 && exit 1 |
| fi |
| if [[ -z "${KUBERNETES_VERSION:-}" ]]; then |
| KUBERNETES_VERSION="v1.26.6" |
| fi |
| if [[ -z "${SOLR_IMAGE:-}" ]]; then |
| SOLR_IMAGE="${SOLR_VERSION:-8.11}" |
| fi |
| if [[ "${SOLR_IMAGE}" != *":"* ]]; then |
| SOLR_IMAGE="solr:${SOLR_IMAGE}" |
| fi |
| |
| export LOCATION="$LOCATION" |
| export VERSION="$VERSION" |
| |
| function add_solr_helm_repo() { |
| if (echo "${LOCATION}" | grep "http"); then |
| helm repo add --force-update "apache-solr-test-${VERSION}" "${LOCATION}/helm-charts" |
| fi |
| } |
| |
| function remove_solr_helm_repo() { |
| if (echo "${LOCATION}" | grep "http"); then |
| helm repo remove "apache-solr-test-${VERSION}" |
| fi |
| } |
| |
| # If LOCATION is not a URL, then get the absolute path |
| if ! (echo "${LOCATION}" | grep "http"); then |
| LOCATION=$(cd "${LOCATION}"; pwd) |
| LOCATION=${LOCATION%%/} |
| |
| OP_HELM_CHART="${LOCATION}/helm-charts/solr-operator-${VERSION#v}.tgz" |
| SOLR_HELM_CHART="${LOCATION}/helm-charts/solr-${VERSION#v}.tgz" |
| else |
| # If LOCATION is a URL, then we want to make sure we have the up-to-date docker image. |
| docker pull "${IMAGE}" |
| |
| OP_HELM_CHART="apache-solr-test-${VERSION}/solr-operator" |
| SOLR_HELM_CHART="apache-solr-test-${VERSION}/solr" |
| fi |
| |
| if ! (which kind); then |
| echo "Install Kind (Kubernetes in Docker)" |
| GO111MODULE="on" go install sigs.k8s.io/kind@v0.20.0 |
| fi |
| |
| CLUSTER_NAME="solr-operator-${VERSION}-rc" |
| KUBE_CONTEXT="kind-${CLUSTER_NAME}" |
| |
| if (kind get clusters | grep "${CLUSTER_NAME}"); then |
| printf "Delete cluster, so the test starts with a clean slate.\n\n" |
| kind delete clusters "${CLUSTER_NAME}" |
| fi |
| |
| echo "Create test Kubernetes ${KUBERNETES_VERSION} cluster in Kind. This will allow us to test the CRDs, Helm chart and the Docker image." |
| kind create cluster --name "${CLUSTER_NAME}" --image "kindest/node:${KUBERNETES_VERSION}" |
| |
| # Load the docker images into the cluster |
| if (docker image inspect "${IMAGE}" &>/dev/null); then |
| kind load docker-image --name "${CLUSTER_NAME}" "${IMAGE}" |
| printf "\nUsing local version of Solr Operator image \"${IMAGE}\".\nIf you want to use an updated version of this image, run \"docker pull ${SOLR_IMAGE}\" before running the smoke test again.\n\n" |
| else |
| printf "\nUsing the remote Solr Operator image \"${IMAGE}\", since it was not found in the local Docker image list.\n\n" |
| fi |
| if (docker image inspect "${SOLR_IMAGE}" &>/dev/null); then |
| kind load docker-image --name "${CLUSTER_NAME}" "${SOLR_IMAGE}" |
| printf "\nUsing local version of Solr image \"${SOLR_IMAGE}\".\nIf you want to use an updated version of this image, run \"docker pull ${SOLR_IMAGE}\" before running the smoke test again.\n\n" |
| else |
| printf "\nUsing the remote Solr image \"${SOLR_IMAGE}\", since it was not found in the local Docker image list.\n\n" |
| fi |
| |
| # Add a temporary directory for backups |
| docker exec "${CLUSTER_NAME}-control-plane" bash -c "mkdir -p /tmp/backup" |
| |
| echo "Import Solr Keys" |
| curl -sL0 "https://dist.apache.org/repos/dist/release/solr/KEYS" | gpg --import --quiet |
| |
| # First generate the old-style public key ring, if it doesn't already exist and contain the information we want. |
| # Only do this if a GPG Key was provided |
| VERIFY_OR_NOT="" |
| if [[ -n "${GPG_KEY:-}" ]]; then |
| VERIFY_OR_NOT="--verify" |
| if ! (gpg --no-default-keyring --keyring=~/.gnupg/pubring.gpg --list-keys "${GPG_KEY}"); then |
| gpg --export >~/.gnupg/pubring.gpg |
| fi |
| fi |
| |
| add_solr_helm_repo |
| |
| # Install the Solr Operator |
| kubectl create -f "${LOCATION}/crds/all-with-dependencies.yaml" || kubectl replace -f "${LOCATION}/crds/all-with-dependencies.yaml" |
| helm install --kube-context "${KUBE_CONTEXT}" ${VERIFY_OR_NOT} solr-operator "${OP_HELM_CHART}" \ |
| --set-string image.tag="${IMAGE##*:}" \ |
| --set image.repository="${IMAGE%%:*}" \ |
| --set image.pullPolicy="Never" |
| |
| printf "\nInstall a test Solr Cluster\n" |
| helm install --kube-context "${KUBE_CONTEXT}" ${VERIFY_OR_NOT} example "${SOLR_HELM_CHART}" \ |
| --set replicas=2 \ |
| --set image.repository="${SOLR_IMAGE%%:*}" \ |
| --set-string image.tag="${SOLR_IMAGE##*:}" \ |
| --set solrOptions.javaMemory="-Xms700m -Xmx700m" \ |
| --set podOptions.resources.limits.cpu="500m" \ |
| --set podOptions.resources.limits.memory="1Gi" \ |
| --set podOptions.resources.requests.cpu="500m" \ |
| --set podOptions.resources.requests.memory="1Gi" \ |
| --set zk.provided.persistence.spec.resources.requests.storage="5Gi" \ |
| --set zk.provided.replicas=1 \ |
| --set "backupRepositories[0].name=local" \ |
| --set "backupRepositories[0].volume.source.hostPath.path=/tmp/backup" |
| |
| # If LOCATION is a URL, then remove the helm repo after use |
| remove_solr_helm_repo |
| |
| # Wait for solrcloud to be ready |
| printf '\nWait for all 2 Solr nodes to become ready.\n\n' |
| grep -q "2 2 2 2" <(exec kubectl get solrcloud example -w); kill $! |
| |
| # Expose the common Solr service to localhost |
| kubectl port-forward service/example-solrcloud-common 18983:80 || true & |
| sleep 2 |
| |
| printf "\nCheck the admin URL to make sure it works\n" |
| curl --silent "http://localhost:18983/solr/admin/info/health" | grep '"status":0' > /dev/null |
| |
| printf "\nCreating a test collection\n" |
| curl --silent "http://localhost:18983/solr/admin/collections?action=CREATE&name=smoke-test&replicationFactor=1&numShards=2" | grep '"status":0' > /dev/null |
| |
| printf "\nQuery the test collection, test for 0 docs\n" |
| curl --silent "http://localhost:18983/solr/smoke-test/select" | grep '\"numFound\":0' > /dev/null |
| |
| printf "\nCreate a Solr Backup to take local backups of the test collection\n" |
| cat <<EOF | kubectl apply -f - |
| apiVersion: solr.apache.org/v1beta1 |
| kind: SolrBackup |
| metadata: |
| name: ex-back |
| spec: |
| solrCloud: example |
| collections: |
| - smoke-test |
| location: test-dir/ |
| repositoryName: local |
| recurrence: |
| schedule: "@every 10s" |
| maxSaved: 3 |
| EOF |
| |
| printf "\nCreate a Solr Prometheus Exporter to expose metrics for the Solr Cloud\n" |
| cat <<EOF | kubectl apply -f - |
| apiVersion: solr.apache.org/v1beta1 |
| kind: SolrPrometheusExporter |
| metadata: |
| name: example |
| spec: |
| solrReference: |
| cloud: |
| name: "example" |
| numThreads: 4 |
| EOF |
| |
| printf "\nWait for the Solr Prometheus Exporter to be ready\n" |
| sleep 5 |
| kubectl rollout status deployment/example-solr-metrics |
| |
| # Expose the Solr Prometheus Exporter service to localhost |
| kubectl port-forward service/example-solr-metrics 18984:80 || true & |
| sleep 15 |
| |
| printf "\nQuery the prometheus exporter, test for 'http://example-solrcloud-*.example-solrcloud-headless.default:8983/solr' (internal) URL being scraped.\n" |
| curl --silent "http://localhost:18984/metrics" | grep 'http://example-solrcloud-.*.example-solrcloud-headless.default:8983/solr' > /dev/null |
| |
| printf "\nWait 22 seconds, so that more backups can be taken.\n" |
| sleep 22 |
| |
| printf "\nList the backups, and make sure that >= 3 have been taken (should be four), but only 3 are saved.\n" |
| BACKUP_RESP=$(curl --silent -L "http://localhost:18983/solr/admin/collections?action=LISTBACKUP&name=ex-back-smoke-test&repository=local&collection=smoke-test&location=/var/solr/data/backup-restore/local/test-dir") |
| SAVED_BACKUPS=$(echo "${BACKUP_RESP}" | jq --raw-output '.backups | length') |
| if [[ "${SAVED_BACKUPS}" != "3" ]]; then |
| echo "Wrong number of saved backups, should be 3, found ${SAVED_BACKUPS}" >&2 |
| exit 1 |
| fi |
| LAST_BACKUP_ID=$(echo "${BACKUP_RESP}" | jq --raw-output '.backups[-1].backupId') |
| if (( "${LAST_BACKUP_ID}" < 4 )); then |
| echo "The last backup id must be > 3, since we should have taken at least 4 backups. Last backup id found: ${LAST_BACKUP_ID}" >&2 |
| exit 1 |
| fi |
| |
| printf "\nStop recurring backup\n" |
| cat <<EOF | kubectl apply -f - |
| apiVersion: solr.apache.org/v1beta1 |
| kind: SolrBackup |
| metadata: |
| name: ex-back |
| spec: |
| solrCloud: example |
| collections: |
| - smoke-test |
| location: test-dir/ |
| repositoryName: local |
| recurrence: |
| schedule: "@every 10s" |
| maxSaved: 3 |
| disabled: true |
| EOF |
| sleep 5 |
| LAST_BACKUP_ID=$(curl --silent -L "http://localhost:18983/solr/admin/collections?action=LISTBACKUP&name=ex-back-smoke-test&repository=local&collection=smoke-test&location=/var/solr/data/backup-restore/local/test-dir" | jq --raw-output '.backups[-1].backupId') |
| |
| printf "\nWait to make sure more backups are not taken\n" |
| sleep 15 |
| FOUND_BACKUP_ID=$(curl --silent -L "http://localhost:18983/solr/admin/collections?action=LISTBACKUP&name=ex-back-smoke-test&repository=local&collection=smoke-test&location=/var/solr/data/backup-restore/local/test-dir" | jq --raw-output '.backups[-1].backupId') |
| if (( "${FOUND_BACKUP_ID}" != "${LAST_BACKUP_ID}" )); then |
| echo "The another backup has been taken since recurrence was stopped. Last backupId should be '${LAST_BACKUP_ID}', but instead found '${FOUND_BACKUP_ID}'." >&2 |
| exit 1 |
| fi |
| |
| |
| printf "\nDo a rolling restart and make sure the cluster is healthy afterwards\n" |
| add_solr_helm_repo |
| helm upgrade --kube-context "${KUBE_CONTEXT}" ${VERIFY_OR_NOT} example "${SOLR_HELM_CHART}" --reuse-values \ |
| --set-string podOptions.annotations.restart="true" |
| printf '\nWait for the rolling restart to begin.\n\n' |
| grep -q "2 [[:digit:]] [[:digit:]] 0" <(exec kubectl get solrcloud example -w); kill $! |
| remove_solr_helm_repo |
| |
| printf '\nWait 5 minutes for all 2 Solr nodes to become ready.\n\n' |
| grep -q "2 2 2 2" <(exec kubectl get solrcloud example -w --request-timeout 300); kill $! |
| |
| # Need a new port-forward, since the last one will have broken due to all pods restarting |
| kubectl port-forward service/example-solrcloud-common 28983:80 || true & |
| sleep 2 |
| |
| printf "\nQuery the test collection, test for 0 docs\n\n" |
| curl --silent "http://localhost:28983/solr/smoke-test/select" | grep '\"numFound\":0' > /dev/null |
| |
| echo "Delete test Kind Kubernetes cluster." |
| kind delete clusters "${CLUSTER_NAME}" |
| |
| printf "\n********************\nLocal end-to-end cluster test successfully run!\n\n" |