it.sh - druid - Git at Google

 #! /bin/bash

 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #--------------------------------------------------------------------

 # Utility script for running the new integration tests, since the Maven
 # commands are unwieldy. Allows straightforward usage of ITs on the desktop
 # and in various build scripts. Handles configuration of various kinds.

 set -e

 # Enable for debugging
 #set -x

 export DRUID_DEV=$(cd $(dirname $0) && pwd)

 function usage
 {
   cat <<EOF
 Usage: $0 cmd [category]
   ci
       build Druid and the distribution for CI pipelines
   build
       Build Druid and the distribution
   dist
       Build the Druid distribution (only)
   tools
       Build druid-it-tools
   image
       Build the test image
   up <category>
       Start the cluster for category
   down <category>
       Stop the cluster for category
   test <category>
       Start the cluster, run the test for category, and stop the cluster
   tail <category>
       Show the last 20 lines of each container log
   gen
       Generate docker-compose.yaml files (done automatically on up)
       run one IT in Travis (build dist, image, run test, tail logs)
   github <category>
       Run one IT in Github Workflows (run test, tail logs)
   prune
       prune Docker volumes

 Environment:
   OVERRIDE_ENV: optional, name of env file to pass to Docker
   USE_INDEXER: Set to middleManager (default if not set)
       or "indexer". If "indexer", requires docker-compose-indexer.yaml exist.
   druid_*: passed to the container.
   Other, test-specific variables.

 See docs for additional details.
 EOF
 }

 function tail_logs
 {
   category=$1
   cd integration-tests-ex/cases/target/$category/logs
   ls *.log | while read log;
   do
     echo "----- $category/$log -----"
     tail -20 $log
   done
 }

 # Many tests require us to pass information into containers using environment variables.
 # The Docker environment is distinct from the environment running this script. We bridge
 # the two by passing into Docker compose a file that contains all env vars we want to
 # "export" from our local environment into the container environment.
 # There are three ways to provide these options:
 #
 # 1. Directly in the environment. (Simplest and best.) We support a fixed set of variables:
 #    <need the list>
 # 2. For ad-hoc use, as var=value pairs in a file with the same name as the
 #    test catagory, in the home folder under ~/druid-it. Example:
 #    BatchIndex.env. Use this to hold credentials and other info which you must
 #    pass into tests when running locally.
 # 3. A file given by the OVERRIDE_ENV environment variable. That is, OVERRIDE_ENV holds
 #    the path to a file of var=value pairs. Historically, this file was created by a
 #    build environment such as Github Actions. However, it is actually simpler just to use
 #    option 1: just set the values in the environment and let Linux pass them through to
 #    this script.
 # 4. Environment variables of the form "druid_" used to create the Druid config file.
 #
 # All of the above are combined into a temporary environment file which is then passed
 # into Docker compose.
 function build_override {

 	mkdir -p target
 	OVERRIDE_FILE="$(pwd)/target/override.env"
 	rm -f "$OVERRIDE_FILE"
 	touch "$OVERRIDE_FILE"

 	# Provided override file
 	if [ -n "$OVERRIDE_ENV" ]; then
 		if [ ! -f "$OVERRIDE_ENV" ]; then
 			echo "Environment override file OVERRIDE_ENV not found: $OVERRIDE_ENV" 1>&2
 			exit 1
 		fi
 		cat "$OVERRIDE_ENV" >> "$OVERRIDE_FILE"
 	fi

 	# User-local settings?
 	LOCAL_ENV="$HOME/druid-it/${CATEGORY}.env"
 	if [ -f "$LOCAL_ENV" ]; then
 		cat "$LOCAL_ENV" >> "$OVERRIDE_FILE"
 	fi

     # Add all environment variables of the form druid_*
     set +e # Grep gives exit status 1 if no lines match. Let's not fail.
     env | grep "^druid_" >> "$OVERRIDE_FILE"
     set -e

     # TODO: Add individual env vars that we want to pass from the local
     # environment into the container.

     # Reuse the OVERRIDE_ENV variable to pass the full list to Docker compose
     export OVERRIDE_ENV="$OVERRIDE_FILE"
 }

 function prepare_category {
 	if [ $# -eq 0 ]; then
 		usage 1>&2
 		exit 1
 	fi
 	export CATEGORY=$1
 }

 function prepare_docker {
     cd $DRUID_DEV/integration-tests-ex/cases
     build_override
 }

 function require_env_var {
 	if [ -z "$1" ]; then
 	  echo "$1 must be set for test category $CATEGORY" 1>&2
 	  exit 1
     fi
 }

 # Verfiy any test-specific environment variables that must be set in this local
 # environment (and generally passed into the Docker container via docker-compose.yaml).
 #
 # Add entries here as you add env var references in docker-compose.yaml. Doing so
 # ensures we get useful error messages when we forget to set something, rather than
 # some cryptic use-specific error.
 function verify_env_vars {
 	case $CATEGORY in
 		"AzureDeepStorage")
 			require_env_var AZURE_ACCOUNT
 			require_env_var AZURE_KEY
 			require_env_var AZURE_CONTAINER
 			;;
 		"GcsDeepStorage")
 			require_env_var GOOGLE_BUCKET
 			require_env_var GOOGLE_PREFIX
 			require_env_var GOOGLE_APPLICATION_CREDENTIALS
 			if [ ! -f "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
 				echo "Required file GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS is missing" 1>&2
 				exit 1
 		    fi
 			;;
 		"S3DeepStorage")
 			require_env_var DRUID_CLOUD_BUCKET
 			require_env_var DRUID_CLOUD_PATH
 			require_env_var AWS_REGION
 			require_env_var AWS_ACCESS_KEY_ID
 			require_env_var AWS_SECRET_ACCESS_KEY
 			;;
 	esac
 }

 if [ $# = 0 ]; then
   usage
   exit 1
 fi

 CMD=$1
 shift
 MAVEN_IGNORE="-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true"

 case $CMD in
   "help" )
     usage
     ;;
   "ci" )
     mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE
     ;;
   "build" )
     mvn clean package -P dist $MAVEN_IGNORE -T1.0C $*
     ;;
   "dist" )
     mvn package -P dist $MAVEN_IGNORE -pl :distribution
     ;;
   "tools" )
     mvn install -pl :druid-it-tools
     ;;
   "image" )
     cd $DRUID_DEV/integration-tests-ex/image
     mvn install -P test-image $MAVEN_IGNORE
     ;;
   "gen")
     # Generate the docker-compose.yaml files. Mostly for debugging
     # since the up command does generation implicitly.
     prepare_category $1
     prepare_docker
     ./cluster.sh gen $CATEGORY
     ;;
   "up" )
     prepare_category $1
     prepare_docker
     verify_env_vars
     ./cluster.sh up $CATEGORY
     ;;
   "down" )
     prepare_category $1
     prepare_docker
     ./cluster.sh down $CATEGORY
     ;;
   "test" )
     prepare_category $1
     prepare_docker
     mvn verify -P skip-static-checks,docker-tests,IT-$CATEGORY \
             -Dmaven.javadoc.skip=true -DskipUTs=true \
             -pl :druid-it-cases
     ;;
   "tail" )
     prepare_category $1
     tail_logs $CATEGORY
     ;;
   "github" )
     prepare_category $1
     $0 test $CATEGORY
     $0 tail $CATEGORY
     ;;
   "prune" )
     # Caution: this removes all volumes, which is generally what you
     # want when testing.
     docker system prune --volumes
     ;;
   * )
     usage
     exit -1
     ;;
 esac
	#! /bin/bash

	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#--------------------------------------------------------------------

	# Utility script for running the new integration tests, since the Maven
	# commands are unwieldy. Allows straightforward usage of ITs on the desktop
	# and in various build scripts. Handles configuration of various kinds.

	set -e

	# Enable for debugging
	#set -x

	export DRUID_DEV=$(cd $(dirname $0) && pwd)

	function usage
	{
	cat <<EOF
	Usage: $0 cmd [category]
	ci
	build Druid and the distribution for CI pipelines
	build
	Build Druid and the distribution
	dist
	Build the Druid distribution (only)
	tools
	Build druid-it-tools
	image
	Build the test image
	up <category>
	Start the cluster for category
	down <category>
	Stop the cluster for category
	test <category>
	Start the cluster, run the test for category, and stop the cluster
	tail <category>
	Show the last 20 lines of each container log
	gen
	Generate docker-compose.yaml files (done automatically on up)
	run one IT in Travis (build dist, image, run test, tail logs)
	github <category>
	Run one IT in Github Workflows (run test, tail logs)
	prune
	prune Docker volumes

	Environment:
	OVERRIDE_ENV: optional, name of env file to pass to Docker
	USE_INDEXER: Set to middleManager (default if not set)
	or "indexer". If "indexer", requires docker-compose-indexer.yaml exist.
	druid_*: passed to the container.
	Other, test-specific variables.

	See docs for additional details.
	EOF
	}

	function tail_logs
	{
	category=$1
	cd integration-tests-ex/cases/target/$category/logs
	ls *.log \| while read log;
	do
	echo "----- $category/$log -----"
	tail -20 $log
	done
	}

	# Many tests require us to pass information into containers using environment variables.
	# The Docker environment is distinct from the environment running this script. We bridge
	# the two by passing into Docker compose a file that contains all env vars we want to
	# "export" from our local environment into the container environment.
	# There are three ways to provide these options:
	#
	# 1. Directly in the environment. (Simplest and best.) We support a fixed set of variables:
	# <need the list>
	# 2. For ad-hoc use, as var=value pairs in a file with the same name as the
	# test catagory, in the home folder under ~/druid-it. Example:
	# BatchIndex.env. Use this to hold credentials and other info which you must
	# pass into tests when running locally.
	# 3. A file given by the OVERRIDE_ENV environment variable. That is, OVERRIDE_ENV holds
	# the path to a file of var=value pairs. Historically, this file was created by a
	# build environment such as Github Actions. However, it is actually simpler just to use
	# option 1: just set the values in the environment and let Linux pass them through to
	# this script.
	# 4. Environment variables of the form "druid_" used to create the Druid config file.
	#
	# All of the above are combined into a temporary environment file which is then passed
	# into Docker compose.
	function build_override {

	mkdir -p target
	OVERRIDE_FILE="$(pwd)/target/override.env"
	rm -f "$OVERRIDE_FILE"
	touch "$OVERRIDE_FILE"

	# Provided override file
	if [ -n "$OVERRIDE_ENV" ]; then
	if [ ! -f "$OVERRIDE_ENV" ]; then
	echo "Environment override file OVERRIDE_ENV not found: $OVERRIDE_ENV" 1>&2
	exit 1
	fi
	cat "$OVERRIDE_ENV" >> "$OVERRIDE_FILE"
	fi

	# User-local settings?
	LOCAL_ENV="$HOME/druid-it/${CATEGORY}.env"
	if [ -f "$LOCAL_ENV" ]; then
	cat "$LOCAL_ENV" >> "$OVERRIDE_FILE"
	fi

	# Add all environment variables of the form druid_*
	set +e # Grep gives exit status 1 if no lines match. Let's not fail.
	env \| grep "^druid_" >> "$OVERRIDE_FILE"
	set -e

	# TODO: Add individual env vars that we want to pass from the local
	# environment into the container.

	# Reuse the OVERRIDE_ENV variable to pass the full list to Docker compose
	export OVERRIDE_ENV="$OVERRIDE_FILE"
	}

	function prepare_category {
	if [ $# -eq 0 ]; then
	usage 1>&2
	exit 1
	fi
	export CATEGORY=$1
	}

	function prepare_docker {
	cd $DRUID_DEV/integration-tests-ex/cases
	build_override
	}

	function require_env_var {
	if [ -z "$1" ]; then
	echo "$1 must be set for test category $CATEGORY" 1>&2
	exit 1
	fi
	}

	# Verfiy any test-specific environment variables that must be set in this local
	# environment (and generally passed into the Docker container via docker-compose.yaml).
	#
	# Add entries here as you add env var references in docker-compose.yaml. Doing so
	# ensures we get useful error messages when we forget to set something, rather than
	# some cryptic use-specific error.
	function verify_env_vars {
	case $CATEGORY in
	"AzureDeepStorage")
	require_env_var AZURE_ACCOUNT
	require_env_var AZURE_KEY
	require_env_var AZURE_CONTAINER
	;;
	"GcsDeepStorage")
	require_env_var GOOGLE_BUCKET
	require_env_var GOOGLE_PREFIX
	require_env_var GOOGLE_APPLICATION_CREDENTIALS
	if [ ! -f "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
	echo "Required file GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS is missing" 1>&2
	exit 1
	fi
	;;
	"S3DeepStorage")
	require_env_var DRUID_CLOUD_BUCKET
	require_env_var DRUID_CLOUD_PATH
	require_env_var AWS_REGION
	require_env_var AWS_ACCESS_KEY_ID
	require_env_var AWS_SECRET_ACCESS_KEY
	;;
	esac
	}

	if [ $# = 0 ]; then
	usage
	exit 1
	fi

	CMD=$1
	shift
	MAVEN_IGNORE="-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true"

	case $CMD in
	"help" )
	usage
	;;
	"ci" )
	mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE
	;;
	"build" )
	mvn clean package -P dist $MAVEN_IGNORE -T1.0C $*
	;;
	"dist" )
	mvn package -P dist $MAVEN_IGNORE -pl :distribution
	;;
	"tools" )
	mvn install -pl :druid-it-tools
	;;
	"image" )
	cd $DRUID_DEV/integration-tests-ex/image
	mvn install -P test-image $MAVEN_IGNORE
	;;
	"gen")
	# Generate the docker-compose.yaml files. Mostly for debugging
	# since the up command does generation implicitly.
	prepare_category $1
	prepare_docker
	./cluster.sh gen $CATEGORY
	;;
	"up" )
	prepare_category $1
	prepare_docker
	verify_env_vars
	./cluster.sh up $CATEGORY
	;;
	"down" )
	prepare_category $1
	prepare_docker
	./cluster.sh down $CATEGORY
	;;
	"test" )
	prepare_category $1
	prepare_docker
	mvn verify -P skip-static-checks,docker-tests,IT-$CATEGORY \
	-Dmaven.javadoc.skip=true -DskipUTs=true \
	-pl :druid-it-cases
	;;
	"tail" )
	prepare_category $1
	tail_logs $CATEGORY
	;;
	"github" )
	prepare_category $1
	$0 test $CATEGORY
	$0 tail $CATEGORY
	;;
	"prune" )
	# Caution: this removes all volumes, which is generally what you
	# want when testing.
	docker system prune --volumes
	;;
	* )
	usage
	exit -1
	;;
	esac