bin/jenkins/dockerized-impala-run-tests.sh

#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


# This script runs Impala's tests with a dockerised minicluster.
# It has been tested on Ubuntu 16.04.
set -x
set -eu -o pipefail

DOCKER_NETWORK="test-impala-cluster"

# Helper to source impala-config.sh, which may have unbound variables
source_impala_config() {
  set +u
  . ./bin/impala-config.sh
  set -u
}

onexit() {
  # Clean up docker containers and networks that may have been created by
  # these tests.
  docker rm -f $(docker ps -a -q) || true
  docker network rm $DOCKER_NETWORK || true
}
trap onexit EXIT

source_impala_config

# Check that docker is running and that our user can interact with it.
docker run hello-world

# Set up the test network.
./docker/configure_test_network.sh $DOCKER_NETWORK

# Pick up the new variables.
source_impala_config

# Dump diagnostics for networks and check connectivity.
ifconfig
ping -c 1 $INTERNAL_LISTEN_HOST

# Check that ssh to localhost via Docker gateway works.
if ! ssh -n $INTERNAL_LISTEN_HOST "echo 'SSH success!'"; then
  echo "Failed to ssh, will try to add docker network gateway to known hosts"
  ssh-keyscan $INTERNAL_LISTEN_HOST >> ~/.ssh/known_hosts
  ssh -n $INTERNAL_LISTEN_HOST "echo 'SSH success!'"
fi

# Build Impala and load data with the non-dockerised path.
# TODO: this is a bit awkward. It would be better to have a single invocation
# to run everything with docker.
# Skip building backend tests, which aren't used.
time -p ./buildall.sh -format -testdata -notests < /dev/null
# Kill non-dockerised cluster.
start-impala-cluster.py --kill

# Build the docker images required to start the cluster.
# parquet-reader is needed for e2e tests but not built for non-test build.
make -j ${IMPALA_BUILD_THREADS} docker_debug_images parquet-reader

source_impala_config

FAIR_SCHED_CONF=/opt/impala/conf/minicluster-fair-scheduler.xml
LLAMA_CONF=/opt/impala/conf/minicluster-llama-site.xml
export TEST_START_CLUSTER_ARGS="--docker_network=${DOCKER_NETWORK}"
TEST_START_CLUSTER_ARGS+=" --data_cache_dir=/tmp --data_cache_size=500m"
TEST_START_CLUSTER_ARGS+=" --impalad_args=-fair_scheduler_allocation_path=${FAIR_SCHED_CONF}"
TEST_START_CLUSTER_ARGS+=" --impalad_args=-llama_site_path=${LLAMA_CONF}"
export MAX_PYTEST_FAILURES=0
export NUM_CONCURRENT_TESTS=$(nproc)
# Frontend tests fail because of localhost hardcoded everywhere
export FE_TEST=false
# No need to run backend tests - they are identical with non-docker build.
export BE_TEST=false
# TODO: custom cluster tests may provide some useful coverage but require work
# to make them start up dockerised clusters and will probably make more assumptions
# about the cluster being tested.
export CLUSTER_TEST=false
./bin/run-all-tests.sh