testdata/cluster/admin

#!/bin/bash

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This will create/control/destroy a local hdfs/yarn/kms/kudu cluster.
#
# All roles run on 127.0.0.1, just like the standard mini cluster included with hadoop.
# The difference is with this cluster, each role runs in its own process and has its own
# configs. For each node, the layout of the configs, logs, start/stop scripts, etc, is
# kept as close as possible to a real cluster. For example, the first node will live in
# the dir "cdh-<version>/node-1" and its logs would be at "cdh-<version>/node-1/var/log".
# TODO: Run each node on its own IP address, e.g. 127.0.0.1, 127.0.0.2, and so on.

set -euo pipefail
. $IMPALA_HOME/bin/report_build_error.sh
setup_report_build_error

: ${IMPALA_KERBERIZE=}
: ${INCLUDE_YARN=}

# For Hive 3, we require Yarn for Tez support.
if "$USE_CDP_HIVE"; then
  INCLUDE_YARN=1
fi

while getopts vky OPT; do
  case $OPT in
    v) set -x;;
    k) export IMPALA_KERBERIZE=1;;
    y) export INCLUDE_YARN=1;;
    ?) echo "Usage: $0 [-v (verbose) -k (kerberize) -y (yarn)] ACTION (see source...)"; exit 1;;
  esac
done
shift $(($OPTIND-1))

DIR=$(dirname $0)
NODES_DIR="$DIR/cdh$CDH_MAJOR_VERSION"
NODE_COUNT=3
if [[ "$TARGET_FILESYSTEM" == "hdfs" && "$ERASURE_CODING" = true ]]; then
  NODE_COUNT=5
fi
NODE_PREFIX=node-
COMMON_NODE_TEMPLATE="$DIR/node_templates/common"
NODE_TEMPLATE="$DIR/node_templates/cdh$CDH_MAJOR_VERSION"
TEMPLATE_SUFFIX=".tmpl"
PY_TEMPLATE_SUFFIX=".xml.py"

# Each process should be marked with this so a "pkill -f" can be done to nuke everything.
export KILL_CLUSTER_MARKER=IBelongToTheMiniCluster

if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then
  # The check above indicates that the regular mini-cluster is in use.
  SUPPORTED_SERVICES=(hdfs kms)
  if [ -n "${INCLUDE_YARN}" ]; then
    SUPPORTED_SERVICES+=(yarn)
  fi
else
  # Either a remote distributed file system or a local non-distributed file system is
  # in use. Currently the only service that is expected to work is Kudu, though in theory
  # the other services could work after the proper configuration changes.
  SUPPORTED_SERVICES=()
fi
if $KUDU_IS_SUPPORTED; then
  SUPPORTED_SERVICES+=(kudu)
fi

# All DataNodes and NodeManagers need a unique but fixed address. The IP is fixed at
# 127.0.0.1, so the only difference is the port. The address must be fixed because it is
# used as an identifier. If a node were restarted with a different address it would be
# considered a new node. The values below are arbitrary and may conflict with existing
# services. Fixed ports were preferred to dynamically chosen free ports for consistency.
DATANODE_FREE_PORT_START=31000
DATANODE_FREE_HTTP_PORT_START=31010
DATANODE_FREE_IPC_PORT_START=31020
DATANODE_FREE_HTTPS_PORT_START=31030
NODEMANAGER_FREE_PORT_START=31100
NODEMANAGER_FREE_LOCALIZER_PORT_START=31120
NODEMANAGER_FREE_WEBUI_PORT_START=31140
KUDU_TS_RPC_FREE_PORT_START=31200
KUDU_TS_WEBUI_FREE_PORT_START=31300

# Used to populate config templates later. Any changes made here have no effect on an
# existing cluster.
export HDFS_WEBUI_PORT=5070   # changed from 50070 so it is not ephemeral
export YARN_WEBUI_PORT=8088   # same as default
export KMS_WEBUI_PORT=9600    # changed to make room for non-ephemeral HBase ports
                              # (HADOOP-12811)
export KUDU_WEBUI_PORT=8051   # same as default

# Empty dirs that should be included in the templates. Since git ignores empty dirs it is
# easier to maintain them here.
EMPTY_NODE_DIRS=$(echo data/dfs/{dn,nn} var/{run,lib/hadoop-hdfs,log} \
    var/{log,run}/kudu/{master,ts} var/lib/kudu/{master,ts}/{wal,data})

EASY_ACCESS_LOG_DIR="$IMPALA_CLUSTER_LOGS_DIR"
MINIKDC_INIT=${IMPALA_HOME}/testdata/bin/minikdc.sh

FIND_EXECUTABLE_FILTER="-executable"
if ! find /dev/null "${FIND_EXECUTABLE_FILTER}" 2> /dev/null; then
  # OSX and RHEL5, among others, don't have -executable
  FIND_EXECUTABLE_FILTER="-perm +0111"
fi

#
# Various initialization routines for a Kerberized cluster setup.  We
# do this if either
#    1) IMPALA_KERBERIZE is set
#    2) The config files exist and use kerberos
# If either are true, then we set IMPALA_KERBERIZE ourselves, ensure the
# minikdc is started, source the appropriate env vars from it, and get
# ourselves a fresh TGT (Ticket-Granting-Ticket).
#
function kerberize_setup {
  if is_kerberized; then
    export IMPALA_KERBERIZE=1
  fi

  # No kerberos?  We're done.
  if [ -z "${IMPALA_KERBERIZE}" ]; then
    return
  fi

  # We must have the 'kinit' binary installed so that the minikdc can
  # grant the user a ticket-granting-ticket.
  if ! type -f kinit > /dev/null 2>&1; then
    echo "Unable to locate 'kinit' on this machine, it must be installed"
    echo "in order for you to actually communicate with the kerberized"
    echo "components of the cluster."
    echo "Try this (on ubuntu):"
    echo " --> $ sudo apt-get install krb5-user"
    echo
    exit 1
  fi

  # While we're checking things, make sure the SASL GSSAPI libraries
  # are installed.  These aren't required until we go to start up
  # the impala daemons, but we might as well fail early...
  IMPALA_SASL_PATH="/usr/lib/sasl2 /usr/lib64/sasl2 /usr/local/lib/sasl2"
  IMPALA_SASL_PATH="${IMPALA_SASL_PATH} /usr/lib/x86_64-linux-gnu/sasl2"
  SASL_GSSAPI_FOUND=0
  for i in ${IMPALA_SASL_PATH}; do
    if ls $i/libgssapi* > /dev/null 2>&1; then
      SASL_GSSAPI_FOUND=1
      break
    fi
  done
  if [ ${SASL_GSSAPI_FOUND} -eq 0 ]; then
    echo "Unable to locate the SASL GSSAPI libraries in the following path:"
    echo "${IMPALA_SASL_PATH}"
    echo "This is required for Kerberos communication using SASL, which"
    echo "the Impala daemons depend on.  To install (on Ubuntu), try:"
    echo " --> $ sudo apt-get install libsasl2-modules-gssapi-mit"
    echo
    exit 1
  fi

  # Starting it has no effect if it's already started...
  ${MINIKDC_INIT} start

  # Source the appropriate minikdc environment variables
  . ${MINIKDC_ENV}

  # Give ourselves a fresh TGT.  Debate here whether that should be
  # ${USER} or 'impala'.  Each has issues; see comments in buildall.sh.
  # Stick with ${USER} for now.
  kinit -k -t "${KRB5_KTNAME}" "${MINIKDC_PRINC_USER}"
  if [ $? -ne 0 ]; then
    echo "kinit failure; aborting"
    exit 1
  fi
}

# Return success if the cluster is kerberized
function is_kerberized {
  HCONFSC="`get_hadoop_client_conf_dir`/core-site.xml"
  if [ -f "${HCONFSC}" ]; then
    if grep -qi "Kerberos settings" "${HCONFSC}"; then
      # If the config exists and has kerberos things in it, treat as kerberized
      return 0
    fi
  fi

  return 1
}

function cluster_exists {
  # Just use the first node as an indicator...
  if [[ ! -e "$NODES_DIR/${NODE_PREFIX}1" ]]; then
    return 1
  fi
}

function create_cluster {
  mkdir -p "$NODES_DIR"

  # Used to populate config templates later
  GROUP=$(id -gn)
  export GROUP

  # Blow away existing config files (so we don't pick up kerberos settings)
  rm -f `get_hadoop_client_conf_dir`/*

  if [ ! -z "${IMPALA_KERBERIZE}" ]; then
    kerberize_setup
    echo "Creating Kerberized cluster."
  else
    # Stop the minikdc in case it was running
    . ${MINIKDC_ENV}
    ${MINIKDC_INIT} stop
  fi

  echo "Hostname for internal communication: ${INTERNAL_LISTEN_HOST}" \
       "and for external communication: ${EXTERNAL_LISTEN_HOST}"
  # For consistency, the first node will host all the master roles.
  for ((NODE_IDX=$NODE_COUNT; NODE_IDX >= 1; NODE_IDX--)); do
    NODE=${NODE_PREFIX}$NODE_IDX
    NODE_DIR=$(get_node_dir $NODE)

    echo "Creating $NODE at $NODE_DIR"

    mkdir -p "$NODE_DIR"
    cp -a "$COMMON_NODE_TEMPLATE"/* "$NODE_DIR"
    if [[ -e "$NODE_TEMPLATE" ]]; then
      cp -a "$NODE_TEMPLATE"/* "$NODE_DIR"
    fi
    if [[ $NODE_IDX -gt 1 ]]; then
      # Remove master role scripts from slave nodes
      rm -f "$NODE_DIR/etc/init.d/"{hdfs-namenode,yarn-resourcemanager} \
            "$NODE_DIR/etc/init.d/"{kms,kudu-master}
      # Only run one YARN nodemanager (more memory-efficient to scale up a
      # single NM than run several)
      rm -f "$NODE_DIR/etc/init.d/yarn-nodemanager"
    fi
    for EMPTY_NODE_DIR in $EMPTY_NODE_DIRS; do
      mkdir -p "$NODE_DIR/$EMPTY_NODE_DIR"
    done

    # Add some easy access links closer to IMPALA_HOME
    EASY_ACCESS_LOG_LINK="$EASY_ACCESS_LOG_DIR/cdh$CDH_MAJOR_VERSION-$NODE"
    if [[ ! -e "$EASY_ACCESS_LOG_LINK" ]]; then
      mkdir -p "$EASY_ACCESS_LOG_DIR"
      ln -s "$NODE_DIR/var/log" "$EASY_ACCESS_LOG_DIR"
      mv "$IMPALA_CLUSTER_LOGS_DIR/log" "$EASY_ACCESS_LOG_LINK"
    fi

    # Template population
    DATANODE_PORT=$((DATANODE_FREE_PORT_START++))
    DATANODE_HTTP_PORT=$((DATANODE_FREE_HTTP_PORT_START++))
    DATANODE_IPC_PORT=$((DATANODE_FREE_IPC_PORT_START++))
    DATANODE_HTTPS_PORT=$((DATANODE_FREE_HTTPS_PORT_START++))
    NODEMANAGER_PORT=$((NODEMANAGER_FREE_PORT_START++))
    NODEMANAGER_LOCALIZER_PORT=$((NODEMANAGER_FREE_LOCALIZER_PORT_START++))
    NODEMANAGER_WEBUI_PORT=$((NODEMANAGER_FREE_WEBUI_PORT_START++))
    KUDU_TS_RPC_PORT=$((KUDU_TS_RPC_FREE_PORT_START++))
    KUDU_TS_WEBUI_PORT=$((KUDU_TS_WEBUI_FREE_PORT_START++))
    echo "$NODE will use ports DATANODE_PORT=$DATANODE_PORT," \
        "DATANODE_HTTP_PORT=$DATANODE_HTTP_PORT," \
        "DATANODE_IPC_PORT=$DATANODE_IPC_PORT," \
        "DATANODE_HTTPS_PORT=$DATANODE_HTTPS_PORT," \
        "NODEMANAGER_PORT=$NODEMANAGER_PORT," \
        "NODEMANAGER_LOCALIZER_PORT=$NODEMANAGER_LOCALIZER_PORT," \
        "NODEMANAGER_WEBUI_PORT=$NODEMANAGER_WEBUI_PORT," \
        "KUDU_TS_RPC_PORT=$KUDU_TS_RPC_PORT," \
        "and KUDU_TS_WEBUI_PORT=$KUDU_TS_WEBUI_PORT"

    export NODE NODE_DIR
    export DATANODE_PORT DATANODE_HTTP_PORT DATANODE_IPC_PORT DATANODE_HTTPS_PORT
    export NODEMANAGER_PORT NODEMANAGER_LOCALIZER_PORT NODEMANAGER_WEBUI_PORT
    export KUDU_TS_RPC_PORT KUDU_TS_WEBUI_PORT
    for TEMPLATE_PATH in $(find "$NODE_DIR" -name "*$TEMPLATE_SUFFIX"); do
      ACTUAL_PATH="${TEMPLATE_PATH%$TEMPLATE_SUFFIX}"

      # Search for strings like ${FOO}, if FOO is defined in the environment then replace
      # "${FOO}" with the environment value.
      perl -wpl -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' \
          "$TEMPLATE_PATH" > "$ACTUAL_PATH.1"

      # Chop out everything between the BEGIN/END Kerberos comments if
      # not kerberized
      if [ -z "${IMPALA_KERBERIZE}" ]; then
        sed '/<!-- BEGIN Kerberos/,/END Kerberos settings -->/d' \
            "$ACTUAL_PATH.1" > "$ACTUAL_PATH"
      else
        cp "$ACTUAL_PATH.1" "$ACTUAL_PATH"
      fi

      # Assumes that environment variables will be ALL CAPS...
      if grep '\${[A-Z_]*}' "$ACTUAL_PATH"; then
        echo "Found undefined variables in $ACTUAL_PATH, aborting"
        exit 1
      fi

      if [[ -x "$TEMPLATE_PATH" ]]; then
        chmod u+x "$ACTUAL_PATH"
      fi
      rm "$TEMPLATE_PATH" "$ACTUAL_PATH.1"
    done
    # Substitute python-templated XML files.
    # TODO(todd): move over all the XML templates to be Python-based.
    for TEMPLATE_PATH in $(find "$NODE_DIR" -name "*$PY_TEMPLATE_SUFFIX"); do
      ACTUAL_PATH="${TEMPLATE_PATH%$PY_TEMPLATE_SUFFIX}".xml
      $IMPALA_HOME/bin/generate_xml_config.py $TEMPLATE_PATH $ACTUAL_PATH
      rm $TEMPLATE_PATH
    done
  done
}

function start_cluster {
  if ! cluster_exists; then
    echo "The cluster must be created first"
    return 1
  fi

  if [ ! -z "${IMPALA_KERBERIZE}" ] && ! is_kerberized; then
    echo "Kerberized start requested, but the config files aren't set up"
    echo "for kerberos.  Destroy the cluster and rebuild it:"
    echo "  --> $ ./testdata/cluster/admin delete_cluster"
    echo "  --> $ IMPALA_KERBERIZE=1 ./testdata/cluster/admin create_cluster"
    exit 1
  fi

  kerberize_setup
  if is_kerberized; then
    echo "Starting Kerberized cluster."
  fi

  if [ ${#SUPPORTED_SERVICES[@]} -gt 0 ]; then
    for SERVICE in ${SUPPORTED_SERVICES[@]-}; do
      # If all of the following are true, we wait for ntp to sync before proceeding:
      #
      # 1. Kudu is being started
      # 2. An internet connection is available (checked by pinging $NTP_CANARY)
      # 3. ntp-wait is installed (checked by calling ntp-wait with the "--help" flag)
      if [[ "${SERVICE}" == "kudu" ]]; then
        NTP_CANARY=pool.ntp.org
        if ! ping -c 1 -w 5 "${NTP_CANARY}" >/dev/null 2>/dev/null; then
          echo "WARNING: cannot reach ${NTP_CANARY}; ntp sync recommended for Kudu"
        else
          # ntp-wait is in /usr/sbin on both CentOS and Debian-based systems, but regular
          # users don't always have /usr/sbin in $PATH. Use the absolute path.
          ntp_wait=$(command -v ntp-wait || echo /usr/sbin/ntp-wait)
          if [[ ! -x $ntp_wait ]]; then
            echo "WARNING: ntp-wait not installed; ntp sync recommended for Kudu"
          elif ! $ntp_wait -v; then
            echo "ntp-wait failed; cannot start kudu"
            return 1
          fi
        fi
      fi
      start $SERVICE
    done

    # If there any services to start, check that the cluster is still alive...
    sleep 10
    check_cluster_status
  fi
  return $?
}

function start {
  if [[ $# -ne 1 ]]; then
    echo start must be called with a single argument -- the service to start. 1>&2
    exit 1
  fi
  local SERVICE=$1

  WEBUI_PORT_VAR="$(echo "$SERVICE" | awk '{print toupper($0)}')_WEBUI_PORT"
  echo "Starting ${SERVICE} (Web UI - http://localhost:${!WEBUI_PORT_VAR})"
  exec_init_script "$SERVICE*" start
  if [ "${SERVICE}" = "hdfs" ]; then
      chmod_hdfs_root
  fi
}

function chmod_hdfs_root {
    # When created, the cluster only contains the root directory with
    # 0755 permissions and owned by hdfs.  In a kerberized
    # environment, this means that no one but hdfs can create
    # anything!  Chmod / to 01777 so that other users can create
    # things.  Insecure, but easy.  While we're here, make other sane
    # and permissive initial directories
    if is_kerberized; then
        PREVIOUS_PRINCIPAL=`klist | grep ^Default | awk '{print $3}'`
        # Become hdfs:
        kinit -k -t ${KRB5_KTNAME} ${MINIKDC_PRINC_HDFS}
        # Do the chmod, etc
        hadoop fs -chmod 1777 /
        if ! hadoop fs -ls /tmp > /dev/null 2>&1; then
            for i in /tmp /home /user; do
                hadoop fs -mkdir $i
                hadoop fs -chmod 1777 $i
            done
        fi
        # Become ourselves again.
        kinit -k -t ${KRB5_KTNAME} ${PREVIOUS_PRINCIPAL}
    fi
}

function exec_init_script {
  local SCRIPT_NAME="$1"
  shift
  local CMD="$1"

  local PIDS=()
  for SCRIPT in $(find "$NODES_DIR" -path "*/$NODE_PREFIX*/etc/init.d/$SCRIPT_NAME" \
      $FIND_EXECUTABLE_FILTER -type f); do
    if "$SCRIPT" status &>/dev/null; then
      RUNNING=true
    else
      RUNNING=false
    fi
    case "$CMD" in
      start) if ! $RUNNING; then "$SCRIPT" start; fi;;
      stop) if $RUNNING; then "$SCRIPT" stop; fi;;
      *) "$SCRIPT" "$@";;
    esac &
    PIDS+=($!)
  done

  # Always wait for all the pids, otherwise the output gets messed up. If this function
  # were to return while the background processes are still running, they may print
  # output that makes it appear as though this process is still running.
  local RET=0
  for PID in ${PIDS[@]}; do
    if ! wait $PID; then
      RET=1
    fi
  done
  return $RET
}

function check_cluster_status {
  if ! cluster_exists; then
    echo "The cluster does not exist"
    return 1
  fi

  ROLE_COUNT=0
  NOT_RUNNING=()
  for NODE_DIR in "$NODES_DIR/$NODE_PREFIX"*; do
    for SERVICE in ${SUPPORTED_SERVICES[@]-}; do
      for SCRIPT in $(find "$NODE_DIR" -path "*/etc/init.d/$SERVICE*" $FIND_EXECUTABLE_FILTER \
          -type f); do
        ROLE_COUNT=$((ROLE_COUNT + 1))
        if ! "$SCRIPT" status &>/dev/null; then
          NOT_RUNNING+=("\n$(basename $SCRIPT) is not running on $(basename $NODE_DIR)")
        fi
      done
    done
  done

  case ${#NOT_RUNNING[@]} in
    0) echo "The cluster is running"; return;;
    $ROLE_COUNT) echo "The cluster is not running"; return 1;;
    *) echo -e "${NOT_RUNNING[@]}"; return 1;;
  esac
}

function stop_cluster {
  if cluster_exists; then
    # Stop services in reverse order to give them a chance to shutdown cleanly
    for ((SERVICE_IDX=${#SUPPORTED_SERVICES[@]} - 1; SERVICE_IDX >= 0; SERVICE_IDX--)); do
      stop "${SUPPORTED_SERVICES[$SERVICE_IDX]}"
    done
  fi
  # Kill everything anyways just in case a git clean -xdf was done
  pkill -u $USER -f $KILL_CLUSTER_MARKER || true
}

function stop {
  if [[ $# -ne 1 ]]; then
    echo stop must be called with a single argument -- the service to stop. 1>&2
    exit 1
  fi
  local SERVICE=$1

  echo "Stopping ${SERVICE}"
  exec_init_script "$SERVICE*" stop
}

function restart {
  if [[ $# -ne 1 ]]; then
    echo restart must be called with a single argument -- the service to restart. 1>&2
    exit 1
  fi
  local SERVICE=$1
  stop $SERVICE
  start $SERVICE
}

function delete_data {
  # Delete namenode, datanode and KMS data while preserving directory structure.
  rm -rf "$NODES_DIR/$NODE_PREFIX"*/data/dfs/{nn,dn}/*
  rm -f "$NODES_DIR/$NODE_PREFIX"*/data/kms.keystore
  delete_kudu_data
}

function delete_kudu_data {
  rm -rf "$NODES_DIR/$NODE_PREFIX"*/var/lib/kudu/{master,ts}/*
}

function delete_cluster {
  pkill -u $USER -f $KILL_CLUSTER_MARKER || true
  rm -rf "$NODES_DIR"
}

function get_node_dir {
  if $IS_OSX; then
    greadlink -f "$NODES_DIR/$1"
  else
    readlink -f "$NODES_DIR/$1"
  fi
}

function get_hadoop_client_conf_dir {
  echo "$NODES_DIR/$NODE_PREFIX"1/etc/hadoop/conf
}

COMMAND=$1
shift
case $COMMAND in
  check_cluster_status | cluster_exists | is_kerberized)
    # Use an "if" to avoid triggering the ERR trap.
    if ! $COMMAND "$@"; then
      exit 1
    fi;;
  *) $COMMAND "$@";;
esac