modules/distribution/src/main/scripts/fluo

#! /usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more contributor license
# agreements. See the NOTICE file distributed with this work for additional information regarding
# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License. You may obtain a
# copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.

SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do
   bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
   SOURCE="$(readlink "$SOURCE")"
   [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE"
done
# Set up variables needed by fluo-env.sh
export bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
export basedir="$( cd -P ${bin}/.. && pwd )"
export conf="$basedir/conf"
export lib="$basedir/lib"
export cmd="$1"
case "$cmd" in
  oracle|worker|init) export app="$2" ;;
esac
export FLUO_VERSION=${project.version}

if [ ! -f "$conf/fluo-env.sh" ]; then
  echo "fluo-env.sh must exist in $conf"
  exit 1
fi
source "$conf/fluo-env.sh"
export CLASSPATH
JAVA="java ${JAVA_OPTS[*]}"
if [ -f "$FLUO_CONN_PROPS" ]; then
  JAVA="$JAVA -Dfluo.conn.props=$FLUO_CONN_PROPS"
fi

deprecated_fluo_props=$conf/fluo.properties

if [[ -f "$deprecated_fluo_props" ]]; then
  echo "Fluo no longer supports running applications in YARN using Twill (Twill does not support Hadoop 3).  The presence of the file $deprecated_fluo_props implies this is desired."
  exit 1
fi

# stop if any command fails
set -e

function copy_config {
  if [ -f "$conf/$1" ]; then
    cp "$conf/$1" "$APP_CONF_DIR/"
  elif [ -f "$conf/examples/$1" ]; then
    cp "$conf/examples/$1" "$APP_CONF_DIR"
  else
    echo "Config file $1 not found in $conf or $conf/examples"
    exit 1
  fi
}

function print_usage {
  echo -e "Usage: fluo <command> (<argument> ...)\n"
  echo -e "Possible commands:\n"
  echo "  init -a <app> -p <appProps>   Initializes Fluo application for <app> using <appProps>. Run with -h to see additional args."
  echo "  remove -a <app>               Removes Fluo application for <app>"
  echo "  classpath                     Prints the classpath setup in fluo-env.sh"
  echo "  config -a <app>               Prints application configuration stored in Zookeeper for <app>"
  echo "  get-jars -a <app> -d <dir>    Copies <app> jars from DFS to local <dir>"
  echo "  list                          Lists all Fluo applications in Fluo instance"
  echo "  scan -a <app>                 Prints snapshot of data in Fluo <app>"
  echo "  status -a <app>               Prints status of Fluo application for <app>"
  echo "  oracle -a <app>               Starts Fluo Oracle process for <app>"
  echo "  worker -a <app>               Starts Fluo Worker process for <app>"
  echo "  version                       Prints the version of Fluo"
  echo "  wait -a <app>                 Waits until all notifications are processed for <app>"
  echo "  exec <app> <class> {<arg>}    Executes <class> with <args> using classpath for <app>";
  echo " "
  exit 1
}

function check_conn_props {
  if [ ! -f "$FLUO_CONN_PROPS" ]; then
    echo "$FLUO_CONN_PROPS must exist!"
    exit 1
  fi
}

function verify_app {
  if [ -z "$1" ]; then
    echo -e "The application name (set by <app>) cannot be an empty string!\n"
    print_usage
  fi
  if [[ $1 = *"-h"* ]]; then
    print_usage
  fi
}

function check_hadoop {
  if [[ -z $HADOOP_PREFIX ]]; then
    echo "HADOOP_PREFIX needs to be set!"
    exit 1
  fi
  if [ ! -d "$HADOOP_PREFIX" ]; then
    echo "HADOOP_PREFIX=$HADOOP_PREFIX is not a valid directory.  Please correct it in your .bashrc or fluo-env.sh"
    exit 1
  fi
}

function setup_service {
  if [[ "$@" =~ ^.*-a\ *([^\ ]*).*$ ]]; then
    app=${BASH_REMATCH[1]}
    verify_app "$app"
    check_conn_props
    # create a temp dir to fetch application jars to
    app_lib=$(mktemp -d "$FLUO_TMP"/fluo-"$app"-XXXXXXXXX) || die "fatal: unable to allocate a temporary directory"
    # schedule removal of app_lib tmp dir when this script exits
    trap "rm -rf '""$app_lib""'" EXIT HUP INT QUIT TERM
    $JAVA org.apache.fluo.command.FluoGetJars -d "$app_lib" "$@"
    export CLASSPATH="$conf:$app_lib/*:$CLASSPATH"
  else
    echo "Application name must be set!"
    print_usage
    exit 1
  fi
}

case "$1" in
config)
  check_conn_props
  $JAVA org.apache.fluo.command.FluoConfig "${@:2}"
  ;;
get-jars)
  check_conn_props
  $JAVA org.apache.fluo.command.FluoGetJars "${@:2}"
  ;;
init)
  if [[ $2 = *"-h"* ]]; then
    $JAVA org.apache.fluo.command.FluoInit -h
    exit 0
  fi
  init_dir=$($JAVA org.apache.fluo.command.FluoInit "${@:2}" --retrieveProperty fluo.observer.init.dir)
  if [ -d "$init_dir" ]; then
    echo "Adding $init_dir/* to CLASSPATH"
    export CLASSPATH="$init_dir/*:$CLASSPATH"
  fi
  $JAVA org.apache.fluo.command.FluoInit "${@:2}"
  ;;
remove)
  if [[ $2 = *"-h"* ]]; then
    $JAVA org.apache.fluo.command.FluoRemove -h
    exit 0
  fi
  $JAVA org.apache.fluo.command.FluoRemove "${@:2}"
  ;;
oracle)
  if [[ $2 = *"-h"* ]]; then
    $JAVA org.apache.fluo.command.FluoOracle -h
    exit 0
  fi
  setup_service "${@:2}"
  $JAVA org.apache.fluo.command.FluoOracle "${@:2}"
  ;;
worker)
  if [[ $2 = *"-h"* ]]; then
    $JAVA org.apache.fluo.command.FluoWorker -h
    exit 0
  fi
  setup_service "${@:2}"
  $JAVA org.apache.fluo.command.FluoWorker "${@:2}"
  ;;
scan)
  if [ -f "$FLUO_CONN_PROPS" ]; then
    $JAVA org.apache.fluo.command.FluoScan "${@:2}"
  else
    check_hadoop
    java org.apache.fluo.cluster.command.FluoCommand "$basedir" "$HADOOP_PREFIX" "$@"
  fi
  ;;
ps)
  jps -m | grep Fluo
  ;;
list)
  if [ -f "$FLUO_CONN_PROPS" ]; then
    $JAVA org.apache.fluo.command.FluoList "${@:2}"
  else
    check_hadoop
    java org.apache.fluo.cluster.command.FluoCommand "$basedir" "$HADOOP_PREFIX" list app "${@:2}"
  fi
  ;;
classpath)
  echo "$CLASSPATH"
  ;;
exec)
  app=$2
  verify_app "$app"
  check_conn_props
  # create a temp dir to fetch application jars to
  app_lib=$(mktemp -d "$FLUO_TMP"/fluo-"$app"-XXXXXXXXX) || die "fatal: unable to allocate a temporary directory"
  # schedule removal of app_lib tmp dir when this script exits
  trap "rm -rf '""$app_lib""'" EXIT HUP INT QUIT TERM
  $JAVA org.apache.fluo.command.FluoGetJars -d "$app_lib" -a "$app"
  export CLASSPATH="$conf:$app_lib/*:$CLASSPATH"
  $JAVA org.apache.fluo.command.FluoExec "${@:2}"
  ;;
status)
  $JAVA org.apache.fluo.command.FluoStatus "${@:2}"
  ;;
version)
  echo "$FLUO_VERSION"
  ;;
wait)
  $JAVA org.apache.fluo.command.FluoWait "${@:2}"
  ;;
*)
  print_usage
esac