#! /usr/bin/env bash

# Copyright 2015 Webindex authors (see AUTHORS)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

BIN_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
export WI_HOME=$( cd "$( dirname "$BIN_DIR" )" && pwd )
export WI_VERSION=0.0.1-SNAPSHOT

if [ -f $WI_HOME/conf/webindex-env.sh ]; then
  . $WI_HOME/conf/webindex-env.sh
else
  . $WI_HOME/conf/webindex-env.sh.example
fi

mkdir -p $WI_HOME/logs

export WI_CONFIG=$WI_HOME/conf/webindex.yml
if [ ! -f $WI_CONFIG ]; then
  export WI_CONFIG=$WI_HOME/conf/webindex.yml.example
  if [ ! -f $WI_CONFIG ]; then
    echo "Could not find webindex.yml or webindex.yml.example in $WI_HOME/conf"
    exit 1
  fi
fi

log4j_config=$WI_HOME/conf/log4j.properties
if [ ! -f $log4j_config ]; then
  log4j_config=$WI_HOME/conf/log4j.properties.example
  if [ ! -f $log4j_config ]; then
    echo "Could not find logj4.properties or log4j.properties.example in $WI_HOME/conf"
    exit 1
  fi
fi

function get_prop {
  echo "`grep $1 $WI_CONFIG | cut -d ' ' -f 2`"
}

COMMAND_LOGFILE=$WI_HOME/logs/$1_`date +%s`.log
DATA_DIR=$WI_HOME/data
mkdir -p $DATA_DIR

case "$1" in
dev)
  pkill -9 -f webindex-dev-server
  cd $WI_HOME
  dev_args="${@:2}"
  mvn -q compile -P webindex-dev-server -Dlog4j.configuration=file:$log4j_config -Dexec.args="$dev_args"
  ;;
getpaths)
  mkdir -p $DATA_DIR
  PATHS_FILE="$2".wat.paths
  if [ ! -f $DATA_DIR/$PATHS_FILE ]; then
    rm -f $DATA_DIR/wat.paths.gz
    PATHS_URL=https://aws-publicdatasets.s3.amazonaws.com/common-crawl/crawl-data/CC-MAIN-$2/wat.paths.gz
    if [[ `wget -S --spider $PATHS_URL 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then
      wget -P $DATA_DIR $PATHS_URL
      gzip -d $DATA_DIR/wat.paths.gz
      mv $DATA_DIR/wat.paths $DATA_DIR/$PATHS_FILE
      echo "Downloaded paths file to $DATA_DIR/$PATHS_FILE"
    else
      echo "Crawl paths file for date $2 does not exist at $PATHS_URL"
      exit 1
    fi
  else
    echo "Crawl paths file already exists at $DATA_DIR/$PATHS_FILE"
  fi
  ;;
copy)
  if [ "$#" -lt 4 -o "$#" -gt 5 ]; then
    echo "Usage: webindex copy <DATE> <RANGE> <DEST> [-fg]"
    exit 1
  fi
  . $BIN_DIR/impl/base.sh
  COMMAND="$SPARK_SUBMIT --class webindex.data.Copy $COMMON_SPARK_OPTS \
    $WI_DATA_DEP_JAR $DATA_DIR/"$2".wat.paths $3 $4"
  if [ "$5" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started copy.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
init)
  if [ "$#" -lt 1 -o "$#" -gt 3 ]; then
    echo "Usage: webindex init <SRC> [-fg]"
    exit 1
  fi
  . $BIN_DIR/impl/base.sh
  COMMAND="$BIN_DIR/impl/init.sh $2"
  if [ "$2" == "-fg" ]; then
    COMMAND="$BIN_DIR/impl/init.sh"
  fi
  if [ "$2" != "-fg" -a "$3" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started init.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
load-hdfs)
  if [ "$#" -lt 2 -o "$#" -gt 3 ]; then
    echo "Usage: webindex load-hdfs <SRC> [-fg]"
    exit 1
  fi
  . $BIN_DIR/impl/base.sh
  FLUO_PROPS=$FLUO_HOME/apps/`get_prop fluoApp`/conf/fluo.properties
  if [ ! -f $FLUO_PROPS ]; then
    echo "Fluo properties file must exist at $FLUO_PROPS"
    exit 1
  fi
  COMMAND="$SPARK_SUBMIT --class webindex.data.LoadHdfs $COMMON_SPARK_OPTS \
    --files $FLUO_PROPS $WI_DATA_DEP_JAR $2"
  if [ "$3" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started load-hdfs.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
	;;
load-s3)
  if [ "$#" -lt 3 -o "$#" -gt 4 ]; then
    echo "Usage: webindex load-s3 <DATE> <RANGE> [-fg]"
    exit 1
  fi
  . $BIN_DIR/impl/base.sh
  FLUO_PROPS=$FLUO_HOME/apps/`get_prop fluoApp`/conf/fluo.properties
  if [ ! -f $FLUO_PROPS ]; then
    echo "Fluo properties file must exist at $FLUO_PROPS"
    exit 1
  fi
  COMMAND="$SPARK_SUBMIT --class webindex.data.LoadS3 $COMMON_SPARK_OPTS \
    --files $FLUO_PROPS $WI_DATA_DEP_JAR $DATA_DIR/"$2".wat.paths $3"
  if [ "$4" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started load-s3.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
test-parser)
  if [ "$#" -lt 3 -o "$#" -gt 4 ]; then
    echo "Usage: webindex test-parser <DATE> <RANGE> [-fg]"
    exit 1
  fi
  . $BIN_DIR/impl/base.sh
  COMMAND="$SPARK_SUBMIT --class webindex.data.TestParser $COMMON_SPARK_OPTS \
    $WI_DATA_DEP_JAR $DATA_DIR/"$2".wat.paths $3"
  if [ "$4" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started data-verify.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
ui)
  pkill -9 -f webindex-web-server
  cd $WI_HOME
  COMMAND="mvn -q compile -P webindex-web-server -Dlog4j.configuration=file:$log4j_config"
  if [ "$2" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started UI.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
splits)
  . $BIN_DIR/impl/base.sh
  COMMAND="$SPARK_SUBMIT --class webindex.data.CalcSplits \
    $COMMON_SPARK_OPTS \
    --conf spark.shuffle.service.enabled=true \
    $WI_DATA_DEP_JAR $2"
  if [ "$2" != "-fg" ]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started splits calculation.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
test)
  COMMAND="$BIN_DIR/impl/test.sh ${@:2}"
  if [[ $@ != *"-fg"* ]]; then
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started webindex test ${@:2}.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
test-id)
  if [ "$#" -gt 3 ]; then
    echo "Usage: webindex test-id <ID> [-fg]"
    exit 1
  fi
  if [ -z "$2" ]; then
    echo "Available tests:"
    echo "----------------"
    cat $WI_HOME/conf/webindex-tests.txt
    exit 1
  fi
  TEST_ARGS="`grep $2 $WI_HOME/conf/webindex-tests.txt | cut -d = -f 2`"
  if [ -z "$TEST_ARGS" ]; then
    echo "Unknown test ID: $2"
    echo "Available tests:"
    echo "----------------"
    cat $WI_HOME/conf/webindex-tests.txt
    exit 1
  fi
  COMMAND="$BIN_DIR/impl/test.sh $TEST_ARGS"
  if [[ $@ != *"-fg"* ]]; then
    COMMAND_LOGFILE=$WI_HOME/logs/test_"$2"_`date +%s`.log
    nohup ${COMMAND} &> $COMMAND_LOGFILE &
    echo "Started webindex run-test $2.  Logs are being output to $COMMAND_LOGFILE"
  else
    ${COMMAND}
  fi
  ;;
kill)
  : ${HADOOP_PREFIX?"HADOOP_PREFIX must be set"}
  FLUO_APP=`get_prop fluoApp`
  FLUO_CMD=$FLUO_HOME/bin/fluo
  if [ ! -f $FLUO_CMD ]; then
    echo "Fluo command script does not exist at $FLUO_CMD"
    exit 1
  fi
  echo "Killing the webindex UI web server..."
  pkill -9 -f webindex-web-server

  echo "Stopping the $FLUO_APP Fluo application (if running)..."
  $FLUO_CMD stop $FLUO_APP

  echo "Killing any webindex Spark jobs running in YARN..."
  YARN=$HADOOP_PREFIX/bin/yarn
  $YARN application -list | grep webindex | while read x; do yarn application -kill `echo $x | cut -d ' ' -f 1` ; done
  ;;
*)
  echo -e "Usage: webindex <command> (<argument>)\n"
  echo -e "Possible commands:\n"
  echo "  dev                         Runs WebIndex development server"
  echo "  getpaths <DATE>             Retrieves paths file for given crawl <DATE> (i.e 2015-18) and stores file in the 'data/' directory"
  echo "                              See https://commoncrawl.org/the-data/get-started/ for possible crawl dates"
  echo "  copy <DATE> <RANGE> <DEST>  Copies CommonCrawl data files from S3 given a <DATE> and <RANGE> (i.e 0-8) into HDFS <DEST> directory"
  echo "  init [<SRC>]                Initializes and starts the WebIndex application. Optionally, a <SRC> HDFS directory can be added to"
  echo "                              to the command to initialize Fluo's table in Accumulo with data before starting the application"
  echo "  load-hdfs <SRC>             Loads data from the HDFS <SRC> directory into Fluo"
  echo "  load-s3 <DATE> <RANGE>      Loads data from S3 into Fluo.  Data is selected using a paths file <DATE> and file <RANGE> (i.e 5-7)"
  echo "  ui                          Starts the webindex UI"
  echo "  splits <SRC>                Calculate splits using data in HDFS <SRC> directory"
  echo "  kill                        Kills the webindex Fluo application and any webindex Spark jobs (if running)"
  echo "  test-id <ID>                Starts a pre-configured webindex test indentified by <ID>.  Run without arguments for a list of tests."
  echo "  test <args>                 Starts a webindex test.  Each test will first remove any previously running test.  It will"
  echo "                              then initialize the webindex application, start the UI, and load data."
  echo "                              Tests can be configured by the following arguments." 
  echo "                              Required args:"
  echo "                                -d <DATE>   Date of common crawl paths file (required)"
  echo "                                -i <RANGE>  Init data range (i.e START-END).  Set to 'none' to not initialize with any data."
  echo "                                -l <RANGE>  Load data range (i.e START-END)"
  echo "                              Optional args:"
  echo "                                -s <SRC>    Source (i.e. hdfs, s3) to use when loading data. If not set, defaults to 's3'"
  echo "                                -e <NUM>    Number of Spark executors to run in all Spark jobs" 
  echo "                                -m <MEM>    Amount of memory (i.e 512m, 1g) to provide to each Spark executor"
  echo "  test-parser <DATE> <RANGE>  Tests parser on data loaded from S3.  Data is selected using a paths file <DATE> and file <RANGE> (i.e 5-7)"
  echo " "
  echo "NOTE: All commands except getpaths will run in background and output to a log by default.  Add -fg to end of these commands"
  echo "to run them in the foreground."
  echo " " 
  exit 1
esac