dev/spark/bin/spark-submit

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is a fake version of "spark-submit" to be used during Livy tests that run Spark as
# a child process. It does some basic parsing of Spark options to figure out the classpath
# to be used, and then just runs the SparkSubmit class directly.
#

PROP_FILE=
DRIVER_CP=
DRIVER_OPTS=
CP_KEY="spark.driver.extraClassPath"
OPTS_KEY="spark.driver.extraJavaOptions"

INDEX=1
ARGS=($@)

if [ -n "$SPARK_CONF_DIR" ]; then
  PROP_FILE="$SPARK_CONF_DIR/spark-defaults.conf"
fi

for IDX in $(seq 0 ${#ARGS[@]}); do
  ARG=${ARGS[$IDX]}
  NEXT=$((IDX + 1))
  case $ARG in
    --conf)
      CONF="${ARGS[$NEXT]}"
      IFS='=' read KEY VALUE <<< "$CONF"
      if [ "$KEY" = "$CP_KEY" ]; then
        DRIVER_CP="$VALUE"
      elif [ "$KEY" = "$OPTS_KEY" ]; then
        DRIVER_OPTS="$VALUE"
      fi
      ;;
    --driver-class-path)
      DRIVER_CP="${ARGS[$NEXT]}"
      ;;
    --properties-file)
      PROP_FILE="${ARGS[$NEXT]}"
      ;;
  esac
done

read_opt() {
  local FILE="$1"
  local KEY="$2"
  CONF=$(grep -s "^$KEY=" "$PROP_FILE" | tail -n 1)
  if [ -n "$CONF" ]; then
    IFS='=' read KEY VALUE <<< "$CONF"
    echo "$VALUE"
  fi
}

if [ -n "$PROP_FILE" ]; then
  if [ -z "$DRIVER_CP" ]; then
    DRIVER_CP=$(read_opt "$PROP_FILE" "$CP_KEY")
  fi
  if [ -z "$DRIVER_OPTS" ]; then
    DRIVER_OPTS=$(read_opt "$PROP_FILE" "$OPTS_KEY")
  fi
fi

if [ -n "$LIVY_TEST_CLASSPATH" ]; then
  DRIVER_CP="$DRIVER_CP:$LIVY_TEST_CLASSPATH"
fi

if [ -n "$HADOOP_CONF_DIR" ]; then
  DRIVER_CP="$HADOOP_CONF_DIR:$DRIVER_CP"
fi

echo "Running Spark: " $JAVA_HOME/bin/java $DRIVER_OPTS org.apache.spark.deploy.SparkSubmit "$@" >&2
exec $JAVA_HOME/bin/java $DRIVER_OPTS -cp "$DRIVER_CP" org.apache.spark.deploy.SparkSubmit "$@"