bin/run-example

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -o pipefail

# make sure Spark home is set and valid
if [ -z "${SPARK_HOME}" ]; then
  echo "SPARK_HOME is not set" >&2
  exit 1
elif [ ! -d "${SPARK_HOME}" ]; then
  echo "SPARK_HOME does not point to a valid directory" >&2
  exit 1
fi

# pinpoint the project root folder
bin_dir=$( dirname "$0" )
project_dir=$( cd "${bin_dir}/.." ; pwd -P )

printUsageAndExit() {
  cat << EOF

SYNOPSIS
  run-example example-class [example args]

DESCRIPTION
  Helper script to run the given example using \${SPARK_HOME}/bin/spark-submit.

  For example this invocation:

     bin/run-example \\
         org.apache.spark.examples.streaming.akka.ActorWordCount localhost 9999

  will be converted into the following spark-submit command:

     \${SPARK_HOME}/bin/spark-submit \\
         --packages org.apache.bahir:spark-streaming-akka_2.11:2.0.0-SNAPSHOT \\
         --class org.apache.spark.examples.streaming.akka.ActorWordCount \\
             streaming-akka/target/spark-streaming-akka_2.11-2.0.0-SNAPSHOT-tests.jar \\
         localhost 9999

  Before running this script, make sure the SPARK_HOME environment variable is set and
  the Bahir project was built and installed into your local Maven repository by running

     mvn clean install

  For information about [optional] arguments consult the JavaDoc of the specific example class.

USAGE EXAMPLES
EOF
  grep -R "bin/run-example org.apache" --no-filename --include=*.{scala,java,py} "${project_dir}" | tr '`' ' ' | sed 's/^ *\* * /     /g' ; \
  grep -R -A1 "bin/run-example \\\\"   --no-filename --include=*.{scala,java,py} "${project_dir}" | tr '`' ' ' | sed 's/^ *\* * /     /g' | sed '/^--$/d' | sed 'N;s/\\\n *//g'
  exit 1
}

if (( $# == 0 )) || [[ $1 =~ "-h" ]]; then
  printUsageAndExit
fi

# split off example class from example arguments
example_class="$1"; shift;
example_args="$@"

# find the module path and jar files needed to run the given example
example_src_path=$( find "${project_dir}" -path "*${example_class//.//}*" | grep "/examples/src/" | head -1 )
module_path=${example_src_path%"/examples/src/"*}

# we may not have found the source file if the given class is nested inside another class
if [ -z "${module_path}" ]; then
  example_target_path=$( find "${project_dir}" -path "*${example_class//.//}.class" | grep "/target/" | head -1 )
  module_path=${example_target_path%"/target/"*}
fi

# if we found neither source file nor class file for the given example, try matching partial path
if [ -z "${module_path}" ]; then
  example_package="${example_class%.*}"
  example_package_path=$( find "${project_dir}" -path "*${example_package//.//}*" | grep "/examples/src/" | head -1 )
  module_path=${example_package_path%"/examples/src/"*}
fi

if [ -z "${module_path}" ]; then
  echo "Could not find module that contains the example \"${example_class}\"" >&2
  printUsageAndExit
fi

# use the module name to find the tests jar file that contains the example to run
module_name=${module_path#"${project_dir}"/}
module_tests_jar_path=$( find "${module_path}" -name "*${module_name}*-tests.jar" | head -1 )

if [ -z "${module_tests_jar_path}" ] || [ ! -e "${module_tests_jar_path}" ]; then
  echo "Could not find module tests jar file in ${module_path}/target/" >&2
  echo "Run \"mvn clean install\" and retry running this example" >&2
  exit 1
fi

# use maven-help-plugin to determine project version and Scala version
module_version=$( cd "${module_path}" && mvn org.apache.maven.plugins:maven-help-plugin:2.2:evaluate -Dexpression=project.version | grep -v "INFO\|WARNING\|ERROR\|Downloading" | tail -1 )
scala_version=$( cd "${module_path}" && mvn org.apache.maven.plugins:maven-help-plugin:2.2:evaluate -Dexpression=scala.binary.version | grep -v "INFO\|WARNING\|ERROR\|Downloading" | tail -1 )

# we are getting all necessary dependencies from maven, requires running "mvn" or "mvn install" first
spark_package="org.apache.bahir:spark-${module_name}_${scala_version}:${module_version}"

# tests jar contains the examples (see [BAHIR-18])
examples_jar="${module_tests_jar_path}"

# ${SPARK_HOME}/bin/spark-submit \
#  --packages org.apache.bahir:spark-streaming-akka_2.11:2.0.0-SNAPSHOT \
#  --class org.apache.spark.examples.streaming.akka.ActorWordCount \
#      streaming-akka/target/spark-streaming-akka_2.11-2.0.0-SNAPSHOT-tests.jar \
#    localhost 9999

# for Python examples add all of the Bahir project's Python sources to PYTHONPATH, which in local
# mode is easier than creating a zip files to be used with the --py-files option (TODO: BAHIR-35)
# Note that --py-files with individual *.py files does not work if those modules are imported at top
# of the example script but rather imports must be pushed down to after SparkContext initialization
if [[ "$example_class" == *.py ]]; then
  export PYTHONPATH="$( find "$project_dir" -path '*/python' -maxdepth 5 -type d | tr '\n' ':' )$PYTHONPATH"
  cmd="${SPARK_HOME}/bin/spark-submit \
    --packages ${spark_package} \
    ${example_class} \
    ${example_args}"
else
  cmd="${SPARK_HOME}/bin/spark-submit \
    --packages ${spark_package} \
    --class ${example_class} ${examples_jar} \
    ${example_args}"
fi

echo "---"
echo "Spark-Submit command: $cmd"
echo "---"

exec $cmd