| #!/usr/bin/env bash |
| |
| #!/bin/bash |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| # echo commands to the terminal output |
| set -ex |
| |
| # Check whether there is a passwd entry for the container UID |
| myuid=$(id -u) |
| mygid=$(id -g) |
| # turn off -e for getent because it will return error code in anonymous uid case |
| set +e |
| uidentry=$(getent passwd $myuid) |
| set -e |
| |
| # If there is no passwd entry for the container UID, attempt to create one |
| if [ -z "$uidentry" ] ; then |
| if [ -w /etc/passwd ] ; then |
| echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:${SPARK_HOME}:/bin/false" >> /etc/passwd |
| else |
| echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" |
| fi |
| fi |
| |
| ###Todo: ensure that MAHOUT_HOME is properly established as an env var. |
| SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*:${$MAHOUT_HOME}/lib/*" |
| env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt |
| readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt |
| |
| if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then |
| SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" |
| fi |
| |
| |
| |
| if ! [ -z ${HADOOP_CONF_DIR+x} ]; then |
| SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH:$MAHOUT_CLASSPATH"; |
| fi |
| |
| case "$1" in |
| driver) |
| shift 1 |
| CMD=( |
| "$SPARK_HOME/bin/spark-submit" |
| --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" |
| --conf "spark.executor.extraLibraryPath"=\ |
| "${MAHOUT_HOME}/lib/mahout-spark_*-dependency-reduced.jar" |
| --deploy-mode client |
| "$@" |
| ) |
| ;; |
| executor) |
| shift 1 |
| CMD=( |
| ${JAVA_HOME}/bin/java |
| "${SPARK_EXECUTOR_JAVA_OPTS[@]}" |
| -Xms$SPARK_EXECUTOR_MEMORY |
| -Xmx$SPARK_EXECUTOR_MEMORY |
| -cp "$SPARK_CLASSPATH:$MAHOUT_CLASSPATH" |
| org.apache.spark.executor.CoarseGrainedExecutorBackend |
| --driver-url $SPARK_DRIVER_URL |
| --executor-id $SPARK_EXECUTOR_ID |
| --conf spark.executor.extraLibraryPath=\ |
| "$MAHOUT_HOME/lib/mahout-spark_*-dependency-reduced.jar" |
| --cores $SPARK_EXECUTOR_CORES |
| --app-id $SPARK_APPLICATION_ID |
| --hostname $SPARK_EXECUTOR_POD_IP |
| ) |
| ;; |
| |
| *) |
| echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..." |
| CMD=("$@") |
| ;; |
| esac |
| |
| # Execute the container CMD under tini for better hygiene |
| exec /sbin/tini -s -- "${CMD[@]}" |