slider-assembly/src/main/bash/slider_setup

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


#
# Base Vars
#
SCRIPT_NAME=`basename $0`
SCRIPT_DIR=`cd $(dirname $0) && pwd`

#
# Functions
#
usage() { 
    echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
    exit 1
}

#
# Parse cmd line args
#
while getopts "f:p:w:n:" opt; do
    case "$opt" in
        f) config=$OPTARG;;
        p) product=$OPTARG;;
        w) worker_cnt=$OPTARG;;
        n) app_name=$OPTARG;;
        *) usage;;
    esac
done
shift $((OPTIND-1))

if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
    usage
fi

# Only support know products for now
if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
   echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
   usage
fi

#
# Source the config
#
source $config

#
# Product URLs
#
if [ $product = "hbase" ]; then
    app_url=$HBASE_APP_URL
elif [ $product = "storm" ]; then
    app_url=$STORM_APP_URL
elif [ $product = "accumulo" ]; then
    app_url=$ACCUMULO_APP_URL
fi

#
# Main
#
echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
mkdir -p $SLIDER_INST_DIR || exit 1
chown yarn:hadoop $SLIDER_INST_DIR || exit 1
echo "SUCCESS"

echo -e "\n## Downloading slider from: $SLIDER_URL"
if [ -f /tmp/${SLIDER_VER}*tar ]; then
   rm -f /tmp/${SLIDER_VER}*tar
fi
cd /tmp && wget $SLIDER_URL || exit 1
echo "SUCCESS"

echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
echo "SUCCESS"

echo -e "\n## Setting conf values"
(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
echo "SUCCESS"

echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
export PATH=$PATH:$JDK_BIN
echo "SUCCESS"

echo -e "\n## Checking version of Hadoop slider was compiled against"
hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
   echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
else
   echo "Compiled against Hadoop version: $hadoop_compiled_ver"
fi
echo "SUCCESS"

echo -e "\n## Setting up HDFS directories for slider"
sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
echo "SUCCESS"

echo -e "\n## Loading the Slider agent"
sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
echo "SUCCESS"

echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
echo "SUCCESS"

echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
echo "SUCCESS"

echo -e "\n## Adding $product to HDFS slider dir"
sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
echo "SUCCESS"

echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
if [ $product = "hbase" ]; then
   component="HBASE_REGIONSERVER"
elif [ $product = "storm" ]; then
   component="SUPERVISOR"
elif [ $product = "accumulo" ]; then
   component="ACCUMULO_TSERVER"
fi
# Update the resource.json file with worker_cnt
python << END
import json
with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
    data = json.load(f)
    data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"

with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
    f.write(json.dumps(data, sort_keys = False, indent = 4))
END
echo "SUCCESS"

# Handle HBase HDFS dir needs
if [ $product = "hbase" ]; then
    echo -e "\n## Creating hbase HDFS dir /apps/hbase"
    sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
    sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
    echo "SUCCESS"
fi

echo -e "\n##Starting app $product with $worker_cnt workers via slider"
sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
    --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
    --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
    --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
echo "SUCCESS"