bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A simple wrapper script around hadoop which enables the use of qfs easily.
# Using the hadoop command with qfs requires users to specify overrides for
# various parameters (e.g. fs.defaultFS) which can become cumbersome to do on
# the command line every time. The goal here is to provide a clean user
# experience in using hadoop and qfs.

export JAVA_LIBRARY_PATH=/usr/lib/qfs

QFS_OPTS=" -Dfs.qfs.impl=com.quantcast.qfs.hadoop.QuantcastFileSystem"
QFS_OPTS="$QFS_OPTS -Dfs.defaultFS=qfs://<%= scope['qfs::common::metaserver_host'] %>:<%= scope['qfs::common::metaserver_client_port'] %>"
QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerHost=<%= scope['qfs::common::metaserver_host'] %>"
QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerPort=<%= scope['qfs::common::metaserver_port'] %>"

usage() {
  echo "
usage: $0 <options>
  $0 is a simple wrapper around the standard hadoop command that allows for an
  easy interface with qfs. A subset of the standard hadoop commands are
  supported (where it makes sense). For all other commands, use the hadoop
  command directly.

  Commands:
     fs                             query the qfs filesystem
     jar <jar> <main class>         submit a job to hadoop that uses qfs
                                        Note: the main class is required
     distcp <srcurl> <desturl>      copy file or directories recursively
                                        Note: supports copy into qfs from hdfs
  "
  exit 1
}

if [ $# == 0 ]; then
    usage
fi

COMMAND=$1
shift

case $COMMAND in
    # usage flags
    --help|-h)
        usage
        ;;
    fs|distcp)
        hadoop $COMMAND $QFS_OPTS "$@"
        ;;
    jar)
        if [ $# -lt 2 ]; then
            echo "$COMMAND: jar file and main class are required"
            usage
        fi

        JAR_FILE=$1
        MAIN_CLASS=$2
        shift 2
        hadoop $COMMAND $JAR_FILE $MAIN_CLASS $QFS_OPTS "$@"
        ;;
    *)
        echo "$COMMAND: unsupported -- try using the hadoop command directly"
        usage
        ;;
esac