solr/scripts/snappuller

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Shell script to copy snapshots of a Solr Lucene collection from the master

orig_dir=$(pwd)
cd ${0%/*}/..
solr_root=$(pwd)
cd ${orig_dir}

unset master_host rsyncd_port master_data_dir master_status_dir snap_name
unset sizeonly stats data_dir user verbose debug compress startStatus
. ${solr_root}/bin/scripts-util

# set up variables
prog=${0##*/}
log=${solr_root}/logs/${prog}.log

# define usage string
USAGE="\
usage: $prog [-M master] [-P portnum] [-D mdir] [-S sdir] [-n snapshot] [-d dir] [-u username] [-svVz]
       -M master   specify hostname of master server from where to pull index
                   snapshot
       -P port     specify rsyncd port number of master server from where to
                   pull index snapshot
       -D          specify directory holding index data on master server
       -S          specify directory holding snapshot status on master server
       -n snapshot pull a specific snapshot by name
       -d          specify directory holding index data on local machine
       -u          specify user to sudo to before running script
       -s          use the --size-only option with rsync
       -v          increase verbosity (-vv show file transfer stats also)
       -V          output debugging info
       -z          enable compression of data
"

# parse args
while getopts M:P:D:S:n:d:u:svVz OPTION
do
    case $OPTION in
    M)
        master_host="$OPTARG"
        ;;
    P)
        rsyncd_port="$OPTARG"
        ;;
    D)
        master_data_dir="$OPTARG"
        ;;
    S)
        master_status_dir="$OPTARG"
        ;;
    n)
        snap_name="$OPTARG"
        ;;
    d)
        data_dir="$OPTARG"
        ;;
    u)
        user="$OPTARG"
        ;;
    s)
        sizeonly="--size-only"
        ;;
    v)
        [[ -n $verbose ]] && stats="--stats" || verbose=v
        ;;
    V)
        debug="V"
        ;;
    z)
        compress="z"
        ;;
    *)
        echo "$USAGE"
        exit 1
    esac
done

[[ -n $debug ]] && set -x

if [[ -z ${master_host} ]]
then
    echo "name of master server missing in $confFile or command line."
    echo "$USAGE"
    exit 1
fi

# try to determine rsyncd port number from $confFile if not specified on
# command line, default to solr_port+10000
if [[ -z ${rsyncd_port} ]]
then
    if [[ "${solr_port}" ]]
    then
        rsyncd_port=`expr 10000 + ${solr_port}`
    else
        echo "rsyncd port number of master server missing in $confFile or command line."
        echo "$USAGE"
        exit 1
    fi
fi

if [[ -z ${master_data_dir} ]]
then
    echo "directory holding index data on master server missing in $confFile or command line."
    echo "$USAGE"
    exit 1
fi

if [[ -z ${master_status_dir} ]]
then
    echo "directory holding snapshot status on master server missing in $confFile or command line."
    echo "$USAGE"
    exit 1
fi

fixUser "$@"

dataDir

# assume relative path to start at ${solr_root}
if [[ "`echo ${master_data_dir}|cut -c1`" != "/" ]]
then
    master_data_dir=${solr_root}/${master_data_dir}
fi
if [[ "`echo ${master_status_dir}|cut -c1`" != "/" ]]
then
    master_status_dir=${solr_root}/${master_status_dir}
fi

# push stats/state to master if necessary
function pushStatus
{
    scp -q -o StrictHostKeyChecking=no ${solr_root}/logs/snappuller.status ${master_host}:${master_status_dir}/snapshot.status.`uname -n`
}

setStartTime

logMessage started by $oldwhoami
logMessage command: $0 $@

if [[ ! -f ${solr_root}/logs/snappuller-enabled ]]
then
    logMessage snappuller disabled
    exit 2
fi

# make sure we can ssh to master
if
    ! ssh -o StrictHostKeyChecking=no ${master_host} id 1>/dev/null 2>&1
then
    logMessage failed to ssh to master ${master_host}
    exit 1
fi

# get directory name of latest snapshot if not specified on command line
if [[ -z ${snap_name} ]]
then
    snap_name=`ssh -o StrictHostKeyChecking=no ${master_host} "perl -e 'chdir q|${master_data_dir}|; print ((sort grep {/^snapshot[.][1-9][0-9]{13}$/} <*>)[-1])'"`
fi
if [[ "${snap_name}" == "" ]]
then
    logMessage no snapshot available on ${master_host} in ${master_data_dir}
    logExit ended 0
else
    name=`basename ${snap_name}`
fi

# clean up after INT/TERM
trap 'echo cleaning up, please wait ...;/bin/rm -rf ${data_dir}/${name} ${data_dir}/${name}-wip;echo ${startStatus} aborted:$(timeStamp)>${solr_root}/logs/snappuller.status;pushStatus;logExit aborted 13' INT TERM

if [[ -d ${data_dir}/${name} || -d ${data_dir}/${name}-wip ]]
then
    logMessage no new snapshot available on ${master_host} in ${master_data_dir}
    logExit ended 0
fi

# take a snapshot of current index so that only modified files will be rsync-ed
# put the snapshot in the 'work-in-progress" directory to prevent it from
# being installed while the copying is still in progress
if [[ "${OS}" == "SunOS" || "${OS}" == "Darwin" || "${OS}" == "FreeBSD"  ]]
then
  orig_dir=$(pwd)
  mkdir ${data_dir}/${name}-wip
  cd ${data_dir}/index
  find . -print|cpio -pdlmu ${data_dir}/${name}-wip 1>/dev/null 2>&1
  cd ${orig_dir}
else
  cp -lr ${data_dir}/index ${data_dir}/${name}-wip
fi
# force rsync of segments and .del files since we are doing size-only
if [[ -n ${sizeonly} ]]
then
    rm -f ${data_dir}/${name}-wip/segments
    rm -f ${data_dir}/${name}-wip/*.del
fi

logMessage pulling snapshot ${name}

# make sure master has directory for hold slaves stats/state
ssh -o StrictHostKeyChecking=no ${master_host} mkdir -p ${master_status_dir}

# start new distribution stats
rsyncStart=`date +'%Y-%m-%d %H:%M:%S'`
if [[ "${OS}" == "Darwin" || "${OS}" == "FreeBSD"  ]]
then
  startTimestamp=`date -j -f '%Y-%m-%d %H:%M:%S' "$rsyncStart" +'%Y%m%d-%H%M%S'`
  rsyncStartSec=`date -j -f '%Y-%m-%d %H:%M:%S' "$rsyncStart" +'%s'`
else
startTimestamp=`date -d "$rsyncStart" +'%Y%m%d-%H%M%S'`
rsyncStartSec=`date -d "$rsyncStart" +'%s'`
fi
startStatus="rsync of `basename ${name}` started:$startTimestamp"
echo ${startStatus} > ${solr_root}/logs/snappuller.status
pushStatus

# rsync over files that have changed
rsync -Wa${verbose}${compress} --delete ${sizeonly} \
${stats} rsync://${master_host}:${rsyncd_port}/solr/${name}/ ${data_dir}/${name}-wip

rc=$?
rsyncEnd=`date +'%Y-%m-%d %H:%M:%S'`
if [[ "${OS}" == "Darwin" || "${OS}" == "FreeBSD"  ]]
then
  endTimestamp=`date -j -f '%Y-%m-%d %H:%M:%S' "$rsyncEnd" +'%Y%m%d-%H%M%S'`
  rsyncEndSec=`date -j -f '%Y-%m-%d %H:%M:%S' "$rsyncEnd" +'%s'`
else
endTimestamp=`date -d "$rsyncEnd" +'%Y%m%d-%H%M%S'`
rsyncEndSec=`date -d "$rsyncEnd" +'%s'`
fi
elapsed=`expr $rsyncEndSec - $rsyncStartSec`
if [[ $rc != 0 ]]
then
  logMessage rsync failed
  /bin/rm -rf ${data_dir}/${name}-wip
  echo ${startStatus} failed:$endTimestamp > ${solr_root}/logs/snappuller.status
  pushStatus
  logExit failed 1
fi

# move into place atomically
mv ${data_dir}/${name}-wip ${data_dir}/${name}

# finish new distribution stats`
echo ${startStatus} ended:$endTimestamp rsync-elapsed:${elapsed} > ${solr_root}/logs/snappuller.status
pushStatus
logExit ended 0