bin/processSinkFiles.sh

#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

pid=$$

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/chukwa-config.sh

if [ "$CHUKWA_IDENT_STRING" = "" ]; then
  export CHUKWA_IDENT_STRING="$USER"
fi

trap 'remove_cron;rm -f $CHUKWA_PID_DIR/chukwa-$CHUKWA_IDENT_STRING-processSinkFiles.sh.pid ${CHUKWA_PID_DIR}/ProcessSinkFiles.pid; exit 0' 1 2 15
echo "${pid}" > "$CHUKWA_PID_DIR/ProcessSinkFiles.pid"


function remove_cron {
    mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
    crontab -l | grep -v ${CHUKWA_HOME}/bin/hourlyRolling.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
    cat /tmp/cron.${CURRENT_DATE} | grep -v ${CHUKWA_HOME}/bin/dailyRolling.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
    crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
    rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
    rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
}

function add_cron {
    mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
    crontab -l > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
    crontest=$?

    if [ "X${crontest}" != "X0" ]; then
      cat > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
16 * * * * ${CHUKWA_HOME}/bin/hourlyRolling.sh --config ${CHUKWA_CONF_DIR} >& ${CHUKWA_LOG_DIR}/hourly.log
30 1 * * * ${CHUKWA_HOME}/bin/dailyRolling.sh --config ${CHUKWA_CONF_DIR} >& ${CHUKWA_LOG_DIR}/dailyRolling.log
CRON
    else
      grep -v "${CHUKWA_HOME}/bin/hourlyRolling.sh" ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}  | grep -v "${CHUKWA_HOME}/bin/dailyRolling.sh" > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
      mv ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2 ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
      cat >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
16 * * * * ${CHUKWA_HOME}/bin/hourlyRolling.sh --config ${CHUKWA_CONF_DIR} >& ${CHUKWA_LOG_DIR}/hourly.log
30 1 * * * ${CHUKWA_HOME}/bin/dailyRolling.sh --config ${CHUKWA_CONF_DIR} >& ${CHUKWA_LOG_DIR}/dailyRolling.log
CRON
    fi

    # save crontab
    echo -n "Registering cron jobs.."
    crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
    rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
    echo "done"
}

if [ "X$1" = "Xstop" ]; then
  echo -n "Shutting down processSinkFiles.sh..."
  kill -TERM `cat ${CHUKWA_PID_DIR}/ProcessSinkFiles.pid`
  echo "done"
  exit 0
fi

if [ "X$1" = "Xwatchdog" ]; then
  add_cron
fi


HADOOP_OPTS="$HADOOP_OPTS -DAPP=demux -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} "
export HADOOP_OPTS
export HADOOP_CONF_DIR
HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "

while [ 1 ]
 do
  debugDate=`date `
  startTime=`date +%s`
  now=`date +%Y%m%d_%H_%M%S`
  strDate=`date +%Y%m%d_%H_%M%S`
  srcDoneHdfsDir="/chukwa/postprocess/srcSink$now/"
  
  
  destArchiveDir=`date +%Y%m%d/%H/%M%S`

  echo "Running $strDate $now" >> "${CHUKWA_LOG_DIR}/mr.log"

  echo "srcDoneHdfsDir: $srcDoneHdfsDir " >> "${CHUKWA_LOG_DIR}/mr.log"

  $HADOOP_CMDE dfs -mkdir $srcDoneHdfsDir/doneFile
  echo "done with mkdir" >> "${CHUKWA_LOG_DIR}/mr.log"
 
  $HADOOP_CMDE dfs -mv "/chukwa/logs/*/*.done" ${srcDoneHdfsDir}/doneFile
  endMoveTime=`date +%s`
  moveDuration=$(( $endMoveTime - $startTime))
  echo "moveDuration $moveDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
  debugDate=`date `
  echo "$debugDate done with mv logs" >> "${CHUKWA_LOG_DIR}/mr.log"
 
  # Build the archive
  $HADOOP_CMDE jar  ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveBuilder Stream ${srcDoneHdfsDir}/doneFile /chukwa/archives/raw/${destArchiveDir}
  endArchiveTime=`date +%s`
  archiveDuration=$(( $endArchiveTime - $endMoveTime))
  echo "archiveDuration $archiveDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
  debugDate=`date `
  echo "$debugDate done with chuckwaArchiveBuilder" >> "${CHUKWA_LOG_DIR}/mr.log"
  
  
  ## Archive available call all processors
  
  
  $HADOOP_CMDE jar  ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.Demux -Dmapred.compress.map.output=true -Dmapred.map.output.compression.codec=org.apache.hadoop.io.compress.LzoCodec -Dmapred.output.compress=true -Dmapred.output.compression.type=BLOCK -r 4 /chukwa/archives/raw/${destArchiveDir} ${srcDoneHdfsDir}/demux
  endDemuxTime=`date +%s`
  demuxDuration=$(( $endDemuxTime - $endArchiveTime))
  echo "demuxDuration $demuxDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
  debugDate=`date `
  echo "$debugDate done with demux job" >> "${CHUKWA_LOG_DIR}/mr.log"
   
  ${JAVA_HOME}/bin/java -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.extraction.database.DatabaseLoader "${srcDoneHdfsDir}/demux" SystemMetrics Df Hadoop_dfs Hadoop_jvm Hadoop_mapred Hadoop_rpc MSSRGraph MRJobCounters NodeActivity HodJob HodMachine Hadoop_dfs_FSDirectory Hadoop_dfs_FSNamesystem Hadoop_dfs_datanode Hadoop_dfs_namenode Hadoop_jvm_metrics Hadoop_mapred_job Hadoop_mapred_jobtracker Hadoop_mapred_shuffleOutput Hadoop_mapred_tasktracker Hadoop_rpc_metrics JobData TaskData HDFSUsage ClientTrace
  endDbLoaderTime=`date +%s`
  dbLoaderDuration=$(( $endDbLoaderTime - $endDemuxTime))
  echo "dbLoaderDuration $dbLoaderDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
  debugDate=`date `
  echo "$debugDate done with dbLoader job" >> "${CHUKWA_LOG_DIR}/mr.log"
   
  $HADOOP_CMDE jar ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.MoveToRepository ${srcDoneHdfsDir}/demux ${chuwaRecordsRepository}
  endMoveToRepoTime=`date +%s`
  moveToRepoDuration=$(( $endMoveToRepoTime - $endDbLoaderTime))
  echo "moveToRepoDuration $moveToRepoDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
  debugDate=`date `
  echo "$debugDate done with MoveToRepository" >> "${CHUKWA_LOG_DIR}/mr.log"
  
  now=`date +%s`
  strDate=`date +%m/%d/%y%n`
  debugDate=`date `
  echo "$debugDate Stopping ${strDate} ${now}" >> "${CHUKWA_LOG_DIR}/mr.log"
  
  endTime=`date +%s`
  duration=$(( $endTime - $startTime))
  echo "Duration: $duration s" >> "${CHUKWA_LOG_DIR}/mr.log"
  
  if [ $duration -lt 300 ]; then
   sleepTime=$(( 300 - $duration)) 
   echo "Sleep: $sleepTime s" >> "${CHUKWA_LOG_DIR}/mr.log"
   SLEEP_COUNTER=`expr $sleepTime / 5`
   while [ $SLEEP_COUNTER -gt 1 ]; do
       sleep 5
       SLEEP_COUNTER=`expr $SLEEP_COUNTER - 1`
   done
  fi
done