#!/usr/bin/perl
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

use JSON;
use Data::Dumper;
use Time::Local;
use Getopt::Long;
use Pod::Usage;
use strict;

our ($json, $result, $threshold, $process, $start, $ooziehost, $end, $hour, $date, $nexthour, $instance_id, $jobthreshold);
our ($help, $httphost, $query, $query_output, $httphost_url, @failed, @running, @succeeded, @waiting, @unknown);

$json = JSON->new->allow_nonref;

$result = GetOptions ("threshold=i" => \$threshold,
                      "jthreshold=i" => \$jobthreshold,
                      "proc=s" => \$process,
                      "start=s" => \$start,
                      "ooh=s" => \$ooziehost,
                      "end=s" => \$end,
                      "help" => \$help);

&pod2usage() if $help;

die "Process (--proc) must be specified, Run --help\n"  if !$process;

if (!$start && !$end) {
    $hour = `date +%H`;
    chomp($hour);
    $date = `date +%F`;
    chomp($date);
    $nexthour = $hour - 3;
    $end = "$date"."T$hour:00Z";
    $start= "$date"."T$nexthour:00Z";
}

if (!$threshold) {
    $threshold = 120;
}

if (!$jobthreshold) {
    $jobthreshold = 40;
}


$threshold = $threshold * 60;
$jobthreshold = $jobthreshold * 60;

$ooziehost = "oozie.com:5999" if !$ooziehost;
$httphost_url = "http://$ooziehost/falcon/api/processinstance/status/$process?start=$start&end=$end";
$query = "curl -s -H'remote-user: user'"." \"$httphost_url\"";
$query_output = `$query` || die "Curl call failed: $!";

if ($query_output eq "") {
    print "Null output. Check if Falcon is down or jobs are hung from long time\n";
    exit 2;
}

my $instance = from_json( $query_output, { utf8  => 1 } );

my $instances_ref = $instance->{instances};

foreach (@$instances_ref) {
    my $instance_ref = $_;
    my $alert_string = "";
    my $instance_id = $instance_ref->{instance};
    if ($instance_ref->{status} eq "FAILED" || $instance_ref->{status} eq "KILLED" || $instance_ref->{status} eq "SUSPENDED") {
        my $actions_ref = $instance_ref->{actions};
        foreach(@$actions_ref) {
            my $action_ref = $_;
            if ($action_ref->{status} eq "FAILED") {
               $alert_string = "CRITICAL: JOB $instance_id failed at action $action_ref->{action}\n";
            }  elsif ($action_ref->{status} eq "KILLED") {
               $alert_string = "CRITICAL: JOB $instance_id killed at action $action_ref->{action}\n";
            } elsif ($action_ref->{status} eq "SUSPENDED") {
               $alert_string = "CRITICAL: JOB $instance_id is in suspended state\n";
            }
            push (@failed, $alert_string);
        }
    } elsif ($instance_ref->{status} eq "WAITING") {
        #&get_process_lag($instance_id);
        $instance_id =~ s/(-|[A-Z])/:/g;
        my ($year, $mon, $day, $hour, $min) = split /:/, $instance_id;
        $mon = $mon -1;
        my $instance_epoch =  timelocal(00, $min, $hour, $day, $mon, $year);
        my $system_epoch = time();
        my $job_lag = int($system_epoch - $instance_epoch);
        if ($job_lag > $threshold) {
            $alert_string = "CRITICAL: JOB $instance_id is in waiting state for more then given thresholds;";
            push (@waiting, $alert_string);
        } 
    } elsif ($instance_ref->{status} eq "RUNNING") {
        #&get_process_lag($instance_id);
        my $instance_id_raw = $instance_id;
        $instance_id =~ s/(-|[A-Z])/:/g;
        my ($year, $mon, $day, $hour, $min) = split /:/, $instance_id;
        $mon = $mon -1;
        my $instance_epoch =  timelocal(00, $min, $hour, $day, $mon, $year);
        my $system_epoch = time();
        my $job_lag = int($system_epoch - $instance_epoch);
        if ($job_lag > $threshold) {
            $alert_string = "CRITICAL: JOB $instance_id is in running state for more then given thresholds;";
            push (@running, $alert_string);
        } 
        # 2012-06-05T06:40Z
        my @falcon_output = `falcon_job_status.pl --proc $process -jid $instance_id_raw`;
        if ($? != 0 ) {
           $alert_string = "CRITICAL: Failed running falcon_job_status.pl --proc $process -jid $instance_id";
           push (@running, $alert_string);
        }
        next if !$falcon_output[0];
        my $jobruntime;
        if ($falcon_output[0] =~ /last(.*)seconds/ ) {
            $jobruntime = $1;
        }
        $jobruntime =~ s/\s+//g;
        if ($jobruntime  > $jobthreshold) {
            $alert_string = "CRITICAL: JOB $instance_id is in running state from last $jobruntime but threshold is $jobthreshold\n";
            push (@running, $alert_string);
        }

    } elsif ($instance_ref->{status} eq "SUCCEEDED") {
        push (@succeeded, $instance_id);
    } else {
        push (@unknown, $instance_id);
    }
}
&nagios_alert();

sub get_process_lag {
        $instance_id =~ s/(-|[A-Z])/:/g;
        my ($year, $mon, $day, $hour, $min) = split /:/, $instance_id;
        $mon = $mon -1;
        my $instance_epoch =  timelocal(00, $min, $hour, $day, $mon, $year);
        my $system_epoch = time();
        my $job_lag = int($system_epoch - $instance_epoch);
        return 1 if ($job_lag > $threshold);
}

sub nagios_alert {
   if (@failed || @waiting || @running) {
       print @failed if @failed;
       print @waiting if @waiting;
       print @running if @running;
       exit 2;
   } elsif (@unknown) {
       print @unknown;
       exit 1;
   } elsif (@succeeded) {
       print "Succeded:@succeeded";
       exit 0;
   } else {
       print "UNKNOWN STATE\n";
       exit 1;
   }
}

__END__

=head1 NAME
 
Check the status of an Falcon job in a given window and report it to Nagios

=head1 SYNOPSIS

 Usage: check_falcon_job.pl [options]

 where options can be

   --threshold	Threshold in minutes. Jobs in waiting or suspended state will be reported if (systemtime - jobid time) exceeds thsi threshold
   --jthreshold	Threshold in minutes. Jobs in waiting or suspended state will be reported if (systemtime - jobid time) exceeds thsi threshold
   --proc	Specify the process name
   --start	Startime for the sample window
   --ooh	To specify a oozie host. By default it is oozie.red.ua2.inmobi.com
   --end	Endtime for the sample window

 Example:

$ perl check_falcon_job.pl --proc download-summary
Succeded:2012-05-28T05:40Z 2012-05-28T06:40Z
gaminik@proc2000:/opt/mkhoj/ops/lib/nrpe

$ perl check_falcon_job.pl --proc <processname> --start 2012-05-28T01:40Z --end 2012-05-28T06:40Z
Succeded:2012-05-28T01:40Z 2012-05-28T02:40Z 2012-05-28T03:40Z 2012-05-28T04:40Z 2012-05-28T05:40Z 2012-05-28T06:40Z
$ 

$ perl check_falcon_job.pl --proc <processname> --start 2012-05-28T07:40Z --end 2012-05-28T08:40Z
UNKNOWN STATE
$

$ perl check_falcon_job.pl --proc <processname> --start 2012-05-28T03:40Z --end 2012-05-28T07:40Z --threshold 10
CRITICAL: JOB 2012:05:28:07:40: is in waiting state for more then given thresholds;CRITICAL: JOB 2012:05:28:06:40: is in running state for more then given thresholds;
$ 

$perl check_falcon_job.pl --proc <processname> --start 2012-05-28T03:40Z --end 2012-05-28T07:40Z --threshold 100
Succeded:2012-05-28T03:40Z 2012-05-28T04:40Z 2012-05-28T05:40Z
$

=head1 AUTHOR

 Kiran Praneeth <kiran.praneeth@gmail.com>

=cut