build/automc/freqsd

#!/local/perl586/bin/perl
#
# freqsd - perform "background" operations required by the automc
# infrastructure, such as building indices and "hit-frequencies" reports

use strict;
use warnings;

use Getopt::Long;
use IPC::DirQueue;

our ( $pidfile, $opt_enq, $opt_kill);
GetOptions(
    "pidfile=s" => \$pidfile,
    "enq=s" => \$opt_enq,
    "kill" => \$opt_kill,
);

my $INHIBIT_SLOW_REPORTS_FLAG_FILE = "/export/home/automc/freqsd/flag.inhibit_slow_reports";
my $dq_fast = IPC::DirQueue->new({ dir => "/export/home/automc/freqsd/dq_fast" });
my $dq_slow = IPC::DirQueue->new({ dir => "/export/home/automc/freqsd/dq_slow" });
$pidfile ||= "/export/home/automc/freqsd/pid";

umask(002);

# ---------------------------------------------------------------------------

# by separating this into two processes, we can get the parent reports issued
# immediately, and the slow reports can gronk away in the background.
# by using IPC::DirQueue,

# the parent process continually generates the faster reports
my $parent_reports = "DETAILS.new DETAILS.all DETAILS.age LOGS.all ".
        "NET.new NET.all NET.age SCOREMAP.new CORPUS.all";

# the child process generates the slow reports
my $child_reports = "OVERLAP.new";

# seconds between build-polls
my $idle_sleep = 600;

# ---------------------------------------------------------------------------

if ($opt_enq) {
  $dq_fast->enqueue_string("", { dir => $opt_enq });
  $dq_slow->enqueue_string("", { dir => $opt_enq });
  exit;
}

if ($opt_kill) {
  die "no -pidfile" unless $pidfile;
  open (IN, "<$pidfile") or die "cannot read $pidfile";
  my $pid = <IN> + 0;
  close IN;
  if ($pid < 2) {
    die "invalid pid: '$pid'";
  }
  kill (15, $pid) or die "kill $pid failed: $!";
  exit;
}

# ---------------------------------------------------------------------------

sub run;
my $am_parent;
my $child_pid = fork();
if ($child_pid < 0) {
  die "fork failed";
}
elsif ($child_pid != 0) {
  $am_parent = 0;
}
else {
  $am_parent = 1;
  $SIG{INT} = \&sigterm_handler;
  $SIG{TERM} = \&sigterm_handler;
}

if ($pidfile) {
  open(OUT, ">$pidfile") or die "cannot write to $pidfile";
  print OUT $$;
  close OUT or die "cannot save $pidfile";
}

sub logit {
  print "LOG: ".join('', @_)." ($$) at ".(scalar localtime time)."\n";
}

# ---------------------------------------------------------------------------

$| = 1;
logit "freqsd starting";
my $is_first_time = 1;

if ($am_parent) {
  while (1) {
    parent_loop();
  }
}
else {
  while (1) {
    child_loop();
  }
}
die "oops! cannot get here";

# ---------------------------------------------------------------------------

sub parent_loop {
  # I'm impatient.  many times when I have to restart this script, I want to
  # see "faster" report results built immediately, without the 10-minute wait.
  # So first time around, just sleep for 5 secs, so that we get started almost
  # immediately.

  my $this_sleep = $idle_sleep;
  if ($is_first_time) {
    $is_first_time = 0;
    $this_sleep = 5;
  }

  my $job = $dq_fast->wait_for_queued_job($this_sleep);

  if ($job && $job->{metadata}->{dir}) {
    # if a dir was specified, it's always a "b" (buildbot) mass-check;
    # that's the assumption here at least
    logit "starting buildbot-requested faster reports";
    run_import_logs($parent_reports, "--tag=b --dir ".$job->{metadata}->{dir});
    make_reports($parent_reports, "--tag=b");
  }
  else {
    logit "starting rsync faster reports";
    run_import_logs($parent_reports, "--tag=n");
    make_reports($parent_reports, "--tag=n");
    # may also be weekly.  no way to differentiate currently until
    # AFTER corpus.hourly is run!  TODO?
  }

  # and the XML indices
  run ("cd masses ; ./rule-qa/automc/gen_info_xml");

  # and the ruleqa CGI page's cache
  run("./masses/rule-qa/automc/ruleqa.cgi -refresh");

  logit "completed faster reports";
  if ($job) { $job->finish(); }
}

# ---------------------------------------------------------------------------

sub child_loop {
  my $job = $dq_fast->wait_for_queued_job($idle_sleep);

  # add switches
  if ($job && $job->{metadata}->{dir}) {
    logit "not running buildbot-requested slow reports; they're too slow!";
    # run_import_logs($child_reports, "--tag=b --dir ".$job->{metadata}->{dir});
  }
  else {
    # create slow reports
    if (-f $INHIBIT_SLOW_REPORTS_FLAG_FILE) {
      logit "inhibited rsync slow reports, $INHIBIT_SLOW_REPORTS_FLAG_FILE exists";
    } else {
      logit "starting rsync slow reports";
      run_import_logs($child_reports, "--tag=n");
      make_reports($child_reports, "--tag=n");
    }

    # recreate the corpus link-farm
    logit "running 'freqsd-infrequent' tasks";
    run ("build/automc/freqsd-infrequent");
  }

  logit "completed slow reports";

  if ($job) { $job->finish(); }
}

sub run_import_logs {
  my ($reports, $opts) = @_;
  run ("cd masses/rule-qa ; ./import-logs ".
        "--override='output_classes=$reports' ".
        "$opts");
}

sub make_reports {
  my ($reports, $opts) = @_;
  run ("cd masses/rule-qa ; ./reports-from-logs ".
        "--override='output_classes=$reports' ".
        "$opts");
}

# ---------------------------------------------------------------------------

sub sigterm_handler {
  warn "received SIGTERM at ".(scalar localtime time)."\n";
  kill(15, $child_pid);
  if ($pidfile) { unlink($pidfile); }
  die "terminated";
}

# ---------------------------------------------------------------------------

sub run {
  my ($cmd, $ignoreexit) = @_;

  print "[$cmd]\n";
  system ($cmd);

  if (!$ignoreexit) {
    warn "command '$cmd' failed with status $?" if (($? >> 8) != 0);
  }
}