| #!/local/perl586/bin/perl |
| # |
| # freqsd - perform "background" operations required by the automc |
| # infrastructure, such as building indices and "hit-frequencies" reports |
| |
| use strict; |
| use warnings; |
| |
| use Getopt::Long; |
| use IPC::DirQueue; |
| |
| our ( $pidfile, $opt_enq, $opt_kill); |
| GetOptions( |
| "pidfile=s" => \$pidfile, |
| "enq=s" => \$opt_enq, |
| "kill" => \$opt_kill, |
| ); |
| |
| my $INHIBIT_SLOW_REPORTS_FLAG_FILE = "/export/home/automc/freqsd/flag.inhibit_slow_reports"; |
| my $dq_fast = IPC::DirQueue->new({ dir => "/export/home/automc/freqsd/dq_fast" }); |
| my $dq_slow = IPC::DirQueue->new({ dir => "/export/home/automc/freqsd/dq_slow" }); |
| $pidfile ||= "/export/home/automc/freqsd/pid"; |
| |
| umask(002); |
| |
| # --------------------------------------------------------------------------- |
| |
| # by separating this into two processes, we can get the parent reports issued |
| # immediately, and the slow reports can gronk away in the background. |
| # by using IPC::DirQueue, |
| |
| # the parent process continually generates the faster reports |
| my $parent_reports = "DETAILS.new DETAILS.all DETAILS.age LOGS.all ". |
| "NET.new NET.all NET.age SCOREMAP.new CORPUS.all"; |
| |
| # the child process generates the slow reports |
| my $child_reports = "OVERLAP.new"; |
| |
| # seconds between build-polls |
| my $idle_sleep = 600; |
| |
| # --------------------------------------------------------------------------- |
| |
| if ($opt_enq) { |
| $dq_fast->enqueue_string("", { dir => $opt_enq }); |
| $dq_slow->enqueue_string("", { dir => $opt_enq }); |
| exit; |
| } |
| |
| if ($opt_kill) { |
| die "no -pidfile" unless $pidfile; |
| open (IN, "<$pidfile") or die "cannot read $pidfile"; |
| my $pid = <IN> + 0; |
| close IN; |
| if ($pid < 2) { |
| die "invalid pid: '$pid'"; |
| } |
| kill (15, $pid) or die "kill $pid failed: $!"; |
| exit; |
| } |
| |
| # --------------------------------------------------------------------------- |
| |
| sub run; |
| my $am_parent; |
| my $child_pid = fork(); |
| if ($child_pid < 0) { |
| die "fork failed"; |
| } |
| elsif ($child_pid != 0) { |
| $am_parent = 0; |
| } |
| else { |
| $am_parent = 1; |
| $SIG{INT} = \&sigterm_handler; |
| $SIG{TERM} = \&sigterm_handler; |
| } |
| |
| if ($pidfile) { |
| open(OUT, ">$pidfile") or die "cannot write to $pidfile"; |
| print OUT $$; |
| close OUT or die "cannot save $pidfile"; |
| } |
| |
| sub logit { |
| print "LOG: ".join('', @_)." ($$) at ".(scalar localtime time)."\n"; |
| } |
| |
| # --------------------------------------------------------------------------- |
| |
| $| = 1; |
| logit "freqsd starting"; |
| my $is_first_time = 1; |
| |
| if ($am_parent) { |
| while (1) { |
| parent_loop(); |
| } |
| } |
| else { |
| while (1) { |
| child_loop(); |
| } |
| } |
| die "oops! cannot get here"; |
| |
| # --------------------------------------------------------------------------- |
| |
| sub parent_loop { |
| # I'm impatient. many times when I have to restart this script, I want to |
| # see "faster" report results built immediately, without the 10-minute wait. |
| # So first time around, just sleep for 5 secs, so that we get started almost |
| # immediately. |
| |
| my $this_sleep = $idle_sleep; |
| if ($is_first_time) { |
| $is_first_time = 0; |
| $this_sleep = 5; |
| } |
| |
| my $job = $dq_fast->wait_for_queued_job($this_sleep); |
| |
| if ($job && $job->{metadata}->{dir}) { |
| # if a dir was specified, it's always a "b" (buildbot) mass-check; |
| # that's the assumption here at least |
| logit "starting buildbot-requested faster reports"; |
| run_import_logs($parent_reports, "--tag=b --dir ".$job->{metadata}->{dir}); |
| make_reports($parent_reports, "--tag=b"); |
| } |
| else { |
| logit "starting rsync faster reports"; |
| run_import_logs($parent_reports, "--tag=n"); |
| make_reports($parent_reports, "--tag=n"); |
| # may also be weekly. no way to differentiate currently until |
| # AFTER corpus.hourly is run! TODO? |
| } |
| |
| # and the XML indices |
| run ("cd masses ; ./rule-qa/automc/gen_info_xml"); |
| |
| # and the ruleqa CGI page's cache |
| run("./masses/rule-qa/automc/ruleqa.cgi -refresh"); |
| |
| logit "completed faster reports"; |
| if ($job) { $job->finish(); } |
| } |
| |
| # --------------------------------------------------------------------------- |
| |
| sub child_loop { |
| my $job = $dq_fast->wait_for_queued_job($idle_sleep); |
| |
| # add switches |
| if ($job && $job->{metadata}->{dir}) { |
| logit "not running buildbot-requested slow reports; they're too slow!"; |
| # run_import_logs($child_reports, "--tag=b --dir ".$job->{metadata}->{dir}); |
| } |
| else { |
| # create slow reports |
| if (-f $INHIBIT_SLOW_REPORTS_FLAG_FILE) { |
| logit "inhibited rsync slow reports, $INHIBIT_SLOW_REPORTS_FLAG_FILE exists"; |
| } else { |
| logit "starting rsync slow reports"; |
| run_import_logs($child_reports, "--tag=n"); |
| make_reports($child_reports, "--tag=n"); |
| } |
| |
| # recreate the corpus link-farm |
| logit "running 'freqsd-infrequent' tasks"; |
| run ("build/automc/freqsd-infrequent"); |
| } |
| |
| logit "completed slow reports"; |
| |
| if ($job) { $job->finish(); } |
| } |
| |
| sub run_import_logs { |
| my ($reports, $opts) = @_; |
| run ("cd masses/rule-qa ; ./import-logs ". |
| "--override='output_classes=$reports' ". |
| "$opts"); |
| } |
| |
| sub make_reports { |
| my ($reports, $opts) = @_; |
| run ("cd masses/rule-qa ; ./reports-from-logs ". |
| "--override='output_classes=$reports' ". |
| "$opts"); |
| } |
| |
| # --------------------------------------------------------------------------- |
| |
| sub sigterm_handler { |
| warn "received SIGTERM at ".(scalar localtime time)."\n"; |
| kill(15, $child_pid); |
| if ($pidfile) { unlink($pidfile); } |
| die "terminated"; |
| } |
| |
| # --------------------------------------------------------------------------- |
| |
| sub run { |
| my ($cmd, $ignoreexit) = @_; |
| |
| print "[$cmd]\n"; |
| system ($cmd); |
| |
| if (!$ignoreexit) { |
| warn "command '$cmd' failed with status $?" if (($? >> 8) != 0); |
| } |
| } |
| |
| |