build/mkupdates/run_nightly

#!/bin/bash

PERL=/usr/bin/perl

LOGDIR="/var/www/automc.spamassassin.org/mkupdates"
REPDIR="/var/www/ruleqa.spamassassin.org/reports"
UPDDIR="/var/www/automc.spamassassin.org/updates"

VERSIONS="3.4.4"

cd /usr/local/spamassassin/automc/svn/trunk

. /etc/profile

# ---------------------------------------------------------------------------

promote_active_rules() {
  # should be in ~/svn/trunk
  pwd
 
  # Make sure we have the source, since listpromotable assumes we do. 
  # Also, make_tarball_for_version removes rulesrc
  svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc

  $PERL build/mkupdates/listpromotable > rules/active.list.new || exit $?
  mv rules/active.list.new rules/active.list

  svn diff rules > $REPDIR/LATEST

  cat $REPDIR/LATEST

  echo "Committing promotions in rules/active.list..."
  svn commit -m 'promotions validated' rules/active.list

  # create a list of "bad" rules in the current sandboxes, updated daily
  $PERL masses/rule-qa/list-bad-rules > $REPDIR/badrules.txt 2>&1

  #On Wednesday's, we send out a bad sandbox rules report to the list
  if [[ `date +%w` = 3 ]] ; then
    (
      echo "From: automc@sa-vm.apache.org (Rules Report Cron)"
      echo "Subject: [auto] bad sandbox rules report"
      echo
      cat $REPDIR/badrules.txt
    ) | /usr/sbin/sendmail -oi dev@spamassassin.apache.org
  fi
}

# ---------------------------------------------------------------------------

make_tarball_for_version() {
  version="$1"

  tmpdir=$HOME/tmp/stage/$version
  rm -rf $tmpdir; mkdir -p $tmpdir         || exit $?

  # extract the new rules files.

  # use "make install" logic, since we want rules as close as possible
  # to what's installed
  # TODO: this *would* be performed in a checkout of the desired
  # version's branch.  right now we're only using 1 version though
  make clean
  $PERL Makefile.PL PREFIX=$tmpdir < /dev/null || exit $?
  make                                     || exit $?

  # remove new features, unsupported in existing code in the field
  # (TODO: need a better way to exclude files that require new features
  # like this; judicious use of "ifplugin" helps)
  # rm rules/60_somerandomfeature.cf

  # ensure the basic lint/rule-sanity test suite passes for this ruleset
  # before we build an update from it.  useful particularly to catch
  # "tflags nopublish" leakage (bug 6297)
  make test \
    TEST_FILES="t/basic_lint.t t/basic_lint_without_sandbox.t t/basic_meta.t" \
    || exit $?

  # remove the rules files for rules we won't be shipping
  rm rules/70_sandbox.cf rules/70_inactive.cf

  # need to put the latest 72_scores.cf in the update.tgz
  svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc
  cp -a rulesrc/scores/72_scores.cf rules/
  # I think this is problematic. I don't see how it is needed HERE. 
  #rm -rf rulesrc

  # double check we still lint without those 2 files
  ./spamassassin --lint                   || exit $?

  rulesdir=`pwd`/rules

  (
    cd $rulesdir

    # Use this to include plugin .pm files:
    # tar cvf - *.cf *.pm                  || exit $?

    # or this, to ban code from the updates:
    tar cvf - *.cf                         || exit $?

  ) | gzip -9 > $tmpdir/update.tgz         || exit $?

  # ensure non-empty
  [ -s $tmpdir/update.tgz ] || exit 3

  linttmp=$tmpdir/lintdir
  rm -rf $linttmp
  mkdir $linttmp
  (
    cd $linttmp
    # check validity of tarball; also extract
    gunzip -cd < $tmpdir/update.tgz | tar xf - || exit $?
  )

  sitetmp=$tmpdir/sitetmp
  rm -rf $sitetmp
  mkdir $sitetmp
  cp rules/*.pre $sitetmp

  # now, ensure the ruleset (entirely as distributed) lints, also.
  # use "-p /dev/null" so any user_prefs data is ignored.
  ./spamassassin -x --configpath=$linttmp --siteconfigpath=$sitetmp \
               -p /dev/null --lint \
                || exit $?

  # sign and get sums
  gpg --batch --homedir $HOME/key \
        -bas $tmpdir/update.tgz                 || exit $?

  shasum -a 1  $tmpdir/update.tgz > $tmpdir/update.tgz.sha1  || exit $?
  shasum -a 256  $tmpdir/update.tgz > $tmpdir/update.tgz.sha256  || exit $?
  shasum -a 512  $tmpdir/update.tgz > $tmpdir/update.tgz.sha512  || exit $?


  # get SVN revision number.
  # note: use 'Last Changed Rev' instead of 'Revision'.  Because we share
  # an SVN repository with other projects, this means that the same
  # rev of *our* codebase may appear under multiple rev#s, as other projects
  # check their changes in.

  tagstamp=`date "+%Y%m%d%H%M%S"`
  tagurl=https://svn.apache.org/repos/asf/spamassassin/tags/sa-update_${version}_${tagstamp}

  # this svn copy is critical, to ensure each version's tarball has a different
  # rev#.  if you remove it, we need to prefix the version# to the svnrev# in
  # the filenames instead so each version doesn't clobber others.
  svn up
  svn copy -m 'promotions validated' . $tagurl < /dev/null

  # for svn 1.3:
  # (svn info --non-interactive $tagurl || svn info $tagurl ) < /dev/null \
                # > $tmpdir/svn 2>&1 || exit $?
  # for crappy zone svn, 1.2:
  (
    rm -rf tmpcheckout
    svn co $tagurl tmpcheckout && svn info tmpcheckout
    rm -rf tmpcheckout
  ) < /dev/null > $tmpdir/svn 2>&1 || exit $?

  svnrev=`(grep 'Last Changed Rev: ' $tmpdir/svn || exit 1) | \
        sed -e 's/^.*: //'`

  if [ "$svnrev" == "" ] ; then
    echo "missing SVN revision"
    cat $tmpdir/svn
    exit 5
  fi

  if [ "$svnrev" -lt 1 ] ; then
    echo "bad SVN revision: $svnrev"
    cat $tmpdir/svn
    exit 5
  fi

  chmod 644 $tmpdir/update.*

  # Integrate with masscheck ruleset updates to prevent duplicates
  RECENT=`find $HOME/tmp/mkupdate-with-scores -name \*.tar.gz -mmin -480`

  if [[ -z "$RECENT" ]]; then

    echo "Recent ruleset from mkupdate-with-scores (massheck) NOT found."
    echo "Proceeding with a ruleset publish..."

    mv $tmpdir/update.tgz      $UPDDIR/${svnrev}.tar.gz            || exit $?
    mv $tmpdir/update.tgz.sha1 $UPDDIR/${svnrev}.tar.gz.sha1       || exit $?
    mv $tmpdir/update.tgz.sha256 $UPDDIR/${svnrev}.tar.gz.sha256       || exit $?
    mv $tmpdir/update.tgz.sha512 $UPDDIR/${svnrev}.tar.gz.sha512       || exit $?
    mv $tmpdir/update.tgz.asc  $UPDDIR/${svnrev}.tar.gz.asc        || exit $?

    # Give the mirrors time to pull the new files above
    sleep 600

    # next, create/update the new DNS record....

    # Versions >= 3.4.1 are CNAMEd to this DNS record:
    /usr/local/bin/updateDNS.sh 3.3.3.updates TXT ${svnrev}
    RC=$?
    if [[ "$RC" -ne 2 ]]; then
      # Set older version TXT records for older sa-update
      /usr/local/bin/updateDNS.sh 0.4.3.updates TXT ${svnrev}
      /usr/local/bin/updateDNS.sh 2.3.3.updates TXT ${svnrev}
      /usr/local/bin/updateDNS.sh 1.3.3.updates TXT ${svnrev}
      /usr/local/bin/updateDNS.sh 0.3.3.updates TXT ${svnrev}
    fi

  else

    echo "Recent ruleset from mkupdate-with-scores (massheck) found:"
    ls -l $RECENT
    echo ""

  fi

  # clean up 4-day-old (and older) update tarballs.  This seems as
  # good a place as any to do this!
  # note: for manual updates, the file permissions should be 0444 so let's clean
  # out only 0644 (automatic) updates.  a bit of a kluge, but ...
  #find $UPDDIR -mtime +4 -perm 0644 -type f -name '*.tar.*' -delete

}

# ---------------------------------------------------------------------------

cycle_logfiles () {
  # cycle the logfiles; keep 6 (3 days worth I think)
  if [[ -e "$LOGDIR/mkupdates.txt" ]]; then
    X=6
    [[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && rm -f $LOGDIR/mkupdates_${X}.txt
    while [[ $X -gt 0 ]]; do
      ((X--))
      Y=$((X+1))
      [[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && mv -f $LOGDIR/mkupdates_${X}.txt $LOGDIR/mkupdates_${Y}.txt
    done
    mv -f $LOGDIR/mkupdates.txt $LOGDIR/mkupdates_${Y}.txt
  fi
}

# ---------------------------------------------------------------------------

[[ -d $UPDDIR ]] || echo "Updates dir '$UPDDIR' not found." 1>&2
[[ -d $UPDDIR ]] || exit 6

set -x

promote_active_rules

for VER in $VERSIONS; do
  make_tarball_for_version $VER
done

set +x

ls -l $UPDDIR/GPG.KEY
ls -l $UPDDIR/MIRRORED.BY
ls -ltr $UPDDIR/*.tar.* | tail -20

cycle_logfiles

rm -rf ruleqa.cache.* 

exit