build/mkupdates/mkupdate-with-scores

#!/bin/bash

# mkupdate-with-scores
#
# This script generates, tests, and publishes rule updates for stable release
# versions.  It does the following:
#
# - retrieves the latest gernerated scores for new active.list rules
# - checks out the trunk revision of code that those scores were generated for
# - generates an update tarball and associated sha1, sha256, sha512, and asc files
# - checks out each of the 3.3 stable release tagged versions, builds and
#   installs that version (in a tmp dir) and then installs the above generated
#   update using sa-update --install to make sure it works with each version
# - if all goes well, it copies the update files to the update www directory,
#   updates the dns zone files and schedules (using the at queue) an update
#   of the zone soa and rndc reload using the tick_zone_serial script
#
# This script is similar to the run_part2 script used for trunk rule updates.
#
# Update May 19, 2011: this script now also takes a fourth parameter to do a
# reversion to an existing rule update.  This is useful for releasing an
# emergency update to correct a bad update that was automatically (or
# otherwise) released.
#
# The script also takes three initial parameters that are used for testing
# purposes.  The first is a root prefix for testing.  The second is a keydir.
# The third is a flag to update the local svn co of this script's directory.
#
# If https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores/DISABLE-AUTOMATIC-UPDATES 
# exists then DNS updates will be skipped so that update publishing is
# effectively disabled.  Note that generated updates will still be visible
# on the mirrors but will not be published in DNS for sa-update clients.
# ALSO NOTE that this only applies to update generation.  DNS *will* be
# update when the script is run with a fourth parameter to revert to an
# existing version/revision update.

set -e
set -x

umask 022

PROGDIR=`dirname $0`
[[ "$PROGDIR" = "." ]] && PROGDIR=`pwd`
PROGNAME=`basename $0 .sh`
HOST=`hostname -f`

TMPDIR="/usr/local/spamassassin/automc/tmp/${PROGNAME}"
UPDATEDIR="/var/www/automc.spamassassin.org/updates"
KEYDIR="/usr/local/spamassassin/automc/key"
UPDATE_BUILD_DIR=0
REVERT_REVISION=0


# if $1 is present redirect output files to a test directory structure
if [ ${#1} -gt 1 ]; then
  UPDATEDIR=$1$UPDATEDIR
  DNSDIR=$1$DNSDIR

  # make the test directory structure
  mkdir -p $UPDATEDIR
  mkdir -p $DNSDIR
fi

if [ ${#2} -gt 1 ]; then
  KEYDIR=$2
fi

if [ ${#3} -gt 1 ]; then
  UPDATE_BUILD_DIR=1
fi

if [ $4 ]; then
  REVERT_REVISION=$4
fi

echo "UPDATEDIR=$UPDATEDIR"
echo "DNSDIR=$DNSDIR"
echo "KEYDIR=$KEYDIR"
echo "REVERT_REVISION=$REVERT_REVISION"

test_version() {
  SA_VERSION=$1
  SA_SVN_TAG=$2

  # to heck with dealing with svn update failures
  rm -rf release_$SA_VERSION

  # test the release on the version(s) of spamassassin the update is meant for
  svn co https://svn.apache.org/repos/asf/spamassassin/$SA_SVN_TAG release_$SA_VERSION

  cd release_$SA_VERSION

  # fix for newer perl and <3.4.2: Unescaped left brace in regex is illegal here in regex;
  #   marked by <-- HERE in m/^(.{ <-- HERE ,200}).*$/ at lib/Mail/SpamAssassin/PerMsgStatus.pm line 921
  perl -p -i -e 's/\.\{,200/.{200/g' lib/Mail/SpamAssassin/PerMsgStatus.pm
  # also do requires ./
  perl -p -i -e 's%= "version\.h\.pl"%= "./version.h.pl"%g' spamc/configure.pl

  # need rules directory or build won't work
  if [ ! -e rules ]; then cp -a ../trunk/rules .; fi

  perl Makefile.PL PREFIX=$TMPDIR/release_$SA_VERSION < /dev/null
  make
  make install

  set +e
  ./sa-update -D --install $TMPDIR/${REVISION}.tar.gz
  STATUS=$?
  set -e

  cd ..
  rm -rf release_$SA_VERSION $TMPDIR/release_$SA_VERSION
  return $STATUS
}

make_rule_update_from_trunk() {
  # to heck with dealing with svn update failures
  rm -rf trunk trunk-rulesrc-scores

  # get the latest scores for new rules
  svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores trunk-rulesrc-scores

  # get the revision number of the rules
  # TODO: have the script that make 72_scores.cf include a revision number
  #REVISION=`head -1 trunk-rulesrc-scores/72_scores.cf | cut -d" " -f6`
  REVISION=`head -1 trunk-rulesrc-scores/scores-set* | cut -d" " -f9 | sort -rn | head -1`

  svn co --revision=$REVISION https://svn.apache.org/repos/asf/spamassassin/trunk trunk

  cd trunk

  if [ $UPDATE_BUILD_DIR ]; then
    svn up build
  fi

  perl Makefile.PL PREFIX=$TMPDIR/trunk < /dev/null
  make

  cd ..

  cp trunk-rulesrc-scores/72_scores.cf trunk/rules/72_scores.cf

  # note: one of set0 or set1 stats might be incorrect (not all of their rules
  #       are included in the update) I can't remember if we eliminate dropped
  #       rules in generate-new-scores or not (we run the sets in a particular
  #       order for some reason)
  cp trunk-rulesrc-scores/stats-set0 trunk/rules/STATISTICS-set0-72_scores.cf.txt
  cp trunk-rulesrc-scores/stats-set1 trunk/rules/STATISTICS-set1-72_scores.cf.txt
  cp trunk-rulesrc-scores/stats-set2 trunk/rules/STATISTICS-set2-72_scores.cf.txt
  cp trunk-rulesrc-scores/stats-set3 trunk/rules/STATISTICS-set3-72_scores.cf.txt

  cd trunk/rules

  # remove files we don't want to ship in updates
  # remember that 3KB == 1GB of traffic on the mirrors as of Jan 1, 2010
  set +e
  rm -f 70_sandbox.cf 70_inactive.cf STATISTICS-set?.txt
  set -e

  mkdir -p $TMPDIR/trunk/etc/mail/spamassassin
  #cp *.pre *.cf *.txt languages user_prefs.template $TMPDIR/trunk/etc/mail/spamassassin/.

  ../spamassassin --lint -D

  tar cvf - *.cf *.txt languages user_prefs.template | gzip -9 > $TMPDIR/${REVISION}.tar.gz

  cd ../..

  shasum  -a 1 $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha1
  shasum  -a 256 $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha256
  shasum  -a 512 $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha512
  gpg --batch --homedir $KEYDIR -bas $TMPDIR/${REVISION}.tar.gz || exit $?

  tar -ztvf $TMPDIR/${REVISION}.tar.gz
}

copy_update_paranoid() {
  SRC=$1
  DST=$2

  set +e
  cp $SRC $DST
  diff -u $SRC $DST
  if [ $? -ne 0 ]; then
    set -e
    return 1
  fi
  set -e
  return 0
}

copy_existing_update_for_reversion_testing() {
  EXIT=0
  (
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz"      "$TMPDIR/$REVISION.tar.gz" &&
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.asc"  "$TMPDIR/$REVISION.tar.gz.asc" &&
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha1" "$TMPDIR/$REVISION.tar.gz.sha1" &&
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha256" "$TMPDIR/$REVISION.tar.gz.sha256" &&
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha512" "$TMPDIR/$REVISION.tar.gz.sha512"
  ) || EXIT=6

  # copying the update files went wrong exit
  if [ $EXIT -gt 0 ]; then
    echo "Could not copy existing revision $REVERT_REVISION, to temporary testing directory, aborting!"
    exit $EXIT
  fi
}

check_for_disable-automatic-update_file_in_svn() {
  rm -rf $TMPDIR/svn-scores-latest
  # checkout the latest scores directory
  svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores/ $TMPDIR/svn-scores-latest

  AUTOUPDATESDISABLED=0

  if [ -f $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES ]; then
    echo "Auto-updates have been previously disabled... continuing with manual update reversion"
    AUTOUPDATESDISABLED=1
  fi

  return $AUTOUPDATESDISABLED
}

disable_auto_update_publishing_and_get_new_update_revision_number() {
  date > $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "Automatic sa-update rule update publishing has been disabled via the revert-stable-update script." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "Current stable update is being reverted to update $REVERT_REVISION." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "Update version $REVERT_REVISION will be republished using this commit's revision number as the new version number." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "To re-enable updates: publish the latest update using the 'revert-stable-update' script (this is optional)" >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "and then delete this file from SVN to re-enable DNS publishing of generated updates." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES

  if [ $AUTOUPDATESDISABLED ]; then
    svn add $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  fi

  svn ci $TMPDIR/svn-scores-latest/* -m "sa-update auto-update disabled; reversion to version $REVERT_REVISION in progress; version $REVERT_REVISION will be republished as the same version number of this commit revision number" > $TMPDIR/NEW-REVERT-REVISION 2>&1

  set +e
  grep revision $TMPDIR/NEW-REVERT-REVISION
  if [ $? -ne 0 ]; then
    echo "Failed to obtain a new revision number to use as the new update version number."
    exit 7
  fi
  set -e

  REVISION=`cat $TMPDIR/NEW-REVERT-REVISION | grep revision | cut -d" " -f3 | cut -d "." -f1`

  echo "New update version/revision will be $REVISION"
}

rm -rf $TMPDIR
mkdir $TMPDIR
cd $TMPDIR

if [ $REVERT_REVISION -eq 0 ]; then
  set +e
  check_for_disable-automatic-update_file_in_svn
  AUTOUPDATESDISABLED=$?
  set -e

  # generate a rule update using rules from trunk at a revision
  # that we have generated scores for
  make_rule_update_from_trunk
else
  if [ ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.asc -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.sha1 -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.sha256 -a ! $UPDATEDIR/$REVERT_REVISION.tar.gz.sha512 ]; then
    echo "Could not find update files for update revision $REVERT_REVISION, aborting."
    exit 8
  fi
  set +e
  check_for_disable-automatic-update_file_in_svn
  AUTOUPDATESDISABLED=$?
  set -e

  disable_auto_update_publishing_and_get_new_update_revision_number
  echo "Copying existing version/revision $REVERT_REVISION to new version/revision $REVISION for testing."
  copy_existing_update_for_reversion_testing
fi

# test to make sure it works with sa-update --install
TESTED_VERSIONS=""

MAJOR_VERS=3
for (( MINOR_VERS=4; 1; MINOR_VERS++ )); do
  for (( UPDATE_VERS=1; 1; UPDATE_VERS++ )); do
    svn info https://svn.apache.org/repos/asf/spamassassin/tags/spamassassin_release_${MAJOR_VERS}_${MINOR_VERS}_${UPDATE_VERS} || break
    test_version ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} tags/spamassassin_release_${MAJOR_VERS}_${MINOR_VERS}_${UPDATE_VERS} &&
      TESTED_VERSIONS="$TESTED_VERSIONS ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS}"
  done
  # we just assume that the next stable version is the branch's current version
  set +e
  svn info https://svn.apache.org/repos/asf/spamassassin/branches_${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} || break 
  test_version ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} branches/${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} &&
    TESTED_VERSIONS="$TESTED_VERSIONS ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS}"
  set -e
done

echo "VERSIONS UPDATE PASSED ON: $TESTED_VERSIONS"

# publish update
if [[ ! -z "$TESTED_VERSIONS" ]]; then
  (
    chmod 644 $TMPDIR/$REVISION.tar.gz* &&
    cp -a $TMPDIR/$REVISION.tar.gz* $UPDATEDIR/
  ) || exit 5

  if [ $AUTOUPDATESDISABLED -eq 1 -a $REVERT_REVISION -eq 0 ]; then
    echo "DNS updating disabled (auto update publishing disabled), skipping DNS reload"
  else
    # Wait 20 minutes for the mirrors to update via rsync
    sleep 1200
    # Newer versions >= 3.4.1 of SpamAssassin are CNAME'd to 3.3.3
    /usr/local/bin/updateDNS.sh 3.3.3.updates TXT $REVISION
    RC=$?
    if [[ "$RC" -ne 2 ]]; then
      # sa-update for these older versions doesn't support CNAME'ing
      /usr/local/bin/updateDNS.sh 0.4.3.updates TXT $REVISION
      /usr/local/bin/updateDNS.sh 2.3.3.updates TXT $REVISION
      /usr/local/bin/updateDNS.sh 1.3.3.updates TXT $REVISION
      /usr/local/bin/updateDNS.sh 0.3.3.updates TXT $REVISION
    fi
  fi
fi