build/mkupdates/mkupdate-with-scores

#!/bin/bash

# mkupdate-with-scores
#
# This script generates, tests, and publishes rule updates for stable release
# versions.  It does the following:
#
# - retrieves the latest gernerated scores for new active.list rules
# - checks out the trunk revision of code that those scores were generated for
# - generates an update tarball and associated sha1 and asc files
# - checks out each of the 3.3 stable release tagged versions, builds and
#   installs that version (in a tmp dir) and then installs the above generated
#   update using sa-update --install to make sure it works with each version
# - if all goes well, it copies the update files to the update www directory,
#   updates the dns zone files and schedules (using the at queue) an update
#   of the zone soa and rndc reload using the tick_zone_serial script
#
# This script is similar to the run_part2 script used for trunk rule updates.
#
# Update May 19, 2011: this script now also takes a fourth parameter to do a
# reversion to an existing rule update.  This is useful for releasing an
# emergency update to correct a bad update that was automatically (or
# otherwise) released.
#
# The script also takes three initial parameters that are used for testing
# purposes.  The first is a root prefix for testing.  The second is a keydir.
# The third is a flag to update the local svn co of this script's directory.
#
# If https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores/DISABLE-AUTOMATIC-UPDATES 
# exists then DNS updates will be skipped so that update publishing is
# effectively disabled.  Note that generated updates will still be visible
# on the mirrors but will not be published in DNS for sa-update clients.
# ALSO NOTE that this only applies to update generation.  DNS *will* be
# update when the script is run with a fourth parameter to revert to an
# existing version/revision update.

set -e
set -x

umask 022

UPDATEDIR=/var/www/buildbot.spamassassin.org/updatestage
DNSDIR=/var/named/updates.spamassassin.org.d
KEYDIR=/home/updatesd/key
UPDATE_BUILD_DIR=0
REVERT_REVISION=0

# if $1 is present redirect output files to a test directory structure
if [ ${#1} -gt 1 ]; then
  UPDATEDIR=$1$UPDATEDIR
  DNSDIR=$1$DNSDIR

  # make the test directory structure
  mkdir -p $UPDATEDIR
  mkdir -p $DNSDIR
fi

if [ ${#2} -gt 1 ]; then
  KEYDIR=$2
fi

if [ ${#3} -gt 1 ]; then
  UPDATE_BUILD_DIR=1
fi

if [ $4 ]; then
  REVERT_REVISION=$4
fi

echo "UPDATEDIR=$UPDATEDIR"
echo "DNSDIR=$DNSDIR"
echo "KEYDIR=$KEYDIR"
echo "REVERT_REVISION=$REVERT_REVISION"

test_version() {
  SA_VERSION=$1
  SA_SVN_TAG=$2

  # to heck with dealing with svn update failures
  rm -rf release_$SA_VERSION

  # test the release on the version(s) of spamassassin the update is meant for
  svn co https://svn.apache.org/repos/asf/spamassassin/$SA_SVN_TAG release_$SA_VERSION

  cd release_$SA_VERSION

  perl Makefile.PL PREFIX=$TMPDIR/release_$SA_VERSION < /dev/null
  make
  make install

  set +e
  ./sa-update -D --install $TMPDIR/${REVISION}.tar.gz
  STATUS=$?
  set -e

  cd ..
  rm -rf release_$SA_VERSION $TMPDIR/release_$SA_VERSION
  return $STATUS
}

update_dns_record() {
  SA_VERSION=$1
  UPDATE_REVISION=$2

  if [ $AUTOUPDATESDISABLED -eq 1 -a $REVERT_REVISION -eq 0 ]; then
    echo "DNS updating disabled (auto update publishing disabled), skipping DNS record update"
    return 0
  fi

  # turn "3.2.0" into "0.2.3"
  RVERS=`echo $SA_VERSION | perl -pe 's/^(\d+)\.(\d+)\.(\d+)$/$3.$2.$1/'`

  DNS_RECORD="$RVERS  TXT  \"$UPDATE_REVISION\""
  echo "DNS Record: $DNS_RECORD"

  DNSFILE="$DNSDIR/$SA_VERSION"
  
  mkdir $TMPDIR/dns-backup

  set +e
  cp $DNSFILE $TMPDIR/dns-backup/.
  set -e

  # set -e should catch any errors here
  echo $DNS_RECORD > $DNSFILE.mkupdate-with-scores.new
  mv $DNSFILE.mkupdate-with-scores.new $DNSFILE

  return 0
}

revert_dns_record() {
  SA_VERSION=$1

  if [ $AUTOUPDATESDISABLED -eq 1 -a $REVERT_REVISION -eq 0 ]; then
    echo "DNS updating disabled (auto update publishing disabled), skipping DNS record revert"
    return 0
  fi

  DNSFILE="$DNSDIR/$SA_VERSION"

  set +e
  cp $TMPDIR/dns-backup/$SA_VERSION $DNSFILE
  set -e
}

copy_update_paranoid() {
  SRC=$1
  DST=$2

  set +e
  cp $SRC $DST
  diff -u $SRC $DST
  if [ $? -ne 0 ]; then
    set -e
    return 1
  fi
  set -e
  return 0
}

make_rule_update_from_trunk() {
  # to heck with dealing with svn update failures
  rm -rf trunk trunk-rulesrc-scores

  # get the latest scores for new rules
  svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores trunk-rulesrc-scores

  # get the revision number of the rules
  # TODO: have the script that make 72_scores.cf include a revision number
  #REVISION=`head -1 trunk-rulesrc-scores/72_scores.cf | cut -d" " -f6`
  REVISION=`head -1 trunk-rulesrc-scores/scores-set* | cut -d" " -f9 | sort -rn | head -1`

  svn co --revision=$REVISION https://svn.apache.org/repos/asf/spamassassin/trunk trunk

  cd trunk

  if [ $UPDATE_BUILD_DIR ]; then
    svn up build
  fi

  perl Makefile.PL PREFIX=$TMPDIR/trunk < /dev/null
  make

  cd ..

  cp trunk-rulesrc-scores/72_scores.cf trunk/rules/72_scores.cf

  # note: one of set0 or set1 stats might be incorrect (not all of their rules
  #       are included in the update) I can't remember if we eliminate dropped
  #       rules in generate-new-scores or not (we run the sets in a particular
  #       order for some reason)
  cp trunk-rulesrc-scores/stats-set0 trunk/rules/STATISTICS-set0-72_scores.cf.txt
  cp trunk-rulesrc-scores/stats-set1 trunk/rules/STATISTICS-set1-72_scores.cf.txt
  cp trunk-rulesrc-scores/stats-set2 trunk/rules/STATISTICS-set2-72_scores.cf.txt
  cp trunk-rulesrc-scores/stats-set3 trunk/rules/STATISTICS-set3-72_scores.cf.txt

  cd trunk/rules

  # remove files we don't want to ship in updates
  # remember that 3KB == 1GB of traffic on the mirrors as of Jan 1, 2010
  set +e
  rm 70_sandbox.cf 70_inactive.cf
  rm STATISTICS-set?.txt
  set -e

  mkdir -p $TMPDIR/trunk/etc/mail/spamassassin
  #cp *.pre *.cf *.txt languages user_prefs.template $TMPDIR/trunk/etc/mail/spamassassin/.

  ../spamassassin --lint -D

  tar cvf - *.cf *.txt languages user_prefs.template | gzip -9 > $TMPDIR/${REVISION}.tar.gz

  cd ../..

  shasum $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha1
  gpg --batch --homedir $KEYDIR -bas $TMPDIR/${REVISION}.tar.gz || exit $?
}

copy_existing_update_for_reversion_testing() {
  EXIT=0
  (
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz"      "$TMPDIR/$REVISION.tar.gz" &&
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.asc"  "$TMPDIR/$REVISION.tar.gz.asc" &&
    copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha1" "$TMPDIR/$REVISION.tar.gz.sha1"
  ) || EXIT=6

  # copying the update files went wrong exit
  if [ $EXIT -gt 0 ]; then
    echo "Could not copy existing revision $REVERT_REVISION, to temporary testing directory, aborting!"
    exit $EXIT
  fi
}

check_for_disable-automatic-update_file_in_svn() {
  rm -rf $TMPDIR/svn-scores-latest
  # checkout the latest scores directory
  svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores/ $TMPDIR/svn-scores-latest

  AUTOUPDATESDISABLED=0

  if [ -f $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES ]; then
    echo "Auto-updates have been previously disabled... continuing with manual update reversion"
    AUTOUPDATESDISABLED=1
  fi

  return $AUTOUPDATESDISABLED
}

disable_auto_update_publishing_and_get_new_update_revision_number() {
  date > $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "Automatic sa-update rule update publishing has been disabled via the revert-stable-update script." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "Current stable update is being reverted to update $REVERT_REVISION." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "Update version $REVERT_REVISION will be republished using this commit's revision number as the new version number." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "To re-enable updates: publish the latest update using the 'revert-stable-update' script (this is optional)" >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  echo "and then delete this file from SVN to re-enable DNS publishing of generated updates." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES

  if [ $AUTOUPDATESDISABLED ]; then
    svn add $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES
  fi

  svn ci $TMPDIR/svn-scores-latest/* -m "sa-update auto-update disabled; reversion to version $REVERT_REVISION in progress; version $REVERT_REVISION will be republished as the same version number of this commit revision number" > $TMPDIR/NEW-REVERT-REVISION 2>&1

  set +e
  grep revision $TMPDIR/NEW-REVERT-REVISION
  if [ $? -ne 0 ]; then
    echo "Failed to obtain a new revision number to use as the new update version number."
    exit 7
  fi
  set -e

  REVISION=`cat $TMPDIR/NEW-REVERT-REVISION | grep revision | cut -d" " -f3 | cut -d "." -f1`

  echo "New update version/revision will be $REVISION"
}

TMPDIR="/tmp/sa-mkupdate-$$"
rm -rf $TMPDIR
mkdir $TMPDIR
cd $TMPDIR

if [ $REVERT_REVISION -eq 0 ]; then
  set +e
  check_for_disable-automatic-update_file_in_svn
  AUTOUPDATESDISABLED=$?
  set -e

  # generate a rule update using rules from trunk at a revision
  # that we have generated scores for
  make_rule_update_from_trunk
else
  if [ ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.asc -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.sha1 ]; then
    echo "Could not find update files for update revision $REVERT_REVISION, aborting."
    exit 8
  fi
  set +e
  check_for_disable-automatic-update_file_in_svn
  AUTOUPDATESDISABLED=$?
  set -e

  disable_auto_update_publishing_and_get_new_update_revision_number
  echo "Copying existing version/revision $REVERT_REVISION to new version/revision $REVISION for testing."
  copy_existing_update_for_reversion_testing
fi

# test to make sure it works with sa-update --install
UPDATED_VERSIONS=0
MINOR_VERS=0

for (( MINOR_VERS=0; 1; MINOR_VERS++ )); do
  set +e
  svn info https://svn.apache.org/repos/asf/spamassassin/tags/spamassassin_release_3_3_$MINOR_VERS | grep Revision
  VERSION_EXISTS=$?
  set -e
  if [ $VERSION_EXISTS -ne 0 ]; then
    break;
  fi
  test_version 3.3.$MINOR_VERS tags/spamassassin_release_3_3_$MINOR_VERS \
    && update_dns_record "3.3.$MINOR_VERS" "$REVISION" && UPDATED_VERSIONS=$((UPDATED_VERSIONS+1))
done

# we just assume that the next stable version is the branch's current version
test_version 3.3.$MINOR_VERS branches/3.3 \
  && update_dns_record "3.3.$MINOR_VERS" "$REVISION" && UPDATED_VERSIONS=$((UPDATED_VERSIONS+1))

echo "VERSIONS UPDATE PASSED ON: $UPDATED_VERSIONS"

# publish update
if [ $UPDATED_VERSIONS -gt 0 ]; then
  # be careful, we want to make sure that the DNS records are always in a good state;
  # even if we end up exiting unexpectedly due to an error
  EXIT=0
  (
    copy_update_paranoid "$TMPDIR/$REVISION.tar.gz"      "$UPDATEDIR/$REVISION.tar.gz" &&
    copy_update_paranoid "$TMPDIR/$REVISION.tar.gz.asc"  "$UPDATEDIR/$REVISION.tar.gz.asc" &&
    copy_update_paranoid "$TMPDIR/$REVISION.tar.gz.sha1" "$UPDATEDIR/$REVISION.tar.gz.sha1" &&
    chmod 544 $UPDATEDIR/$REVISION.tar.gz*
  ) || EXIT=5

  # copying the update files went wrong, revert dns and exit
  if [ $EXIT -gt 0 ]; then
    for (( I=0; I<=$MINOR_VERS; I++ )); do
      revert_dns_record "3.3.$I"
    done
    exit $EXIT
  fi

  # schedule dns commit/reload atq job
  # note that we're probably not going to be able to commit the DNS changes
  # until the update tarball mirrors have had time to sync since the ASF
  # name servers will probably update and reload via a commit hook trigger or
  # on a set interval

  if [ $AUTOUPDATESDISABLED -eq 1 -a $REVERT_REVISION -eq 0 ]; then
    echo "DNS updating disabled (auto update publishing disabled), skipping DNS reload"
  else
    # delete any existing jobs in at queue 'n' used for named reloads
    for JOB in `at -l -q n | cut -d" " -f1`; do atrm $JOB; done

    # schedule a job to tick the zone serial and reload named in 16 minutes
    # (mirror rsyncs are done every 15 minutes)
    cd
    echo "/export/home/updatesd/svn/spamassassin/build/mkupdates/tick_zone_serial" | at -q n now + 16min
  fi
fi

cd
rm -rf $TMPDIR