| #!/bin/bash |
| |
| # mkupdate-with-scores |
| # |
| # This script generates, tests, and publishes rule updates for stable release |
| # versions. It does the following: |
| # |
| # - retrieves the latest gernerated scores for new active.list rules |
| # - checks out the trunk revision of code that those scores were generated for |
| # - generates an update tarball and associated sha1, sha256, sha512, and asc files |
| # - checks out each of the 3.3 stable release tagged versions, builds and |
| # installs that version (in a tmp dir) and then installs the above generated |
| # update using sa-update --install to make sure it works with each version |
| # - if all goes well, it copies the update files to the update www directory, |
| # updates the dns zone files and schedules (using the at queue) an update |
| # of the zone soa and rndc reload using the tick_zone_serial script |
| # |
| # This script is similar to the run_part2 script used for trunk rule updates. |
| # |
| # Update May 19, 2011: this script now also takes a fourth parameter to do a |
| # reversion to an existing rule update. This is useful for releasing an |
| # emergency update to correct a bad update that was automatically (or |
| # otherwise) released. |
| # |
| # The script also takes three initial parameters that are used for testing |
| # purposes. The first is a root prefix for testing. The second is a keydir. |
| # The third is a flag to update the local svn co of this script's directory. |
| # |
| # If https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores/DISABLE-AUTOMATIC-UPDATES |
| # exists then DNS updates will be skipped so that update publishing is |
| # effectively disabled. Note that generated updates will still be visible |
| # on the mirrors but will not be published in DNS for sa-update clients. |
| # ALSO NOTE that this only applies to update generation. DNS *will* be |
| # update when the script is run with a fourth parameter to revert to an |
| # existing version/revision update. |
| |
| set -e |
| set -x |
| |
| umask 022 |
| |
| PROGDIR=`dirname $0` |
| [[ "$PROGDIR" = "." ]] && PROGDIR=`pwd` |
| PROGNAME=`basename $0 .sh` |
| HOST=`hostname -f` |
| |
| TMPDIR="/usr/local/spamassassin/automc/tmp/${PROGNAME}" |
| UPDATEDIR="/var/www/automc.spamassassin.org/updates" |
| KEYDIR="/usr/local/spamassassin/automc/key" |
| UPDATE_BUILD_DIR=0 |
| REVERT_REVISION=0 |
| |
| |
| # if $1 is present redirect output files to a test directory structure |
| if [ ${#1} -gt 1 ]; then |
| UPDATEDIR=$1$UPDATEDIR |
| DNSDIR=$1$DNSDIR |
| |
| # make the test directory structure |
| mkdir -p $UPDATEDIR |
| mkdir -p $DNSDIR |
| fi |
| |
| if [ ${#2} -gt 1 ]; then |
| KEYDIR=$2 |
| fi |
| |
| if [ ${#3} -gt 1 ]; then |
| UPDATE_BUILD_DIR=1 |
| fi |
| |
| if [ $4 ]; then |
| REVERT_REVISION=$4 |
| fi |
| |
| echo "UPDATEDIR=$UPDATEDIR" |
| echo "DNSDIR=$DNSDIR" |
| echo "KEYDIR=$KEYDIR" |
| echo "REVERT_REVISION=$REVERT_REVISION" |
| |
| test_version() { |
| SA_VERSION=$1 |
| SA_SVN_TAG=$2 |
| |
| # to heck with dealing with svn update failures |
| rm -rf release_$SA_VERSION |
| |
| # test the release on the version(s) of spamassassin the update is meant for |
| svn co https://svn.apache.org/repos/asf/spamassassin/$SA_SVN_TAG release_$SA_VERSION |
| |
| cd release_$SA_VERSION |
| |
| # fix for newer perl and <3.4.2: Unescaped left brace in regex is illegal here in regex; |
| # marked by <-- HERE in m/^(.{ <-- HERE ,200}).*$/ at lib/Mail/SpamAssassin/PerMsgStatus.pm line 921 |
| perl -p -i -e 's/\.\{,200/.{200/g' lib/Mail/SpamAssassin/PerMsgStatus.pm |
| # also do requires ./ |
| perl -p -i -e 's%= "version\.h\.pl"%= "./version.h.pl"%g' spamc/configure.pl |
| |
| # need rules directory or build won't work |
| if [ ! -e rules ]; then cp -a ../trunk/rules .; fi |
| |
| perl Makefile.PL PREFIX=$TMPDIR/release_$SA_VERSION < /dev/null |
| make |
| make install |
| |
| set +e |
| ./sa-update -D --install $TMPDIR/${REVISION}.tar.gz |
| STATUS=$? |
| set -e |
| |
| cd .. |
| rm -rf release_$SA_VERSION $TMPDIR/release_$SA_VERSION |
| return $STATUS |
| } |
| |
| make_rule_update_from_trunk() { |
| # to heck with dealing with svn update failures |
| rm -rf trunk trunk-rulesrc-scores |
| |
| # get the latest scores for new rules |
| svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores trunk-rulesrc-scores |
| |
| # get the revision number of the rules |
| # TODO: have the script that make 72_scores.cf include a revision number |
| #REVISION=`head -1 trunk-rulesrc-scores/72_scores.cf | cut -d" " -f6` |
| REVISION=`head -1 trunk-rulesrc-scores/scores-set* | cut -d" " -f9 | sort -rn | head -1` |
| |
| svn co --revision=$REVISION https://svn.apache.org/repos/asf/spamassassin/trunk trunk |
| |
| cd trunk |
| |
| if [ $UPDATE_BUILD_DIR ]; then |
| svn up build |
| fi |
| |
| perl Makefile.PL PREFIX=$TMPDIR/trunk < /dev/null |
| make |
| |
| cd .. |
| |
| cp trunk-rulesrc-scores/72_scores.cf trunk/rules/72_scores.cf |
| |
| # note: one of set0 or set1 stats might be incorrect (not all of their rules |
| # are included in the update) I can't remember if we eliminate dropped |
| # rules in generate-new-scores or not (we run the sets in a particular |
| # order for some reason) |
| cp trunk-rulesrc-scores/stats-set0 trunk/rules/STATISTICS-set0-72_scores.cf.txt |
| cp trunk-rulesrc-scores/stats-set1 trunk/rules/STATISTICS-set1-72_scores.cf.txt |
| cp trunk-rulesrc-scores/stats-set2 trunk/rules/STATISTICS-set2-72_scores.cf.txt |
| cp trunk-rulesrc-scores/stats-set3 trunk/rules/STATISTICS-set3-72_scores.cf.txt |
| |
| cd trunk/rules |
| |
| # remove files we don't want to ship in updates |
| # remember that 3KB == 1GB of traffic on the mirrors as of Jan 1, 2010 |
| set +e |
| rm -f 70_sandbox.cf 70_inactive.cf STATISTICS-set?.txt |
| set -e |
| |
| mkdir -p $TMPDIR/trunk/etc/mail/spamassassin |
| #cp *.pre *.cf *.txt languages user_prefs.template $TMPDIR/trunk/etc/mail/spamassassin/. |
| |
| ../spamassassin --lint -D |
| |
| tar cvf - *.cf *.txt languages user_prefs.template | gzip -9 > $TMPDIR/${REVISION}.tar.gz |
| |
| cd ../.. |
| |
| shasum -a 1 $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha1 |
| shasum -a 256 $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha256 |
| shasum -a 512 $TMPDIR/${REVISION}.tar.gz > $TMPDIR/${REVISION}.tar.gz.sha512 |
| gpg --batch --homedir $KEYDIR -bas $TMPDIR/${REVISION}.tar.gz || exit $? |
| |
| tar -ztvf $TMPDIR/${REVISION}.tar.gz |
| } |
| |
| copy_update_paranoid() { |
| SRC=$1 |
| DST=$2 |
| |
| set +e |
| cp $SRC $DST |
| diff -u $SRC $DST |
| if [ $? -ne 0 ]; then |
| set -e |
| return 1 |
| fi |
| set -e |
| return 0 |
| } |
| |
| copy_existing_update_for_reversion_testing() { |
| EXIT=0 |
| ( |
| copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz" "$TMPDIR/$REVISION.tar.gz" && |
| copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.asc" "$TMPDIR/$REVISION.tar.gz.asc" && |
| copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha1" "$TMPDIR/$REVISION.tar.gz.sha1" && |
| copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha256" "$TMPDIR/$REVISION.tar.gz.sha256" && |
| copy_update_paranoid "$UPDATEDIR/$REVERT_REVISION.tar.gz.sha512" "$TMPDIR/$REVISION.tar.gz.sha512" |
| ) || EXIT=6 |
| |
| # copying the update files went wrong exit |
| if [ $EXIT -gt 0 ]; then |
| echo "Could not copy existing revision $REVERT_REVISION, to temporary testing directory, aborting!" |
| exit $EXIT |
| fi |
| } |
| |
| check_for_disable-automatic-update_file_in_svn() { |
| rm -rf $TMPDIR/svn-scores-latest |
| # checkout the latest scores directory |
| svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores/ $TMPDIR/svn-scores-latest |
| |
| AUTOUPDATESDISABLED=0 |
| |
| if [ -f $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES ]; then |
| echo "Auto-updates have been previously disabled... continuing with manual update reversion" |
| AUTOUPDATESDISABLED=1 |
| fi |
| |
| return $AUTOUPDATESDISABLED |
| } |
| |
| disable_auto_update_publishing_and_get_new_update_revision_number() { |
| date > $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| echo "Automatic sa-update rule update publishing has been disabled via the revert-stable-update script." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| echo "Current stable update is being reverted to update $REVERT_REVISION." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| echo "Update version $REVERT_REVISION will be republished using this commit's revision number as the new version number." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| echo "To re-enable updates: publish the latest update using the 'revert-stable-update' script (this is optional)" >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| echo "and then delete this file from SVN to re-enable DNS publishing of generated updates." >> $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| |
| if [ $AUTOUPDATESDISABLED ]; then |
| svn add $TMPDIR/svn-scores-latest/DISABLE-AUTOMATIC-UPDATES |
| fi |
| |
| svn ci $TMPDIR/svn-scores-latest/* -m "sa-update auto-update disabled; reversion to version $REVERT_REVISION in progress; version $REVERT_REVISION will be republished as the same version number of this commit revision number" > $TMPDIR/NEW-REVERT-REVISION 2>&1 |
| |
| set +e |
| grep revision $TMPDIR/NEW-REVERT-REVISION |
| if [ $? -ne 0 ]; then |
| echo "Failed to obtain a new revision number to use as the new update version number." |
| exit 7 |
| fi |
| set -e |
| |
| REVISION=`cat $TMPDIR/NEW-REVERT-REVISION | grep revision | cut -d" " -f3 | cut -d "." -f1` |
| |
| echo "New update version/revision will be $REVISION" |
| } |
| |
| rm -rf $TMPDIR |
| mkdir $TMPDIR |
| cd $TMPDIR |
| |
| if [ $REVERT_REVISION -eq 0 ]; then |
| set +e |
| check_for_disable-automatic-update_file_in_svn |
| AUTOUPDATESDISABLED=$? |
| set -e |
| |
| # generate a rule update using rules from trunk at a revision |
| # that we have generated scores for |
| make_rule_update_from_trunk |
| else |
| if [ ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.asc -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.sha1 -a ! -f $UPDATEDIR/$REVERT_REVISION.tar.gz.sha256 -a ! $UPDATEDIR/$REVERT_REVISION.tar.gz.sha512 ]; then |
| echo "Could not find update files for update revision $REVERT_REVISION, aborting." |
| exit 8 |
| fi |
| set +e |
| check_for_disable-automatic-update_file_in_svn |
| AUTOUPDATESDISABLED=$? |
| set -e |
| |
| disable_auto_update_publishing_and_get_new_update_revision_number |
| echo "Copying existing version/revision $REVERT_REVISION to new version/revision $REVISION for testing." |
| copy_existing_update_for_reversion_testing |
| fi |
| |
| # test to make sure it works with sa-update --install |
| TESTED_VERSIONS="" |
| |
| MAJOR_VERS=3 |
| for (( MINOR_VERS=4; 1; MINOR_VERS++ )); do |
| for (( UPDATE_VERS=1; 1; UPDATE_VERS++ )); do |
| svn info https://svn.apache.org/repos/asf/spamassassin/tags/spamassassin_release_${MAJOR_VERS}_${MINOR_VERS}_${UPDATE_VERS} || break |
| test_version ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} tags/spamassassin_release_${MAJOR_VERS}_${MINOR_VERS}_${UPDATE_VERS} && |
| TESTED_VERSIONS="$TESTED_VERSIONS ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS}" |
| done |
| # we just assume that the next stable version is the branch's current version |
| set +e |
| svn info https://svn.apache.org/repos/asf/spamassassin/branches_${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} || break |
| test_version ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} branches/${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS} && |
| TESTED_VERSIONS="$TESTED_VERSIONS ${MAJOR_VERS}.${MINOR_VERS}.${UPDATE_VERS}" |
| set -e |
| done |
| |
| echo "VERSIONS UPDATE PASSED ON: $TESTED_VERSIONS" |
| |
| # publish update |
| if [[ ! -z "$TESTED_VERSIONS" ]]; then |
| ( |
| chmod 644 $TMPDIR/$REVISION.tar.gz* && |
| cp -a $TMPDIR/$REVISION.tar.gz* $UPDATEDIR/ |
| ) || exit 5 |
| |
| if [ $AUTOUPDATESDISABLED -eq 1 -a $REVERT_REVISION -eq 0 ]; then |
| echo "DNS updating disabled (auto update publishing disabled), skipping DNS reload" |
| else |
| # Wait 20 minutes for the mirrors to update via rsync |
| sleep 1200 |
| # Newer versions >= 3.4.1 of SpamAssassin are CNAME'd to 3.3.3 |
| /usr/local/bin/updateDNS.sh 3.3.3.updates TXT $REVISION |
| RC=$? |
| if [[ "$RC" -ne 2 ]]; then |
| # sa-update for these older versions doesn't support CNAME'ing |
| /usr/local/bin/updateDNS.sh 0.4.3.updates TXT $REVISION |
| /usr/local/bin/updateDNS.sh 2.3.3.updates TXT $REVISION |
| /usr/local/bin/updateDNS.sh 1.3.3.updates TXT $REVISION |
| /usr/local/bin/updateDNS.sh 0.3.3.updates TXT $REVISION |
| fi |
| fi |
| fi |