build/mkupdates/run_part2 - spamassassin - Git at Google

 #!/bin/bash
 #
 # Continue the updates-building process, post the user approval step.
 # Currently this is interactive as it requires a GPG passphrase entry.
 #
 # usage: sudo -u updatesd /home/updatesd/svn/spamassassin/build/mkupdates/run_part2
 #
 # required setup, in /etc/sudoers or /opt/sfw/etc/sudoers:
 #   updatesd     ALL = NOPASSWD: /usr/sbin/rndc reload

 set -x

 cd /home/updatesd/svn/spamassassin

 . /etc/profile
 #Switched to system perl on spamassassin-vm box
 PERL=/usr/bin/perl
 #PERL=/local/perl586/bin/perl

 # download stage, where update tarballs are deposited for downloaders
 #
 stagedir=/var/www/buildbot.spamassassin.org/updatestage

 # directory where "0.2.3" and other version-specific files live.
 # it's assumed that the *real* zone $INCLUDEs files from this dir.
 # it must be writable by the user this script runs as.
 #
 # dev, testing:
 # dnsdir=/var/named/updates.dev.spamassassin.org.d
 # live:
 # dnsdir=/var/named/updates.spamassassin.org.d
 #
 dnsdir=/var/named/updates.spamassassin.org.d

 # directory where "counter", "soa_line.tmpl", "soa_line" live.
 # it's assumed that the *real* zone $INCLUDEs files from this dir.
 # it must be writable by the user this script runs as.
 #
 soadir=/var/named/spamassassin.org.d

 versions="3.4.1"

 # ---------------------------------------------------------------------------
 # TODO: if/when this becomes fully automatic, this commit will be superfluous

 echo "Committing promotions in rules/active.list..."
 svn commit -m 'promotions validated' rules/active.list

 # ---------------------------------------------------------------------------

 make_tarball_for_version () {

 # to be honest, right now this is unused.
 version="$1"

 tmpdir=/home/updatesd/tmp/stage/$version
 rm -rf $tmpdir; mkdir -p $tmpdir         || exit $?


 # extract the new rules files.

 # use "make install" logic, since we want rules as close as possible
 # to what's installed
 # TODO: this *would* be performed in a checkout of the desired
 # version's branch.  right now we're only using 1 version though
 make clean
 $PERL Makefile.PL PREFIX=$tmpdir < /dev/null || exit $?
 make                                     || exit $?

 # remove new features, unsupported in existing code in the field
 # (TODO: need a better way to exclude files that require new features
 # like this; judicious use of "ifplugin" helps)
 # rm rules/60_somerandomfeature.cf

 # ensure the basic lint/rule-sanity test suite passes for this ruleset
 # before we build an update from it.  useful particularly to catch
 # "tflags nopublish" leakage (bug 6297)
 make test \
     TEST_FILES="t/basic_lint.t t/basic_lint_without_sandbox.t t/basic_meta.t" \
     || exit $?

 # remove the rules files for rules we won't be shipping
 rm rules/70_sandbox.cf rules/70_inactive.cf

 # double check we still lint without those 2 files
 ./spamassassin --lint                   || exit $?

 rulesdir=`pwd`/rules

 (
   cd $rulesdir

   # Use this to include plugin .pm files:
   # tar cvf - *.cf *.pm                  || exit $?

   # or this, to ban code from the updates:
   tar cvf - *.cf                         || exit $?

 ) | gzip -9 > $tmpdir/update.tgz         || exit $?

 # ensure non-empty
 [ -s $tmpdir/update.tgz ] || exit 3

 linttmp=$tmpdir/lintdir
 rm -rf $linttmp
 mkdir $linttmp
 (
   cd $linttmp
   # check validity of tarball; also extract
   gunzip -cd < $tmpdir/update.tgz | tar xf - || exit $?
 )

 sitetmp=$tmpdir/sitetmp
 rm -rf $sitetmp
 mkdir $sitetmp
 cp rules/*.pre $sitetmp

 # now, ensure the ruleset (entirely as distributed) lints, also.
 # use "-p /dev/null" so any user_prefs data is ignored.
 ./spamassassin -x --configpath=$linttmp --siteconfigpath=$sitetmp \
                -p /dev/null --lint \
                 || exit $?

 # sign and get sums
 gpg --batch --homedir /home/updatesd/key \
 	-bas $tmpdir/update.tgz  		|| exit $?

 $PERL build/sha1sum.pl $tmpdir/update.tgz > $tmpdir/update.tgz.sha1  || exit $?


 # get SVN revision number.
 # note: use 'Last Changed Rev' instead of 'Revision'.  Because we share
 # an SVN repository with other projects, this means that the same
 # rev of *our* codebase may appear under multiple rev#s, as other projects
 # check their changes in.

 tagstamp=`date "+%Y%m%d%H%M%S"`
 tagurl=https://svn.apache.org/repos/asf/spamassassin/tags/sa-update_${version}_${tagstamp}

 # this svn copy is critical, to ensure each version's tarball has a different
 # rev#.  if you remove it, we need to prefix the version# to the svnrev# in
 # the filenames instead so each version doesn't clobber others.
 svn up
 svn copy -m 'promotions validated' . $tagurl < /dev/null

 # for svn 1.3:
 # (svn info --non-interactive $tagurl || svn info $tagurl ) < /dev/null \
                 # > $tmpdir/svn 2>&1 || exit $?

 # for crappy zone svn, 1.2:
 (
   rm -rf tmpcheckout
   svn co $tagurl tmpcheckout && svn info tmpcheckout
   rm -rf tmpcheckout
 ) < /dev/null > $tmpdir/svn 2>&1 || exit $?

 svnrev=`(grep 'Last Changed Rev: ' $tmpdir/svn || exit 1) | \
         sed -e 's/^.*: //'`

 if [ "$svnrev" == "" ] ; then
   echo "missing SVN revision"
   cat $tmpdir/svn
   exit 5
 fi

 if [ "$svnrev" -lt 1 ] ; then
   echo "bad SVN revision: $svnrev"
   cat $tmpdir/svn
   exit 5
 fi

 mv $tmpdir/update.tgz      $stagedir/$svnrev.tar.gz            || exit $?
 mv $tmpdir/update.tgz.sha1 $stagedir/$svnrev.tar.gz.sha1       || exit $?
 mv $tmpdir/update.tgz.asc  $stagedir/$svnrev.tar.gz.asc        || exit $?

 chmod 644 $stagedir/$svnrev.*


 # next, create the new DNS record....

 # turn "3.2.0" into "0.2.3"
 rvers=`echo "$version" | perl -pe 's/^(\d+)\.(\d+)\.(\d+)$/$3.$2.$1/'`

 dnsfile="$dnsdir/$version"
 if echo "
 $rvers	TXT	\"$svnrev\"
 " > $dnsfile.new
 then
   mv $dnsfile.new $dnsfile || exit $?
 else
   echo "failed to create $dnsfile.new" 1>&2 ; exit 1
 fi

 # increment the zone serial.
 ./build/mkupdates/tick_zone_serial || exit $?


 # clean up 4-day-old (and older) update tarballs.  This seems as
 # good a place as any to do this!
 # note: for manual updates, the file permissions should be 0444 so let's clean
 # out only 0644 (automatic) updates.  a bit of a kluge, but ...
 find $stagedir -mtime +4 -perm 0644 -type f -name '*.tar.*' | xargs rm

 }

 # ---------------------------------------------------------------------------

 cycle_logfiles () {
 # cycle the logfiles; keep 6 (3 days worth I think)
 (
   cd /var/www/buildbot.spamassassin.org/updatesd
   rm mkupdatespt2_6.txt
   mv mkupdatespt2_5.txt  mkupdatespt2_6.txt
   mv mkupdatespt2_4.txt  mkupdatespt2_5.txt
   mv mkupdatespt2_3.txt  mkupdatespt2_4.txt
   mv mkupdatespt2_2.txt  mkupdatespt2_3.txt
   mv mkupdatespt2_1.txt  mkupdatespt2_2.txt
   mv mkupdatespt2.txt    mkupdatespt2_1.txt
 )
 }

 # ---------------------------------------------------------------------------

 [ -d $stagedir ] || echo "no stagedir" 1>&2
 [ -d $stagedir ] || exit 6

 for version in $versions ; do
   make_tarball_for_version $version
 done

 ls -l $stagedir
 cycle_logfiles
 exit
	#!/bin/bash
	#
	# Continue the updates-building process, post the user approval step.
	# Currently this is interactive as it requires a GPG passphrase entry.
	#
	# usage: sudo -u updatesd /home/updatesd/svn/spamassassin/build/mkupdates/run_part2
	#
	# required setup, in /etc/sudoers or /opt/sfw/etc/sudoers:
	# updatesd ALL = NOPASSWD: /usr/sbin/rndc reload

	set -x

	cd /home/updatesd/svn/spamassassin

	. /etc/profile
	#Switched to system perl on spamassassin-vm box
	PERL=/usr/bin/perl
	#PERL=/local/perl586/bin/perl

	# download stage, where update tarballs are deposited for downloaders
	#
	stagedir=/var/www/buildbot.spamassassin.org/updatestage

	# directory where "0.2.3" and other version-specific files live.
	# it's assumed that the real zone $INCLUDEs files from this dir.
	# it must be writable by the user this script runs as.
	#
	# dev, testing:
	# dnsdir=/var/named/updates.dev.spamassassin.org.d
	# live:
	# dnsdir=/var/named/updates.spamassassin.org.d
	#
	dnsdir=/var/named/updates.spamassassin.org.d

	# directory where "counter", "soa_line.tmpl", "soa_line" live.
	# it's assumed that the real zone $INCLUDEs files from this dir.
	# it must be writable by the user this script runs as.
	#
	soadir=/var/named/spamassassin.org.d

	versions="3.4.1"

	# ---------------------------------------------------------------------------
	# TODO: if/when this becomes fully automatic, this commit will be superfluous

	echo "Committing promotions in rules/active.list..."
	svn commit -m 'promotions validated' rules/active.list

	# ---------------------------------------------------------------------------

	make_tarball_for_version () {

	# to be honest, right now this is unused.
	version="$1"

	tmpdir=/home/updatesd/tmp/stage/$version
	rm -rf $tmpdir; mkdir -p $tmpdir \|\| exit $?



	# extract the new rules files.

	# use "make install" logic, since we want rules as close as possible
	# to what's installed
	# TODO: this would be performed in a checkout of the desired
	# version's branch. right now we're only using 1 version though
	make clean
	$PERL Makefile.PL PREFIX=$tmpdir < /dev/null \|\| exit $?
	make \|\| exit $?

	# remove new features, unsupported in existing code in the field
	# (TODO: need a better way to exclude files that require new features
	# like this; judicious use of "ifplugin" helps)
	# rm rules/60_somerandomfeature.cf

	# ensure the basic lint/rule-sanity test suite passes for this ruleset
	# before we build an update from it. useful particularly to catch
	# "tflags nopublish" leakage (bug 6297)
	make test \
	TEST_FILES="t/basic_lint.t t/basic_lint_without_sandbox.t t/basic_meta.t" \
	\|\| exit $?

	# remove the rules files for rules we won't be shipping
	rm rules/70_sandbox.cf rules/70_inactive.cf

	# double check we still lint without those 2 files
	./spamassassin --lint \|\| exit $?

	rulesdir=`pwd`/rules

	(
	cd $rulesdir

	# Use this to include plugin .pm files:
	# tar cvf - .cf .pm \|\| exit $?

	# or this, to ban code from the updates:
	tar cvf - *.cf \|\| exit $?

	) \| gzip -9 > $tmpdir/update.tgz \|\| exit $?

	# ensure non-empty
	[ -s $tmpdir/update.tgz ] \|\| exit 3

	linttmp=$tmpdir/lintdir
	rm -rf $linttmp
	mkdir $linttmp
	(
	cd $linttmp
	# check validity of tarball; also extract
	gunzip -cd < $tmpdir/update.tgz \| tar xf - \|\| exit $?
	)

	sitetmp=$tmpdir/sitetmp
	rm -rf $sitetmp
	mkdir $sitetmp
	cp rules/*.pre $sitetmp

	# now, ensure the ruleset (entirely as distributed) lints, also.
	# use "-p /dev/null" so any user_prefs data is ignored.
	./spamassassin -x --configpath=$linttmp --siteconfigpath=$sitetmp \
	-p /dev/null --lint \
	\|\| exit $?

	# sign and get sums
	gpg --batch --homedir /home/updatesd/key \
	-bas $tmpdir/update.tgz \|\| exit $?

	$PERL build/sha1sum.pl $tmpdir/update.tgz > $tmpdir/update.tgz.sha1 \|\| exit $?



	# get SVN revision number.
	# note: use 'Last Changed Rev' instead of 'Revision'. Because we share
	# an SVN repository with other projects, this means that the same
	# rev of our codebase may appear under multiple rev#s, as other projects
	# check their changes in.

	tagstamp=`date "+%Y%m%d%H%M%S"`
	tagurl=https://svn.apache.org/repos/asf/spamassassin/tags/sa-update_${version}_${tagstamp}

	# this svn copy is critical, to ensure each version's tarball has a different
	# rev#. if you remove it, we need to prefix the version# to the svnrev# in
	# the filenames instead so each version doesn't clobber others.
	svn up
	svn copy -m 'promotions validated' . $tagurl < /dev/null

	# for svn 1.3:
	# (svn info --non-interactive $tagurl \|\| svn info $tagurl ) < /dev/null \
	# > $tmpdir/svn 2>&1 \|\| exit $?

	# for crappy zone svn, 1.2:
	(
	rm -rf tmpcheckout
	svn co $tagurl tmpcheckout && svn info tmpcheckout
	rm -rf tmpcheckout
	) < /dev/null > $tmpdir/svn 2>&1 \|\| exit $?

	svnrev=`(grep 'Last Changed Rev: ' $tmpdir/svn \|\| exit 1) \| \
	sed -e 's/^.*: //'`

	if [ "$svnrev" == "" ] ; then
	echo "missing SVN revision"
	cat $tmpdir/svn
	exit 5
	fi

	if [ "$svnrev" -lt 1 ] ; then
	echo "bad SVN revision: $svnrev"
	cat $tmpdir/svn
	exit 5
	fi

	mv $tmpdir/update.tgz $stagedir/$svnrev.tar.gz \|\| exit $?
	mv $tmpdir/update.tgz.sha1 $stagedir/$svnrev.tar.gz.sha1 \|\| exit $?
	mv $tmpdir/update.tgz.asc $stagedir/$svnrev.tar.gz.asc \|\| exit $?

	chmod 644 $stagedir/$svnrev.*



	# next, create the new DNS record....

	# turn "3.2.0" into "0.2.3"
	rvers=`echo "$version" \| perl -pe 's/^(\d+)\.(\d+)\.(\d+)$/$3.$2.$1/'`

	dnsfile="$dnsdir/$version"
	if echo "
	$rvers TXT \"$svnrev\"
	" > $dnsfile.new
	then
	mv $dnsfile.new $dnsfile \|\| exit $?
	else
	echo "failed to create $dnsfile.new" 1>&2 ; exit 1
	fi

	# increment the zone serial.
	./build/mkupdates/tick_zone_serial \|\| exit $?


	# clean up 4-day-old (and older) update tarballs. This seems as
	# good a place as any to do this!
	# note: for manual updates, the file permissions should be 0444 so let's clean
	# out only 0644 (automatic) updates. a bit of a kluge, but ...
	find $stagedir -mtime +4 -perm 0644 -type f -name '.tar.' \| xargs rm

	}

	# ---------------------------------------------------------------------------

	cycle_logfiles () {
	# cycle the logfiles; keep 6 (3 days worth I think)
	(
	cd /var/www/buildbot.spamassassin.org/updatesd
	rm mkupdatespt2_6.txt
	mv mkupdatespt2_5.txt mkupdatespt2_6.txt
	mv mkupdatespt2_4.txt mkupdatespt2_5.txt
	mv mkupdatespt2_3.txt mkupdatespt2_4.txt
	mv mkupdatespt2_2.txt mkupdatespt2_3.txt
	mv mkupdatespt2_1.txt mkupdatespt2_2.txt
	mv mkupdatespt2.txt mkupdatespt2_1.txt
	)
	}

	# ---------------------------------------------------------------------------

	[ -d $stagedir ] \|\| echo "no stagedir" 1>&2
	[ -d $stagedir ] \|\| exit 6

	for version in $versions ; do
	make_tarball_for_version $version
	done

	ls -l $stagedir
	cycle_logfiles
	exit