spamassassin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1796117 - in /spamassassin/trunk/build/mkupdates: run_nightly run_part2 tick_zone_serial
Date Wed, 24 May 2017 23:36:19 GMT
Author: davej
Date: Wed May 24 23:36:19 2017
New Revision: 1796117

URL: http://svn.apache.org/viewvc?rev=1796117&view=rev
Log:
Updated nightly run scripts for new pathing after server rebuild.

Removed:
    spamassassin/trunk/build/mkupdates/tick_zone_serial
Modified:
    spamassassin/trunk/build/mkupdates/run_nightly
    spamassassin/trunk/build/mkupdates/run_part2

Modified: spamassassin/trunk/build/mkupdates/run_nightly
URL: http://svn.apache.org/viewvc/spamassassin/trunk/build/mkupdates/run_nightly?rev=1796117&r1=1796116&r2=1796117&view=diff
==============================================================================
--- spamassassin/trunk/build/mkupdates/run_nightly (original)
+++ spamassassin/trunk/build/mkupdates/run_nightly Wed May 24 23:36:19 2017
@@ -1,11 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 #
 # driver for cron.
-# 30 8 * * * bash /home/updatesd/svn/spamassassin/build/mkupdates/run_nightly > /var/www/buildbot.spamassassin.org/updatesd/mkupdates.txt
2>&1
+# 30 8 * * * automc bash /usr/local/spamassassin/svn/trunk/build/mkupdates/run_nightly >
/var/www/bbmass.spamassassin.org/mkupdates/mkupdates.txt 2>&1
 
 set -x
 
-cd /home/updatesd/svn/spamassassin
+LOGDIR="/var/www/bbmass.spamassassin.org/mkupdates"
+
+cd /usr/local/spamassassin/svn/trunk/
 
 . /etc/profile
 #Switched to system perl on spamassassin-vm box
@@ -39,29 +41,27 @@ echo "
 
 To commit this and proceed with update tarball creation:
 
-    ssh spamassassin.zones.apache.org
-    sudo -u updatesd /home/updatesd/svn/spamassassin/build/mkupdates/run_part2
+    ssh sa-vm1.apache.org
+    sudo -u automc /usr/local/spamassassin/svn/trunk/build/mkupdates/run_part2
 
 "
 
 svn diff
-) > /home/updatesd/LATEST
+) > $HOME/LATEST
 
-cat /home/updatesd/LATEST
+cat $HOME/LATEST
 # mailx -s "[admin] planned commit for active.list" \
 #              dev@Spamassassin.apache.org < /home/updatesd/LATEST
 
 # cycle the logfiles; keep 6 (3 days worth I think)
-(
-  cd /var/www/buildbot.spamassassin.org/updatesd
-  rm mkupdates_6.txt
-  mv mkupdates_5.txt  mkupdates_6.txt
-  mv mkupdates_4.txt  mkupdates_5.txt
-  mv mkupdates_3.txt  mkupdates_4.txt
-  mv mkupdates_2.txt  mkupdates_3.txt
-  mv mkupdates_1.txt  mkupdates_2.txt
-  mv mkupdates.txt    mkupdates_1.txt
-)
+X=6
+[[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && rm -f $LOGDIR/mkupdates_${X}.txt
+while [[ $X -gt 0 ]]; do
+  ((X--))
+  Y=$((X+1))
+  [[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && mv -f $LOGDIR/mkupdates_${X}.txt $LOGDIR/mkupdates_${Y}.txt
+done
+mv -f $LOGDIR/mkupdates.txt $LOGDIR/mkupdates_${Y}.txt
 
 # create a list of "bad" rules in the current sandboxes, updated daily
 $PERL masses/rule-qa/list-bad-rules \
@@ -70,10 +70,9 @@ $PERL masses/rule-qa/list-bad-rules \
 #On Wednesday's, we send out a bad sandbox rules report to the list
 if [ `date +%w` = 3 ] ; then
   (
-    echo "From: nobody@spamassassin.apache.org (Rules Report Cron)"
+    echo "From: noreply@sa-vm1.apache.org (Rules Report Cron)"
     echo "Subject: [auto] bad sandbox rules report"
     echo
     cat /var/www/ruleqa.spamassassin.org/reports/badrules.txt
   ) | /usr/sbin/sendmail -oi dev@spamassassin.apache.org
 fi
-

Modified: spamassassin/trunk/build/mkupdates/run_part2
URL: http://svn.apache.org/viewvc/spamassassin/trunk/build/mkupdates/run_part2?rev=1796117&r1=1796116&r2=1796117&view=diff
==============================================================================
--- spamassassin/trunk/build/mkupdates/run_part2 (original)
+++ spamassassin/trunk/build/mkupdates/run_part2 Wed May 24 23:36:19 2017
@@ -3,14 +3,26 @@
 # Continue the updates-building process, post the user approval step.
 # Currently this is interactive as it requires a GPG passphrase entry.
 #
-# usage: sudo -u updatesd /home/updatesd/svn/spamassassin/build/mkupdates/run_part2
+# usage: sudo -u automc /usr/local/spamassassin/svn/trunk/build/mkupdates/run_part2
 #
-# required setup, in /etc/sudoers or /opt/sfw/etc/sudoers:
-#   updatesd     ALL = NOPASSWD: /usr/sbin/rndc reload
+# The PowerDNS $APIKEY is loaded from /usr/local/spamassassin/automc/.powerdns.
+
+HOME="/usr/local/spamassassin/automc"
+
+# load the APIKEY
+. $HOME/.powerdns
+if [[ -z "$APIKEY" ]]; then
+  echo "PowerDNS \$APIKEY not found."
+  echo "It should have been loaded from:"
+  echo " $HOME/.powerdns."
+  exit 53
+fi
 
 set -x
 
-cd /home/updatesd/svn/spamassassin
+LOGDIR="/var/www/bbmass.spamassassin.org/mkupdates"
+
+cd /usr/local/spamassassin/svn/trunk/
 
 . /etc/profile
 #Switched to system perl on spamassassin-vm box
@@ -19,24 +31,7 @@ PERL=/usr/bin/perl
 
 # download stage, where update tarballs are deposited for downloaders
 #
-stagedir=/var/www/buildbot.spamassassin.org/updatestage
-
-# directory where "0.2.3" and other version-specific files live.
-# it's assumed that the *real* zone $INCLUDEs files from this dir.
-# it must be writable by the user this script runs as.
-#
-# dev, testing:
-# dnsdir=/var/named/updates.dev.spamassassin.org.d
-# live:
-# dnsdir=/var/named/updates.spamassassin.org.d
-#
-dnsdir=/var/named/updates.spamassassin.org.d
-
-# directory where "counter", "soa_line.tmpl", "soa_line" live.
-# it's assumed that the *real* zone $INCLUDEs files from this dir.
-# it must be writable by the user this script runs as.
-#
-soadir=/var/named/spamassassin.org.d
+stagedir=/var/www/bbmass.spamassassin.org/updates
 
 versions="3.4.2"
 
@@ -50,159 +45,146 @@ svn commit -m 'promotions validated' rul
 
 make_tarball_for_version () {
 
-# to be honest, right now this is unused.
-version="$1"
-
-tmpdir=/home/updatesd/tmp/stage/$version
-rm -rf $tmpdir; mkdir -p $tmpdir         || exit $?
+  # to be honest, right now this is unused.
+  version="$1"
 
+  tmpdir=/usr/local/spamassassin/automc/tmp/stage/$version
+  rm -rf $tmpdir; mkdir -p $tmpdir         || exit $?
 
+  # extract the new rules files.
 
-# extract the new rules files.
-
-# use "make install" logic, since we want rules as close as possible
-# to what's installed
-# TODO: this *would* be performed in a checkout of the desired
-# version's branch.  right now we're only using 1 version though
-make clean
-$PERL Makefile.PL PREFIX=$tmpdir < /dev/null || exit $?
-make                                     || exit $?
-
-# remove new features, unsupported in existing code in the field
-# (TODO: need a better way to exclude files that require new features
-# like this; judicious use of "ifplugin" helps)
-# rm rules/60_somerandomfeature.cf
-
-# ensure the basic lint/rule-sanity test suite passes for this ruleset
-# before we build an update from it.  useful particularly to catch
-# "tflags nopublish" leakage (bug 6297)
-make test \
+  # use "make install" logic, since we want rules as close as possible
+  # to what's installed
+  # TODO: this *would* be performed in a checkout of the desired
+  # version's branch.  right now we're only using 1 version though
+  make clean
+  $PERL Makefile.PL PREFIX=$tmpdir < /dev/null || exit $?
+  make                                     || exit $?
+
+  # remove new features, unsupported in existing code in the field
+  # (TODO: need a better way to exclude files that require new features
+  # like this; judicious use of "ifplugin" helps)
+  # rm rules/60_somerandomfeature.cf
+
+  # ensure the basic lint/rule-sanity test suite passes for this ruleset
+  # before we build an update from it.  useful particularly to catch
+  # "tflags nopublish" leakage (bug 6297)
+  make test \
     TEST_FILES="t/basic_lint.t t/basic_lint_without_sandbox.t t/basic_meta.t" \
     || exit $?
 
-# remove the rules files for rules we won't be shipping
-rm rules/70_sandbox.cf rules/70_inactive.cf
+  # remove the rules files for rules we won't be shipping
+  rm rules/70_sandbox.cf rules/70_inactive.cf
 
-# double check we still lint without those 2 files
-./spamassassin --lint                   || exit $?
+  # double check we still lint without those 2 files
+  ./spamassassin --lint                   || exit $?
 
-rulesdir=`pwd`/rules
+  rulesdir=`pwd`/rules
 
-(
-  cd $rulesdir 
+  (
+    cd $rulesdir 
 
-  # Use this to include plugin .pm files:
-  # tar cvf - *.cf *.pm                  || exit $?
+    # Use this to include plugin .pm files:
+    # tar cvf - *.cf *.pm                  || exit $?
 
-  # or this, to ban code from the updates:
-  tar cvf - *.cf                         || exit $?
+    # or this, to ban code from the updates:
+    tar cvf - *.cf                         || exit $?
 
-) | gzip -9 > $tmpdir/update.tgz         || exit $?
+  ) | gzip -9 > $tmpdir/update.tgz         || exit $?
 
-# ensure non-empty
-[ -s $tmpdir/update.tgz ] || exit 3
+  # ensure non-empty
+  [ -s $tmpdir/update.tgz ] || exit 3
 
-linttmp=$tmpdir/lintdir
-rm -rf $linttmp
-mkdir $linttmp
-(
-  cd $linttmp
-  # check validity of tarball; also extract
-  gunzip -cd < $tmpdir/update.tgz | tar xf - || exit $?
-)
+  linttmp=$tmpdir/lintdir
+  rm -rf $linttmp
+  mkdir $linttmp
+  (
+    cd $linttmp
+    # check validity of tarball; also extract
+    gunzip -cd < $tmpdir/update.tgz | tar xf - || exit $?
+  )
 
-sitetmp=$tmpdir/sitetmp
-rm -rf $sitetmp
-mkdir $sitetmp
-cp rules/*.pre $sitetmp
+  sitetmp=$tmpdir/sitetmp
+  rm -rf $sitetmp
+  mkdir $sitetmp
+  cp rules/*.pre $sitetmp
 
-# now, ensure the ruleset (entirely as distributed) lints, also.
-# use "-p /dev/null" so any user_prefs data is ignored.
-./spamassassin -x --configpath=$linttmp --siteconfigpath=$sitetmp \
+  # now, ensure the ruleset (entirely as distributed) lints, also.
+  # use "-p /dev/null" so any user_prefs data is ignored.
+  ./spamassassin -x --configpath=$linttmp --siteconfigpath=$sitetmp \
                -p /dev/null --lint \
                 || exit $?
 
-# sign and get sums
-gpg --batch --homedir /home/updatesd/key \
+  # sign and get sums
+  gpg --batch --homedir $HOME/key \
 	-bas $tmpdir/update.tgz  		|| exit $?
 
-$PERL build/sha1sum.pl $tmpdir/update.tgz > $tmpdir/update.tgz.sha1  || exit $?
-
-
+  $PERL build/sha1sum.pl $tmpdir/update.tgz > $tmpdir/update.tgz.sha1  || exit $?
 
-# get SVN revision number.
-# note: use 'Last Changed Rev' instead of 'Revision'.  Because we share
-# an SVN repository with other projects, this means that the same
-# rev of *our* codebase may appear under multiple rev#s, as other projects
-# check their changes in.
 
-tagstamp=`date "+%Y%m%d%H%M%S"`
-tagurl=https://svn.apache.org/repos/asf/spamassassin/tags/sa-update_${version}_${tagstamp}
+  # get SVN revision number.
+  # note: use 'Last Changed Rev' instead of 'Revision'.  Because we share
+  # an SVN repository with other projects, this means that the same
+  # rev of *our* codebase may appear under multiple rev#s, as other projects
+  # check their changes in.
+
+  tagstamp=`date "+%Y%m%d%H%M%S"`
+  tagurl=https://svn.apache.org/repos/asf/spamassassin/tags/sa-update_${version}_${tagstamp}
+
+  # this svn copy is critical, to ensure each version's tarball has a different
+  # rev#.  if you remove it, we need to prefix the version# to the svnrev# in
+  # the filenames instead so each version doesn't clobber others.
+  svn up
+  svn copy -m 'promotions validated' . $tagurl < /dev/null
 
-# this svn copy is critical, to ensure each version's tarball has a different
-# rev#.  if you remove it, we need to prefix the version# to the svnrev# in
-# the filenames instead so each version doesn't clobber others.
-svn up
-svn copy -m 'promotions validated' . $tagurl < /dev/null
-
-# for svn 1.3:
-# (svn info --non-interactive $tagurl || svn info $tagurl ) < /dev/null \
+  # for svn 1.3:
+  # (svn info --non-interactive $tagurl || svn info $tagurl ) < /dev/null \
                 # > $tmpdir/svn 2>&1 || exit $?
 
-# for crappy zone svn, 1.2:
-( 
-  rm -rf tmpcheckout
-  svn co $tagurl tmpcheckout && svn info tmpcheckout
-  rm -rf tmpcheckout
-) < /dev/null > $tmpdir/svn 2>&1 || exit $?
+  # for crappy zone svn, 1.2:
+  ( 
+    rm -rf tmpcheckout
+    svn co $tagurl tmpcheckout && svn info tmpcheckout
+    rm -rf tmpcheckout
+  ) < /dev/null > $tmpdir/svn 2>&1 || exit $?
 
-svnrev=`(grep 'Last Changed Rev: ' $tmpdir/svn || exit 1) | \
+  svnrev=`(grep 'Last Changed Rev: ' $tmpdir/svn || exit 1) | \
         sed -e 's/^.*: //'`
 
-if [ "$svnrev" == "" ] ; then
-  echo "missing SVN revision"
-  cat $tmpdir/svn
-  exit 5
-fi
+  if [ "$svnrev" == "" ] ; then
+    echo "missing SVN revision"
+    cat $tmpdir/svn
+    exit 5
+  fi
 
-if [ "$svnrev" -lt 1 ] ; then
-  echo "bad SVN revision: $svnrev"
-  cat $tmpdir/svn
-  exit 5
-fi
+  if [ "$svnrev" -lt 1 ] ; then
+    echo "bad SVN revision: $svnrev"
+    cat $tmpdir/svn
+    exit 5
+  fi
 
-mv $tmpdir/update.tgz      $stagedir/$svnrev.tar.gz            || exit $?
-mv $tmpdir/update.tgz.sha1 $stagedir/$svnrev.tar.gz.sha1       || exit $?
-mv $tmpdir/update.tgz.asc  $stagedir/$svnrev.tar.gz.asc        || exit $?
+  mv $tmpdir/update.tgz      $stagedir/$svnrev.tar.gz            || exit $?
+  mv $tmpdir/update.tgz.sha1 $stagedir/$svnrev.tar.gz.sha1       || exit $?
+  mv $tmpdir/update.tgz.asc  $stagedir/$svnrev.tar.gz.asc        || exit $?
 
-chmod 644 $stagedir/$svnrev.*
+  chmod 644 $stagedir/$svnrev.*
 
 
+  # next, create the new DNS record....
 
-# next, create the new DNS record....
+  # turn "3.2.0" into "0.2.3"
+  rvers=`echo "$version" | rev`
 
-# turn "3.2.0" into "0.2.3"
-rvers=`echo "$version" | perl -pe 's/^(\d+)\.(\d+)\.(\d+)$/$3.$2.$1/'`
-
-dnsfile="$dnsdir/$version"
-if echo "
-$rvers	TXT	\"$svnrev\"
-" > $dnsfile.new
-then
-  mv $dnsfile.new $dnsfile || exit $?
-else
-  echo "failed to create $dnsfile.new" 1>&2 ; exit 1
-fi
+  curl -X PATCH --data "{\"rrsets\": [ {\"name\": \"${rvers}.updates.spamassassin.org.\",
\"type\": \"TXT\", \"ttl\": 3600, \"changetype\": \"REPLACE\", \"records\": [ {\"content\":
\"${svnrev}\", \"disabled\": false } ] } ] }" -H "X-API-Key: ${APIKEY}" http://127.0.0.1:8081/api/v1/servers/localhost/zones/spamassassin.org.
| jq .
 
-# increment the zone serial.
-./build/mkupdates/tick_zone_serial || exit $?
+  curl -X PUT -H "X-API-Key: ${APIKEY}" http://127.0.0.1:8081/api/v1/servers/localhost/zones/spamassassin.org./notify
| jq .
 
 
-# clean up 4-day-old (and older) update tarballs.  This seems as
-# good a place as any to do this!
-# note: for manual updates, the file permissions should be 0444 so let's clean
-# out only 0644 (automatic) updates.  a bit of a kluge, but ...
-find $stagedir -mtime +4 -perm 0644 -type f -name '*.tar.*' | xargs rm
+  # clean up 4-day-old (and older) update tarballs.  This seems as
+  # good a place as any to do this!
+  # note: for manual updates, the file permissions should be 0444 so let's clean
+  # out only 0644 (automatic) updates.  a bit of a kluge, but ...
+  find $stagedir -mtime +4 -perm 0644 -type f -name '*.tar.*' -delete
 
 }
 
@@ -210,22 +192,20 @@ find $stagedir -mtime +4 -perm 0644 -typ
 
 cycle_logfiles () {
 # cycle the logfiles; keep 6 (3 days worth I think)
-(
-  cd /var/www/buildbot.spamassassin.org/updatesd
-  rm mkupdatespt2_6.txt
-  mv mkupdatespt2_5.txt  mkupdatespt2_6.txt
-  mv mkupdatespt2_4.txt  mkupdatespt2_5.txt
-  mv mkupdatespt2_3.txt  mkupdatespt2_4.txt
-  mv mkupdatespt2_2.txt  mkupdatespt2_3.txt
-  mv mkupdatespt2_1.txt  mkupdatespt2_2.txt
-  mv mkupdatespt2.txt    mkupdatespt2_1.txt
-)
+  X=6
+  [[ -e "$LOGDIR/mkupdatespt2_${X}.txt" ]] && rm -f $LOGDIR/mkupdatespt2_${X}.txt
+  while [[ $X -gt 0 ]]; do
+    ((X--))
+    Y=$((X+1))
+    [[ -e "$LOGDIR/mkupdatespt2_${X}.txt" ]] && mv -f $LOGDIR/mkupdatespt2_${X}.txt
$LOGDIR/mkupdatespt2_${Y}.txt
+  done
+  mv -f $LOGDIR/mkupdatespt2.txt $LOGDIR/mkupdatespt2_${Y}.txt
 }
 
 # ---------------------------------------------------------------------------
 
-[ -d $stagedir ] || echo "no stagedir" 1>&2
-[ -d $stagedir ] || exit 6
+[[ -d $stagedir ]] || echo "no stagedir" 1>&2
+[[ -d $stagedir ]] || exit 6
 
 for version in $versions ; do
   make_tarball_for_version $version



Mime
View raw message