spamassassin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j.@apache.org
Subject svn commit: r795248 - in /spamassassin/trunk/backend/nitemc: gen_targets_files run_all run_one_nitemc svn_checkout
Date Fri, 17 Jul 2009 22:18:40 GMT
Author: jm
Date: Fri Jul 17 22:18:40 2009
New Revision: 795248

URL: http://svn.apache.org/viewvc?rev=795248&view=rev
Log:
change nitemc to be more efficient when multiple users are being mass-checked each night

Added:
    spamassassin/trunk/backend/nitemc/run_all   (with props)
    spamassassin/trunk/backend/nitemc/svn_checkout   (with props)
Modified:
    spamassassin/trunk/backend/nitemc/gen_targets_files
    spamassassin/trunk/backend/nitemc/run_one_nitemc

Modified: spamassassin/trunk/backend/nitemc/gen_targets_files
URL: http://svn.apache.org/viewvc/spamassassin/trunk/backend/nitemc/gen_targets_files?rev=795248&r1=795247&r2=795248&view=diff
==============================================================================
--- spamassassin/trunk/backend/nitemc/gen_targets_files (original)
+++ spamassassin/trunk/backend/nitemc/gen_targets_files Fri Jul 17 22:18:40 2009
@@ -22,8 +22,8 @@
 
   $submitter =~ s,[^A-Za-z0-9],,gs;
 
-  open (OUT, ">$TGTS_DIR/targets.$submitter")
-    or warn "cannot write to $TGTS_DIR/targets.$submitter";
+  open (OUT, ">$TGTS_DIR/$submitter.targets")
+    or warn "cannot write to $TGTS_DIR/$submitter.targets";
 
   print OUT "ham:detect:$RAWCOR_DIR/$submitter/ham/*\n",
             "spam:detect:$RAWCOR_DIR/$submitter/spam/*\n";

Added: spamassassin/trunk/backend/nitemc/run_all
URL: http://svn.apache.org/viewvc/spamassassin/trunk/backend/nitemc/run_all?rev=795248&view=auto
==============================================================================
--- spamassassin/trunk/backend/nitemc/run_all (added)
+++ spamassassin/trunk/backend/nitemc/run_all Fri Jul 17 22:18:40 2009
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# driver for nightly mass-checks on the zone. run from cron as:
+# 0 9 * * * /export/home/svn-trunk/backend/nitemc/run_all
+#
+# details: http://wiki.apache.org/spamassassin/NightlyMassCheck
+
+# Ensure we're running after 0900 UTC; complex due to daylight savings. If
+# we're running in the 08xx UTC hour-space, sleep for 1 hour; when we wake, we
+# will be in 09xx.  By doing it this way, we'll work both for (a) mass-checks
+# set to run from cron at 09xx (cron doesn't understand DST/UTC issues) and (b)
+# mass-checks starting at other times that != 09xx.
+(
+  TIME="%e,%U,%S"; TZ=UTC; export TIME TZ
+  if date | egrep '^... ... .. 08:'; then
+    date; echo "sleeping for 1 hour to compensate for DST"
+    sleep 3600 ; date
+  fi
+)
+
+echo "nitemc run_all starting at" ; date
+
+pkill -15 -f -u nitemc perl
+pkill -15 -f -u nitemc ssh
+
+all_users=`ls /export/home/bbmass/uploadedcorpora`
+nitedir=/export/home/svn-trunk/backend/nitemc
+
+$nitedir/gen_targets_files
+$nitedir/svn_checkout
+for user in $all_users ; do
+  $nitedir/run_one_nitemc $user
+done
+

Propchange: spamassassin/trunk/backend/nitemc/run_all
------------------------------------------------------------------------------
    svn:executable = *

Modified: spamassassin/trunk/backend/nitemc/run_one_nitemc
URL: http://svn.apache.org/viewvc/spamassassin/trunk/backend/nitemc/run_one_nitemc?rev=795248&r1=795247&r2=795248&view=diff
==============================================================================
--- spamassassin/trunk/backend/nitemc/run_one_nitemc (original)
+++ spamassassin/trunk/backend/nitemc/run_one_nitemc Fri Jul 17 22:18:40 2009
@@ -5,54 +5,47 @@
 [ "$user" = "" ] && exit 99
 
 # where the $HOMEs are:
-dir="/export/home/nitemc/$user"
+userdir=/export/home/nitemc/$user
+svndir=/export/home/nitemc/svn
 
-# keep one day's logs around
-mv "$dir/corpus-nightly.log" "$dir/corpus-nightly.log.1"
+mkdir -p $userdir
+log=$userdir/log
+mv $log $log.1
 
 (
-echo "Starting corpus-nightly for $user ($dir): $dir/corpus-nightly.log"
+echo "Starting corpus-nightly for $user ($svndir , $userdir)"
 
 . /etc/profile
-
-# ensure we use the user-wide SVN permissions and server identities,
-# otherwise we won't be able to perform noninteractive SVN ops
-rm -rf $dir/.subversion 
-ln -s /export/home/nitemc/.subversion $dir/.subversion 
-
-# oh Solaris, you really suck
-HOME=$dir
+HOME=$userdir
 PERL=/local/perl586/bin/perl
 TMPDIR=/tmpfs
 
 export HOME PERL TMPDIR
 
-# attempt to ensure the "build" dir and raw .corpus file is up to date 
-# (don't worry about tags etc. just yet)
-(
-  cd $dir
-  rm -rf svn
-  svn co http://svn.apache.org/repos/asf/spamassassin/trunk svn ||
-    svn co http://svn.apache.org/repos/asf/spamassassin/trunk svn ||
-    svn co http://svn.apache.org/repos/asf/spamassassin/trunk svn
-
-) < /dev/null
-
 # create the .corpus file, keeping the rsync password secret;
 # the string __RSYNC_PASSWORD__ will be replaced with the
 # contents of /export/home/nitemc/USERNAME/rsync_password
+
+src=$svndir/backend/nitemc/corpus.$user
+[ -f $src ] || src=$svndir/backend/nitemc/corpus.default
+
 perl -pe '
   s{__RSYNC_PASSWORD__}{ incfile("rsync_password") }eg;
   s{__CLIENTHOSTS__}{ incfile("svn/backend/nitemc/clienthosts") }eg;
 
   sub incfile {
     my $file = shift;
-    my $new=`cat '"$dir"'/$file`; chop $new; $new =~ s/\s+/ /gs;
+    my $new=`cat '"$userdir"'/$file`; chop $new; $new =~ s/\s+/ /gs;
     return "\"".$new."\"";
   }
-' $dir/svn/backend/nitemc/corpus.$user > $dir/.corpus
+' $src > $userdir/.corpus
+
+rm -rf $svndir/masses/tmp
+rm -rf $svndir/masses/spamassassin
+rm -rf $svndir/masses/*.log
 
 # use bash to work around Solaris breakage
-exec nice bash $dir/svn/masses/rule-qa/corpus-nightly
+exec nice bash $svndir/masses/rule-qa/corpus-nightly
+
+) 2>&1 | tee $log
 
-) 2>&1 | tee "$dir/corpus-nightly.log"

Added: spamassassin/trunk/backend/nitemc/svn_checkout
URL: http://svn.apache.org/viewvc/spamassassin/trunk/backend/nitemc/svn_checkout?rev=795248&view=auto
==============================================================================
--- spamassassin/trunk/backend/nitemc/svn_checkout (added)
+++ spamassassin/trunk/backend/nitemc/svn_checkout Fri Jul 17 22:18:40 2009
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+. /etc/profile
+HOME=/export/home/nitemc
+PERL=/local/perl586/bin/perl
+TMPDIR=/tmpfs
+
+export HOME PERL TMPDIR
+
+# attempt to ensure the "build" dir and raw .corpus file is up to date
+# (don't worry about tags etc. just yet)
+(
+  cd /export/home/nitemc
+  rm -rf svn
+  svn co http://svn.apache.org/repos/asf/spamassassin/trunk svn ||
+    svn co http://svn.apache.org/repos/asf/spamassassin/trunk svn ||
+    svn co http://svn.apache.org/repos/asf/spamassassin/trunk svn
+
+) < /dev/null
+

Propchange: spamassassin/trunk/backend/nitemc/svn_checkout
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message