incubator-alois-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fla...@apache.org
Subject svn commit: r1033434 - in /incubator/alois/trunk: prisma/bin/ prisma/data/prisma/prisma_database/model/ prisma/data/prisma/pumpy_database/model/ rails/app/controllers/ rails/app/models/ rails/script/
Date Wed, 10 Nov 2010 12:27:27 GMT
Author: flavio
Date: Wed Nov 10 12:27:26 2010
New Revision: 1033434

URL: http://svn.apache.org/viewvc?rev=1033434&view=rev
Log:
Made prisma restart on db connection loss and corrected statistics display.

Modified:
    incubator/alois/trunk/prisma/bin/prisma
    incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb
    incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb
    incubator/alois/trunk/rails/app/controllers/prisma_controller.rb
    incubator/alois/trunk/rails/app/models/sentinel.rb
    incubator/alois/trunk/rails/script/sentinel

Modified: incubator/alois/trunk/prisma/bin/prisma
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/bin/prisma?rev=1033434&r1=1033433&r2=1033434&view=diff
==============================================================================
--- incubator/alois/trunk/prisma/bin/prisma (original)
+++ incubator/alois/trunk/prisma/bin/prisma Wed Nov 10 12:27:26 2010
@@ -68,22 +68,63 @@ when "start"
   end
   
   Prisma::Database.load_all(options)
-  Prisma::Database.check("prisma" , options)
-  Prisma::Database.check("pumpy" , options)
 
   if daemon
     Prisma::Util.define_new_logger("prisma")
     daemonize(:pid_file => "/var/run/prisma")
-    Prisma::Database.reconnect
   end
 
   Signal.trap("TERM") do; $terminate = true; end
   Signal.trap("USR1") do; $terminate = true; $restart = true; end
 
-  source = SourceDbMeta.new.prisma_initialize(type, eval(params[:sourceclass]), count,nil,
false, waiting_time)
-  $enable_dublette_recognition = source.may_contain_dublettes
+  klass = eval(params[:sourceclass])
   Prisma::Util.define_new_logger(params[:sourceclass])
-  source.transform
+
+  source = nil
+  $restart = true
+  while $restart
+    Prisma::Database.reconnect if daemon
+    $terminate = false
+    $restart = type != :all
+    begin
+      Prisma::Database.check("prisma" , options)
+      Prisma::Database.check("pumpy" , options)
+      $log.info{"Process #{type} records of class #{klass.name}, #{count} per step"}
+      if source 
+        source.finished = true
+        source.save	      
+      end
+      source = SourceDbMeta.new.prisma_initialize(type, klass, count,nil, false, waiting_time)
+      $enable_dublette_recognition = source.may_contain_dublettes
+      source.transform
+    rescue ActiveRecord::Transactions::TransactionError
+      raise $!
+    rescue	    
+      $log.error{ "Processing class #{klass.name} threw error #{$!}!" }
+      for line in $!.backtrace
+        $log.error{"#{line}"}
+      end
+      if Prisma::Database.check_connections
+        $log.fatal{"Connections are good, so something bad happened. Will not risk to restart
queue #{klass.name}."}
+        $terminate = true
+        $restart = false
+      else
+        $log.info{"At least one prisma connection is down."}
+        connection_wait_count = 1
+        while not (Prisma::Database.check_connections or $terminate)
+          wait_time = connection_wait_count
+          wait_time = 30 if wait_time > 30 
+          $log.warn{"#{connection_wait_count} Waiting #{wait_time} seconds."}
+          Prisma::Util.save_sleep(wait_time)
+          connection_wait_count += 1
+        end
+        if !$terminate
+          $log.info{"Connection are good again. Restarting queue #{klass.name}."}		
+        end
+      end
+    end
+  end
+  
   exit 0  
 else
   raise "Unexpected action #{action}"

Modified: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb?rev=1033434&r1=1033433&r2=1033434&view=diff
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb (original)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb Wed Nov
10 12:27:26 2010
@@ -105,7 +105,7 @@
 	  last = iter.last
 	  
 	  begin
-	    Prisma::Database.transaction(self) do
+	    Prisma::Database.transaction(self.class) do
 	      iter.each { |raw_message|
 		loop_count += 1
 #		cost = Benchmark.measure {		

Modified: incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb?rev=1033434&r1=1033433&r2=1033434&view=diff
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb (original)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb Wed Nov 10
12:27:26 2010
@@ -19,7 +19,7 @@
     def self.get_percentage
       percents = []
       count = 0
-      for klass in Prisma.get_classes(:raw)
+      for klass in Prisma::Database.get_classes(:raw)
 	recs = find(:all,:order => "id DESC", :limit => 1, :conditions => "table_name =
'#{klass.table_name}'")
 	if recs.length == 1 
 	  percents.push(recs[0].percentage)

Modified: incubator/alois/trunk/rails/app/controllers/prisma_controller.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/rails/app/controllers/prisma_controller.rb?rev=1033434&r1=1033433&r2=1033434&view=diff
==============================================================================
--- incubator/alois/trunk/rails/app/controllers/prisma_controller.rb (original)
+++ incubator/alois/trunk/rails/app/controllers/prisma_controller.rb Wed Nov 10 12:27:26 2010
@@ -25,8 +25,9 @@ class PrismaController < ApplicationCont
     dobby = (prisma_connection and prisma_connection.spec[:host]) or "localhost"
     dobby = "localhost.localdomain" if dobby == "localhost"
 
-    reporter = (alois_connection and alois_connection.spec[:host]) or "localhost"
-    reporter = "localhost.localdomain" if reporter == "localhost"
+    reporter = open("|hostname -f") {|f| f.readlines.join.strip}
+    # reporter = (alois_connection and alois_connection.spec[:host]) or "localhost"
+    # reporter = "localhost.localdomain" if reporter == "localhost"
 
     sink = (pumpy_connection and pumpy_connection.spec[:host]) or "localhost"
     sink = "localhost.localdomain" if sink == "localhost"

Modified: incubator/alois/trunk/rails/app/models/sentinel.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/rails/app/models/sentinel.rb?rev=1033434&r1=1033433&r2=1033434&view=diff
==============================================================================
--- incubator/alois/trunk/rails/app/models/sentinel.rb (original)
+++ incubator/alois/trunk/rails/app/models/sentinel.rb Wed Nov 10 12:27:26 2010
@@ -82,7 +82,7 @@
     end
     
     ## To use this add the following line into the sudoers file
-    # www-data        ALL=(prisma)    NOPASSWD:/usr/bin/prisma-sentinel --generate-crontab
+    # www-data        ALL=(prisma)    NOPASSWD:/usr/bin/alois-sentinel --generate-crontab
     if RAILS_ENV == 'production'
       UPDATE_COMMAND = "/usr/bin/sudo -S -p '' -u prisma /usr/bin/alois-sentinel --generate-crontab
< /dev/null"
     else

Modified: incubator/alois/trunk/rails/script/sentinel
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/rails/script/sentinel?rev=1033434&r1=1033433&r2=1033434&view=diff
==============================================================================
--- incubator/alois/trunk/rails/script/sentinel (original)
+++ incubator/alois/trunk/rails/script/sentinel Wed Nov 10 12:27:26 2010
@@ -1,3 +1,4 @@
+
 #!/usr/bin/ruby
 require 'optparse'
 report_directory = "/var/lib/prisma"
@@ -8,7 +9,7 @@ sentinel_id = nil
 generate_crontab = false
 list_crontab = false
 opts = OptionParser.new do |o|
-  o.banner = "Usage: prisma-sentinel [options]"
+  o.banner = "Usage: alois-sentinel [options]"
 
   o.on( "-s", "--send", "Send message to ossim-server." ) do |a|
     send_event = true
@@ -60,7 +61,7 @@ if generate_crontab or list_crontab
   require 'etc'
   prisma_uid = Etc::getpwnam('prisma').uid
   
-  # use as prisma-sentinel --generate_crontab | sudo -u primsa crontab -
+  # use as alois-sentinel --generate_crontab | sudo -u primsa crontab -
   crontab_entry = ""
   crontab_entry += "# /etc/cron.d/alois-prisma: crontab entries for the prisma package\n"
   crontab_entry += "# DO NOT EDIT THIS FILE\n"
@@ -73,9 +74,9 @@ if generate_crontab or list_crontab
     #      throw "Suspicious interval found in filter!" if not filter.cron_interval =~ "[\d\/]*"
     crontab_entry += "## Sentinel: #{sentinel.name}\n"
     if sentinel.cron_interval =~ Sentinel::CRONTAB_REGEX 
-      crontab_entry += "#{sentinel.cron_interval}\t/usr/bin/prisma-sentinel --id #{sentinel.id}\n"
     
+      crontab_entry += "#{sentinel.cron_interval}\t/usr/bin/alois-sentinel --id #{sentinel.id}\n"
     
     else	
-      crontab_entry += "# INVALID/IGNORED --- #{sentinel.cron_interval}\t/usr/sbin/prisma-sentinel
--id #{sentinel.id}\n\n"
+      crontab_entry += "# INVALID/IGNORED --- #{sentinel.cron_interval}\t/usr/bin/alois-sentinel
--id #{sentinel.id}\n\n"
     end
   end
   crontab_entry += "\n# -- END OF FILE\n"



Mime
View raw message