incubator-alois-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fla...@apache.org
Subject svn commit: r1031127 [5/22] - in /incubator/alois/trunk: ./ bin/ debian/ doc/ etc/ etc/alois/ etc/alois/apache2/ etc/alois/environments/ etc/alois/prisma/ etc/cron.d/ etc/default/ etc/logrotate.d/ prisma/ prisma/bin/ prisma/conf/ prisma/conf/prisma/ pr...
Date Thu, 04 Nov 2010 18:27:42 GMT
Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/002_enlarge_syslogd_msg_field.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/002_enlarge_syslogd_msg_field.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/002_enlarge_syslogd_msg_field.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/002_enlarge_syslogd_msg_field.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,25 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class EnlargeSyslogdMsgField < ActiveRecord::Migration
+
+  def self.up
+    change_column "syslogd_raws", "msg", :binary
+  end    
+
+  def self.down
+    raise IrreversibleMigration
+  end
+  
+end

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/003_enlarge_file_raws_msg_field.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/003_enlarge_file_raws_msg_field.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/003_enlarge_file_raws_msg_field.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/003_enlarge_file_raws_msg_field.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,25 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class EnlargeFileRawsMsgField < ActiveRecord::Migration
+
+  def self.up
+    change_column "file_raws", "msg", :longblob
+  end    
+
+  def self.down
+    raise IrreversibleMigration
+  end
+  
+end

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/004_migrate_to_new_schema_version.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/004_migrate_to_new_schema_version.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/004_migrate_to_new_schema_version.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/004_migrate_to_new_schema_version.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,26 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class MigrateToNewSchemaVersion < ActiveRecord::Migration
+
+  def self.up
+    # empty, so new table schema_migrations will
+    # be created for upgrade to rails 2.3
+  end    
+
+  def self.down
+    raise IrreversibleMigration
+  end
+  
+end

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/file_raw.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/file_raw.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/file_raw.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/file_raw.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,53 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class FileRaw < ActiveRecord::Base
+
+    description "File raw record on pumpy."
+    
+    def self.default_waiting_time
+      60 * 15
+    end
+    
+    def self.default_count; 1; end
+
+    def self.create(file, options={})
+      options[:extname] ||= File.extname(file)
+      if options.is_a?(Hash)
+	options = options.map {|key,val| "#{key}=#{val}"}.join(",")
+      end
+
+      m = self.new
+      m.dirname = File.dirname(file)
+      m.basename = File.basename(file)
+      m.ftype = File.ftype(file)
+      m.size = File.size(file)
+      m.mtime = File.mtime(file)
+      m.atime = File.atime(file)
+      m.ctime = File.ctime(file)
+      m.options = options
+      #      m.umask = f.umask
+      #      m.uid = File.uid(file)
+      #      m.gid = File.gid(file)
+      File.open(file, "rb") do |io_r|
+	m.msg = io_r.read(m.size)
+      end
+      m.save
+      return m
+    end
+
+    def self.may_contain_dublettes
+      true
+    end
+  end

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/pumpy_schema_migration.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/pumpy_schema_migration.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/pumpy_schema_migration.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/pumpy_schema_migration.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,24 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  # Class for accessing pumpy database schema version
+  class PumpySchemaMigration < ActiveRecord::Base
+    set_table_name "schema_migrations"
+    
+    def self.version
+      self.find(:all).sort_by {|m| m.version.to_i}[-1].version.to_i
+    end
+
+  end
+

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/raws_state.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,41 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class RawsState < ActiveRecord::Base
+
+    description "Syslogd raw status table on pumpy."
+
+    def self.get_percentage
+      percents = []
+      count = 0
+      for klass in Prisma.get_classes(:raw)
+	recs = find(:all,:order => "id DESC", :limit => 1, :conditions => "table_name = '#{klass.table_name}'")
+	if recs.length == 1 
+	  percents.push(recs[0].percentage)
+	  count = count + 1
+	end
+      end
+      percents.max
+    end
+        
+    def percentage
+      begin
+	(100. / count_limit.to_f) * count_value.to_f
+      rescue
+	100
+      end
+    end
+
+  end
+

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/syslogd_raw.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/syslogd_raw.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/syslogd_raw.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/model/syslogd_raw.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,48 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class SyslogdRaw < ActiveRecord::Base
+
+    # kernel = 0, user = 1, mail = 2, ...
+    FACILITIES = ["kernel","user","mail","system","auth"]
+    
+    # emerg = 0, alert = 1, ...
+    LEVELS = ["emerg","alert","crit","err","warning","notice","info","debug"]
+    
+    PRIORITIES = ["emerg","alert","cirt","err","warning","notice","info","debug","none"]
+
+
+    description "Syslogd raw record on pumpy."    
+
+    attr_accessor :origin
+
+    def self.create_random_test_message(date_time = nil)
+      date_time ||= DateTime.now
+      
+      SyslogdRaw.create(:ip => "127.0.0.111",
+			:host => "testhost",
+			:facility => FACILITIES.rand,
+			:priority => PRIORITIES.rand,
+			:level => LEVELS.rand,
+			:tag => "",
+			:date => date_time.strftime("%F"),
+			:time => date_time.strftime("%T"),
+			:program => "Alois Random",
+			:msg => "Alois random genearated log.")
+    end
+    
+    def to_s
+      "SyslogdRaw.#{id} #{@origin}"
+    end
+  end

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/schema.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/schema.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/schema.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/schema.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,72 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated from the current state of the database. Instead of editing this file, 
+# please use the migrations feature of Active Record to incrementally modify your database, and
+# then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your database schema. If you need
+# to create the application database on another system, you should be using db:schema:load, not running
+# all the migrations from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended to check this file into your version control system.
+
+ActiveRecord::Schema.define(:version => 4) do
+
+  create_table "file_raws", :force => true do |t|
+    t.string   "dirname"
+    t.string   "basename"
+    t.string   "ftype",      :limit => 20
+    t.integer  "size"
+    t.datetime "mtime"
+    t.datetime "atime"
+    t.datetime "ctime"
+    t.integer  "umask"
+    t.integer  "uid"
+    t.integer  "gid"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.string   "options"
+    t.binary   "msg"
+  end
+
+  create_table "raws_states", :force => true do |t|
+    t.string   "table_name",   :limit => 20
+    t.integer  "count_limit"
+    t.float    "count_time"
+    t.integer  "count_value"
+    t.float    "delete_time"
+    t.integer  "delete_value"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "raws_states", ["table_name"], :name => "raws_states_table_name_index"
+
+  create_table "syslogd_raws", :force => true do |t|
+    t.string   "ip",         :limit => 40
+    t.string   "host"
+    t.string   "facility",   :limit => 10
+    t.string   "priority",   :limit => 10
+    t.string   "level",      :limit => 10
+    t.string   "tag",        :limit => 10
+    t.date     "date"
+    t.time     "time"
+    t.integer  "program",    :limit => 15
+    t.datetime "created_at"
+    t.binary   "msg",        :limit => 255
+  end
+
+end

Added: incubator/alois/trunk/prisma/doc/syslog-ng.conf.example
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/doc/syslog-ng.conf.example?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/doc/syslog-ng.conf.example (added)
+++ incubator/alois/trunk/prisma/doc/syslog-ng.conf.example Thu Nov  4 18:27:22 2010
@@ -0,0 +1,52 @@
+# Alois SQL Logging
+# =================
+# SQL logging support
+
+# adapt the wiki page if you change something here
+# wiki: https://socrates2.grey.in.here/mgmtwiki/InFra/AloIs
+
+filter f_alois_ignore {
+	# add here filters of messages that should be ingored
+	# at the source
+         match('.*');
+};
+filter f_not_alois_ignore { not(filter(f_alois_ignore));};
+
+source s_foreign {
+        udp();
+        tcp();
+};
+
+destination d_mysql {
+        pipe("/var/run/mysql.pipe"
+                template("INSERT INTO syslogd_raws (host,ip, facility, priority, level, tag, date,
+                          time, program, msg) VALUES ( '$FULLHOST', '$SOURCEIP', '$FACILITY', '$PRIORITY', '$LEVEL', '$TAG',
+                          '$YEAR-$MONTH-$DAY', '$HOUR:$MIN:$SEC', '$PROGRAM', '$MSG' );\n"
+                         )
+                template-escape(yes)
+            );
+};
+
+log {
+	# use this source if you only want to log foreign
+	# hosts per udp and tcp
+        # source(s_foreign);
+        source(s_all);
+        filter(f_alois_ignore);
+        destination(d_mysql);
+};
+
+destination df_alois { file("/var/log/alois-fallback.log"); };
+destination df_alois_ignored { file("/var/log/alois-ignored.log"); };
+
+log {
+        source(s_foreign);
+        filter(f_alois_ignore);
+        destination(df_alois);
+};
+
+log {
+        source(s_foreign);
+        filter(f_not_alois_ignore);
+        destination(df_alois_ignored);
+};

Added: incubator/alois/trunk/prisma/lib/prisma.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,84 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+raise "PRISMA_ENV constant not defined!" unless defined?(PRISMA_ENV)
+
+require Pathname.new(__FILE__).dirname + "prisma/base_mixin.rb"
+class ActiveRecord::Base
+  include BaseMixin
+end
+
+require Pathname.new(__FILE__).dirname + "prisma/database.rb"
+require Pathname.new(__FILE__).dirname + "prisma/transform.rb"
+require Pathname.new(__FILE__).dirname + "prisma/archive.rb"
+require Pathname.new(__FILE__).dirname + "prisma/archivator.rb"
+
+module Prisma
+  class Util
+    # Log performance, if logger has a perf
+    # function log to it, log to warn otherwise
+    def Util.perf
+      if $log.respond_to?(:perf)
+        $log.perf {yield}
+      else
+        $log.warn {yield}
+      end
+    end
+
+    # Sleep function that awakes if $terminate variable is set
+    # this is used for prisma daemon, if no new logs are in the
+    # queue, the process must sleep but immediately teminate
+    # if it sould.
+    def Util.save_sleep(time)
+      cnt = 0
+      while cnt < time and not $terminate 
+        sleep(1)
+        cnt = cnt + 1
+      end
+    end
+    
+    # Defines a new logger with the given name. A new Logfile will
+    # be opened and all future log calls to $log will be logged there.
+    def Util.define_new_logger(name)
+      return if ENV["LOG_OUTPUT"]
+      new_level = $log.level
+      file = File.join(PRISMA_LOG,name + '.log')
+      $log.info("Creating new logfile: #{file}")
+
+      PRISMA_LOG.mkpath unless PRISMA_LOG.exist?
+      $log.info("Define new logger '#{name}' to #{file.to_s}.") if $log
+      Log.new_logger(name, file)
+      #    $log = Log4r::Logger.new(file)
+      $log.info("Defined new logger '#{name}' to #{file.to_s}.") if $log
+      $log.level = new_level #Logger::WARN
+    end    
+
+  
+    # Generate random syslogd_raws records - for testing purpose only.
+    def Util.generate_random_raws(amount = 100,
+				  range = "beginning yesterday until the end of day".to_time_range.to_a)
+      range = range.to_time_range.to_a if range.class == String
+      amount.times{SyslogdRaw.create_random_test_message(range.rand)}
+    end    
+
+    # Generate random syslogd_raws and transform them, the result will be some meta records - for testing purpose only.
+    def Util.generate_random_data(amount = 100, 
+				  range = "beginning yesterday until the end of day".to_time_range.to_a)
+      
+      generate_random_raws(amount,range)
+      transform_all_raws(SyslogdRaw)
+    end
+    
+  end
+end

Added: incubator/alois/trunk/prisma/lib/prisma/archivator.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/archivator.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/archivator.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/archivator.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,209 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  # Class for archivating raw messages
+  class Archivator
+
+    # Find messages in the archive, conditions is a hash any of
+    #  :regexps: array of regular expression that must match the raw line
+    #  :id: The id of the raw message
+    def self.find(conditions, archive_pattern = $archive_pattern)
+      path = Archivator.archive_path(archive_pattern, conditions)
+      path = path.gsub(/\%./,"*")
+      archive_files = Dir.glob(path)
+      archive_files += Dir.glob(path + ".gz")
+      conditions[:regexps] ||= []
+      archive_files.map {|archive_file|
+	arr = []
+
+	if conditions[:regexps].length == 0 and conditions[:id]
+	  arr << Archivator.fast_find_message(archive_file, conditions[:id])
+	else
+	  raise "Not implemented" if conditions[:id]
+          Archivator.messages(archive_file, conditions[:regexps]) {|m| arr.push(m)}
+	end
+	arr
+      }.flatten.compact
+    end
+
+    # Find a archivated raw message in the archfile by its original id
+    def self.fast_find_message(archivfile, id)
+      case archivfile 
+      when /arch\.gz$/
+	grepper = "zgrep"
+      when /arch.bz?$/
+	grepper = "bzgrep"	
+      else
+	grepper = "grep"
+      end
+      cmd = "|#{grepper} ' id: \\\"#{id}\\\"' '#{archivfile}' | head -1".gsub("\\","\\\\\\")
+      $log.debug{"Find command for original: #{cmd.inspect}"}
+      ret = open(cmd) {|f| f.readlines[0]}
+      return nil if ret.nil? or ret.strip == ""
+      oret = Object.from_yaml(eval(ret))
+      raise "Found object has not correct id #{oret.id.inspect} expected #{id.inspect}" unless
+        oret.id.to_s == id.to_s
+      oret
+    end
+    
+    # Find a archivated raw messages in the archfile byregular expressions
+    def self.messages(archivfile, regexps = [])
+      raise LocalJumpError unless block_given?
+      regexps.reject! {|r| r == //}
+      archivfile = "| /bin/gunzip -c #{archivfile}" if archivfile =~ /\.gz$/
+      open(archivfile,"r") {|f|
+	f.each_with_index {|line,i|
+	  match = true
+	  regexps.each { |reg|
+	    if match
+              match = line =~ reg
+              $log.debug {"#{line.inspect} =~ #{reg.inspect} => #{match.inspect}"}
+	    end
+	  }
+	  next unless match
+	  msg = nil
+	  begin	 
+	    # leave this for security (evaluating string)
+	    throw "Leading and/or tailing \" not found in file '#{archivfile}:#{i}'!" unless line =~ /^".*\"$/
+ 	    throw "Suspicious line found in file '#{archivfile}:#{i}' (unquoted \" found)!" if line =~ /\".*[^\\]\".*\"/
+	    msg = Object.from_yaml(eval(line))
+
+	    msg.origin = "File '#{archivfile}:#{i}'" if msg.respond_to?(:origin)
+	  rescue 
+	    $log.error "Error getting archive record '#{archivfile}:#{i}'. (#{$!.message})" if $log.error?
+	  end
+	  yield msg if msg
+	}
+      }
+    end
+
+    # Substitute placeholder in the path
+    def self.archive_path(path, conditions)
+      path = path.to_s
+      if raw_class = conditions[:raw_class]
+	path = path.gsub(/\%t/,raw_class.table_name)
+	path = path.gsub(/\%c/,raw_class.name)
+      end
+      if conditions[:incoming_date]
+	path = path.gsub(/\%i/, conditions[:incoming_date].to_s)
+      end
+      if conditions[:log_date]
+	path = path.gsub(/\%d/, conditions[:log_date].to_s)
+      end
+      path
+    end
+
+    # Initialize a new Archivator class for raw_class class
+    def initialize(path, raw_class)
+      path = path.to_s
+      $log.info{"Create archivator with path '#{path}' and raw_class '#{raw_class}'."}
+      throw "Archivate path have to contain the pattern \%i." unless path =~ /\%i/
+
+      @path = Archivator.archive_path(path, {:raw_class => raw_class })
+
+      @use_message_date = raw_class.respond_to?(:date)
+      @open_files = {}
+      @used_files = {}
+    end
+    
+    # Do really open the filename (not just lazy opening)
+    def really_open_file(filename)
+      $log.info{"Going to open archive file '#{filename}'."}
+      f = File.new(filename,(File::WRONLY | File::APPEND | File::CREAT))
+      FileUtils.chown("prisma","www-data",filename) if PRISMA_ENV == "production"
+      
+      @open_files[filename] = f
+      @used_files[filename] = true
+      f
+    end
+
+
+    # Archivate the raw_message in the archive. The
+    # class will be serialized to yaml and archivated
+    # to the right file depending on the global archivate path.
+    def archivate(raw_message)
+      if raw_message.respond_to?(:date)
+	filename = Archivator.archive_path(@path, {:incoming_date => Date.today, :log_date => raw_message.date})
+      else
+	filename = Archivator.archive_path(@path, {:incoming_date => Date.today, :log_date => Date.today})
+      end
+      
+      $log.info{"Archivate '#{raw_message.class.name}.#{raw_message.id}' to #{filename}."}
+      dir = File.dirname(filename)
+      
+      unless File.directory?(dir)
+	throw "Wanted to create directory '#{dir}' but there exists already a file." if File.exists?(dir)
+	FileUtils.mkdir_p(dir)
+      end
+      FileUtils.chown("prisma","www-data",dir) if PRISMA_ENV == "production"
+      
+#      @thread.join if @thread
+#      # spin off archivation, to_yaml is slow and dic acces maybe also
+#      @thread = Thread.fork {
+        file = @open_files[filename] || really_open_file(filename)
+        dump = "#{raw_message.to_yaml.dump}\n"
+        file.write(dump)
+#        @thread = nil
+#      }
+      @used_files[filename] = true
+    end
+
+    # Close all archive files, that are not used
+    # anymore
+    def close_unused_files
+      @used_files.each { |filename,used|
+	if used
+	  @used_files[filename] = false 
+	else
+	  close_file(filename)
+	end
+      }
+    end
+    
+    # Close all archive files
+    def close_all_files
+      @used_files.each { |filename,used|
+	close_file(filename)
+      }
+    end
+
+    # Close file by filename
+    def close_file(filename)
+      throw "Filename nil" unless filename
+      $log.info{"Closing archive file '#{filename}'."}
+      @open_files[filename].close
+      @open_files.delete(filename)
+      @used_files.delete(filename)
+    end    
+
+    # Get open files
+    def open_files
+      return @open_files.length
+    end
+
+    # Compress old archive files (pattern is a path
+    # with wildchards used for globbing)
+    def self.compress_old_files(pattern)
+      $log.info{"Looking for archive files '#{pattern}'."}
+      for file in Dir.glob(pattern)
+	unless file.to_s.include?(Date.today.to_s)
+	  throw "Zipped file already exists!" if File.exists?("#{file}.gz")
+	  $log.info{"Zipping file '#{file}'."}
+	  throw "Error zipping '#{file}'." unless system("/bin/gzip \"#{file}\"")
+	else
+	  $log.info{"Not zipping file from today '#{file}'."}
+	end
+      end
+    end
+  end

Added: incubator/alois/trunk/prisma/lib/prisma/archive.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/archive.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/archive.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/archive.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,40 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Prisma
+  class Archive
+
+  # Compresses all archive files
+  def Archive.compress_archives
+    pattern = $archive_pattern.to_s.gsub(/\%c/,"*")
+    pattern = pattern.gsub(/\%t/,"*")
+    pattern = pattern.gsub(/\%i/,"*")
+    pattern = pattern.gsub(/\%d/,"*")
+    Archivator.compress_old_files(pattern)
+  end
+
+  # Import and transform archivated logs, simply provide filename to import.
+  def Archive.import_archive(filename)
+    Signal.trap("TERM") do
+      # End current tranform.
+      $log.info "Caught #{Process.pid} signal TERM." if $log.info?
+      $log.info "Going to stop..." if $log.info?
+      $terminate = true
+    end
+    source = ArchiveMeta.new.prisma_initialize(filename)
+    source.transform
+  end
+
+  end
+end

Added: incubator/alois/trunk/prisma/lib/prisma/base_mixin.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/base_mixin.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/base_mixin.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/base_mixin.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,696 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This class contains helper and basic functions for all
+# meta and raw classes.
+require "active_record"
+
+module BaseMixin
+
+  module ClassMethods
+    
+    # defines finder function for messages, all
+    # meta classes can have a message associated
+    # to it
+    def inherited(subclass)
+      super
+      subclass.set_table_name undecorated_table_name(subclass.name)
+      subclass.module_eval do
+	has_many :messages,
+        :foreign_key => "meta_id",
+        :conditions => "meta_type_name = 'Prisma::#{subclass.name}'"
+        #	  :dependent => :delete_all
+        #	  :finder_sql => "SELECT * FROM messages WHERE 	"AND meta_id = {id}"
+      end
+    end
+    
+    # The foreign column name that another class would have
+    # if it refers to this
+    def foreign_column_name(klass_name = nil)
+      return undecorated_table_name((klass_name or self.name)) + "_id"
+    end
+    
+    # Defines preseed expression. Preseed expressions are
+    # regular expressions that must match for all logs
+    # that will be stored in this class. This functionality is mainly for
+    # speedup. After a log matched any of these preseed expressions,
+    # normal processing/parsing wild be done.
+    def preseed_expression(expr = nil)
+      @preseed_expression = expr if expr
+      return @preseed_expression
+    end
+    
+    # Check if this (parent) class with its message
+    # is applyable for the current class. True if any
+    # preseed_expression matches the message.
+    $expression_count = 0
+    def applyable?(parent, message)
+      if self.respond_to?(:preseed_expression) and self.preseed_expression != nil then
+	$expression_count += 1
+	$log.debug {"Comparing '#{message.msg}' =~ #{self.preseed_expression}"}
+	case message
+	when Message
+	  return message.msg =~ self.preseed_expression
+	when String
+	  return message =~ self.preseed_expression
+	else
+	  raise "Unexpected message class #{message.class}"
+	end
+      end
+      return true
+    end
+    
+    # Adds relations to source classes
+    def sources(array = nil)
+      return (@sources or []) unless array
+      @sources = array
+      @sources.each { |klass_name|	
+	# f_col_name = klass.table_name + "_id"
+	#	unless column_names.index(f_col_name)
+	#	  connection.add_column(table_name, f_col_name, :integer)
+	#	  connection.add_index(table_name,f_col_name)
+	#	  reset_column_information()
+	#	  $log.warn{"Added column #{f_col_name} to table #{table_name}"}
+	#	end
+	#      	p "#{self.name} belongs_to #{undecorated_table_name(klass.name).singularize.to_sym}, with key #{undecorated_table_name(self.name) + "_id"}"
+	belongs_to undecorated_table_name(klass_name).singularize.to_sym,
+        :class_name => klass_name,
+        :foreign_key => self.foreign_column_name(klass_name)
+      }
+    end
+    
+    # returns all child meta classes of this class
+    def child_classes
+      Prisma::Database.get_classes(:meta).select {|klass|
+	klass.columns_hash[self.foreign_column_name]
+      }
+    end
+    # returns all parent meta classes of this class
+    def parent_classes
+      Prisma::Database.get_classes(:meta).select {|klass|
+	self.columns_hash[klass.foreign_column_name]
+      }
+    end
+    # checks weather the given class is a parent of this
+    def parent_class?(klass, done = [])
+      parents = self.parent_classes.reject {|p| done.include?(p)}      
+      my_done = done.dup
+      my_done.push(self)
+      parents.include?(klass) or 
+	parents.select {|p| p.parent_class?(klass,my_done)}.length > 0
+    end
+    # checks weather the given class is a child of this
+    def child_class?(klass, done = [])
+      children = self.child_classes.reject {|p| done.include?(p)}      
+      my_done = done.dup
+      my_done.push(self)
+      children.include?(klass) or 
+	children.select {|p| p.child_class?(klass,my_done)}.length > 0
+    end
+    
+    # Return true if this class can parse the given class
+    def can_transform?(klass)
+      true if sources.index(klass.name) 
+    end
+
+    # Return true if the this class can have messages
+    def may_have_messages?; true; end
+    
+    # for compatibility reason. With this on both, views
+    # and tables the table can be gotten with obj.table
+    def table
+      self
+    end
+    
+    # A description for this class can be defined.
+    def description text = nil
+      return @description unless text
+      
+      @description = text if text
+      @description
+    end
+    # Returns the description in html format
+    def description_html
+      r = "<p>#{description}</p>"
+      if self.respond_to?(:expressions)
+        self.expressions.each {|exp|
+          r += "<p>#{exp[:regex].inspect} => #{exp[:fields].inspect}</p>"
+        }
+      else
+	r += "<p style='background-color:orange'>No regularexpressions to use</p>"
+      end
+      r
+    end
+    
+    
+    # Check if table exist. If not, a exception is raised.
+    def check_table
+      throw "Table '#{table_name}' does not exists." unless table_exists?
+    end
+    
+    # Return a table status, if exist return "OK", a error message instead.
+    # no exception will be risen.
+    def status
+      begin
+	check_table
+	return "OK"
+      rescue ActiveRecord::Transactions::TransactionError
+        # This have to be, that transactions are working
+	raise $!
+      rescue
+	return $!.message.to_s
+      end
+    end
+    
+    # Create a new record (not saved yet), if this is possible for
+    # the given parent meta.
+    # expressions are used to parse message.
+    def create_meta(meta, message)
+      return nil unless self.applyable?(meta, message)
+      if self.respond_to?(:expressions) then
+	ret = nil
+	my_message = case message
+		     when Message
+		       message.msg
+		     when String
+		       message
+		     end
+	$log.debug("Trying to match #{my_message.inspect}") if $log.debug?
+	match, values = self.match_regexps(my_message, meta.class, meta)
+	return self.new.prisma_initialize(meta,values) if values
+      else
+	$log.warn("#{name} has no create method defined.") if $log.warn?
+	return nil
+      end
+      return nil
+    end
+    
+    # Try to match regular expressions against my_message. If regular expressions
+    # match, the first match will be returned with the matched values. (a array with
+    # [regexp, {:key => :val ...}]
+    # TODO: explain that in detail
+    def match_regexps(my_message, meta_class, meta_instance)
+      return nil unless self.respond_to?(:expressions)
+      for expr in self.expressions
+	next if expr[:condition] and !expr[:condition].call(my_message, meta_class, meta_instance)
+        
+	$log.debug{"Comparing to #{expr[:regex]}."}
+	my_message = self.before_filter(my_message) if self.respond_to?(:before_filter)
+	$expression_count += 1
+	if my_message =~ expr[:regex] then
+	  $log.debug{"Regular expression #{expr[:regex]} matched."}
+	  results = Regexp.last_match.to_a[1..-1]
+	  if expr[:result_filter]
+	    $log.debug{"Results before filter: #{results.inspect}"}
+	    results = expr[:result_filter].call(results, meta_instance) 
+	  end
+	  $log.debug{"Results: #{results.inspect}"}
+	  $log.debug("Fields:  #{expr[:fields].inspect}") if $log.debug?
+	  if results.size != expr[:fields].length
+	    throw "Regexp matched (#{results.size}) more than fields (#{expr[:fields].length}) defined. (Tansform against: #{self.name})"
+	  end
+	  
+	  values = {}
+	  expr[:fields].each_with_index {|field, i|
+	    next if field.nil?
+	    val = results[i]
+	    next if val.nil?
+	    if values[field].nil?
+	      values[field] = results[i]
+	    else
+	      values[field] += results[i]
+	    end
+	  }
+	  $log.debug{"Regexp created values: #{values.inspect}"}
+	  for name,value in values
+	    throw "#{name} has no value!" if value == nil
+	    #                throw "#{name} has no value!" if value == ""
+	  end
+	  return [expr, values]
+	end
+      end      
+      return nil
+    end
+        
+  end
+
+  module InstanceMethods
+
+    # Returns the message of this instance (if there is any)
+    # TOOD: Rework this. rescue LocalJumpError is quite hackish
+    def message
+      return @message_fast if @message_fast
+      msgs = nil
+      begin
+	msgs = messages
+      rescue LocalJumpError
+      end
+      return nil unless msgs
+      ms = msgs.size
+      if ms > 1	
+	messages.each { |m| $log.warn{"Found message '#{m.to_s}'"}}
+	throw "More than one (#{ms}) message in messages for #{self.class.name}.#{self.id}!"
+      end
+      return nil if ms == 0 
+      return msgs[0]
+    end
+
+    # use this if you already set association
+    # and you do not want collection perfomance loss
+    def message_fast=(val)
+      @message_fast = val
+    end
+
+    # Sets a new message for this class
+    def message=(val)
+      messages.each { |m| m.prisma_destroy }
+      if val
+	if val.class == String
+	  # new will automatically add
+	  # the message
+	  Message.new.prisma_initialize(self, val)
+	else
+	  messages << val
+	end
+      end
+    end
+
+    # Initializes a new meta with parent and values (like attribtues in active_record),
+    # after values are set, after_filling_values(values) will be called if it exist. Maybe
+    # a corresponding activerecord method with callbacks exists now.
+    # If a value :message is given, a message object is created too.
+    # At the end self.after_init(values) will be called if it exist.
+    # Maybe this is a little bit confusing because new does already save the new record.
+    # This is because of historical reason and could maybe changed in future. This corresponds
+    # more to create of activerecord than to new.
+    def prisma_initialize(parent = nil, values = nil)
+      $log.debug("Creating instance of '#{self.class.name}' with parent '#{parent.class.name}'")
+      raise "Parent is nil" if parent.nil?
+
+      unless self.class.name =~ /Meta$/
+	$log.debug{"This is not a meta record. exiting base_mixin constructor."}
+	return self
+      end
+
+      if parent
+	parent.save if parent and parent.new_record?
+	self.parent = parent
+      end
+
+      if self.respond_to?(:init)
+	# call init from class
+	self.init(values)
+      else
+	# write values to instance
+	if values != nil then
+	  for name, value in values
+	    next if name == :message
+	    if self.respond_to?("#{name}=") 
+	      # write value to property		
+	      if name == :date and name.class == String
+		begin
+		  value = DateTime.strptime(value, "%d.%m.%Y")		  
+		rescue ActiveRecord::Transactions::TransactionError
+		  raise $!
+		rescue
+		  $log.warn{"#{value} not parsed as date (#{$!.message})."}
+		end
+	      end
+	      self.send("#{name}=",value) unless name == :message
+	    else
+	      # Maybe there is an anonymous field?
+	      if self.respond_to?("#{name}_anonym=") 
+		# yes - so anonymise that
+		nonym = Nonym.find_or_create(value)
+	      self.send("#{name}_anonym=",nonym.id)		  
+	      else
+		# no. Giving up!
+		throw "Neither field '#{name}' nor anonymous field '#{name}_anonym' found in class #{self.class.name}."
+	      end
+	    end
+	  end
+	end
+      end
+
+      self.after_filling_values(values) if self.respond_to?(:after_filling_values)
+      
+      # validation is too expensive, but we know the records are correct.
+      self.save_without_validation
+      
+      # special treatment of columns message and user	
+      if values and values.has_key?(:message) and values[:message]
+	msg = Message.new.prisma_initialize(self,values[:message], {:fast_association => true})		
+        msg.save_without_validation
+	
+	# write hash
+	if self.respond_to?(:hash_value)
+	  self.hash_value = self.get_hash
+	end
+#	msg.meta_id = self.id
+      end
+      
+      self.after_init(values) if self.respond_to?(:after_init)
+      self
+    end
+    
+    
+    # Reproducable means, that all data in the log is now stored in the
+    # database and thus the original message can be deleted. However,
+    # the raw/original message should still be accessible in archive)
+    def reproducable?
+      true
+    end
+    
+    # Main "parsing" function. When transform for a raw-message or any
+    # other meta message is called, all resulting meta message will be
+    # generated. Probably (if there is a transformation), the related
+    # message of this meta will be deleted.
+    def transform(options = {})
+      begin
+	default_transform(options)
+	#p ("---------" + SourceDbMeta.count.to_s)
+      rescue ActiveRecord::Transactions::TransactionError
+	raise $!
+      rescue
+	begin
+	  $log.error{"Transform class #{self.class.name}.#{self.id} threw error: #{$!}!" }
+	  msg = (self.message.msg[0,256] rescue nil)
+	  $log.error("Message is #{msg.inspect}") if msg
+	  $log.error("#{self.inspect}")
+	  for line in $!.backtrace
+	    $log.error{"#{line}"}
+	  end
+	  if not Prisma::Database.check_connections
+	    throw "Prisma connection failed!"
+	  end
+	  raise if options[:raise_exception]
+	rescue ActiveRecord::Transactions::TransactionError
+	  raise $!
+	rescue
+	  raise if options[:raise_exception]
+	  $log.fatal("Printing error threw exception #{$!}!") 
+	end
+      end
+    end
+        
+    # This method will be called from transform method. If you
+    # need special treatment, overwrite transform and not default
+    # transform. Default transform uses the regular expressions,
+    # and normally overwriting after_filling or after_init gives
+    # you enough power for your customisation.
+    def default_transform(options = {})
+      if $enable_dublette_recognition
+	if self.respond_to?(:hash_value) and self.hash_value 
+	  others = self.class.find(:all,:conditions => "hash_value = #{self.hash_value}")
+	  
+	  others.each { |other| 
+	    if other.id != self.id
+	      raw_meta1 = other.raw_meta
+	      raw_meta2 = self.raw_meta
+	      if raw_meta1.class != raw_meta2.class or raw_meta1.id != raw_meta2.id
+		$log.warn{"Found dublette #{other.id} (hash: #{self.hash_value}, msg:#{self.message.msg[0,100]})!"}
+		self.destroy
+		return
+	      end
+	    end
+	  }
+	end
+      end
+      
+      $log.debug{"Begin transform #{self.class.name}.#{self.id}."}
+
+      def msgs
+	# only return the only message
+	# if message_fast is set
+	if @message_fast
+	  yield message
+	return
+      end
+	begin
+	  messages.each {|m| yield m }
+	rescue LocalJumpError
+	  $log.debug("Got local jump error. giving block.")
+	  messages { |m|
+	    yield m
+	  }
+	end
+      end
+
+      count = 0
+      msgs { |message|
+	$log.debug("Processing message #{message.to_s}") if $log.debug?
+	$log.debug("Message '#{message.msg}'") if $log.debug? and message.msg.length < 200
+	reproducable = false
+	for klass in Prisma::Database.transformer_classes(self.class)
+	  $log.debug{"Transforming against #{klass.name}"}
+	  # (fp) enable this again if you want to go trough messages again
+	  #            if Meta.find_by_parent_id_and_meta_type_name(self.meta.id, klass.name) != nil and
+	  #                not message.class.name =~ /Raw$/ then
+	  #                  $log.info "Meta for #{klass.name} already exists." if $log.info?
+	  #              next
+	  #            end
+	  applyable = klass.applyable?(self, message)
+	  $log.debug{"Message applyable to #{klass}: #{if applyable then "yes" else "no" end}"}
+
+	  if applyable
+	    $log.debug{"Creating meta #{klass.name}"}
+	    new_meta = klass.create_meta(self, message)
+	    if new_meta
+	      count = count + 1
+	      $log.debug{"Transformation #{klass.name} returned #{new_meta}"}
+	      new_meta.transform(options)
+	      reproducable = new_meta.reproducable? or reproducable
+	    else
+	      $log.debug{"Transformation returned nil"}
+	    end
+	  end
+	end
+	$log.debug{"End transform #{self.class.name}.#{self.id}."}
+	$log.debug{"Transform result: repr(#{reproducable}) saveable(#{message.respond_to?(:save)})"}
+	if not reproducable and message.respond_to?(:save)
+	  $log.debug{"Saving message. #{message.id}"}
+	  throw "Message not reproducable but already frozen, probably deleted!" if message.frozen?
+	  message.create if message.new_record?
+	else 
+	  message.destroy unless message.new_record? or  message.class.name =~ /Raw$/ 
+	end
+      }
+      return count
+    end
+=begin    
+    def to_sss
+      c = ""
+      self.children {|ch| c += ch.to_s }
+      c = c[0..1023] if c.length > 1024 
+      unless self.class == SourceDbMeta
+	m = self.messages.to_s
+	m = m[0..1023] if m.length > 1024 
+      end
+      return "#{self.class.name}.#{self.id} Msgs<#{m}> Chld<#{c}>"
+    end
+=end   
+
+    # Yields all meta chilren of this class.
+    def children      
+      return [] if self.class.name =~ /.*Raw$/
+      $log.debug{"Looking for children of #{self.class.name}.#{self.id}."}
+      for klass in Prisma::Database.get_classes(:meta)
+	for column in klass.columns
+	  if column.name == "#{self.class.table_name}_id" then
+
+            limit = 10
+            current = 0
+            while current >= 0
+	      records = klass.find(:all,
+				   :conditions => "#{self.class.table_name}_id = #{self.id}", 
+				   :limit => limit, 
+				   :offset => current)
+	      for rec in records
+		yield rec
+	      end
+	      if records.length == limit
+		for rec in records
+		  # only increment to current if record has not been deleted
+		  current += 1 if !rec.frozen?
+		end
+	      else
+		current = -1
+	      end
+            end
+
+	  end
+	end
+      end   
+      return []
+    end
+
+    # Returns a array of all children
+    def children_a
+      a = []
+      children {|c| 
+	a.push(c)
+      }
+      a.push(message) if message
+      a
+    end
+
+    # Destroys the meta and all its children
+    # currently quite slow, needs reimplementation
+    # of meta classes modell.
+    def prisma_destroy
+      $log.debug{"Destroying #{self.inspect}"}
+      if Prisma::Database.speed_transaction_enabled      
+	$destroy_calls_speed ||= 0
+	$destroy_calls_speed += 1
+      else
+	$destroy_calls ||= 0
+	$destroy_calls += 1
+      end
+      unless self.class == Message
+	children {|c|
+	  Prisma::Database.speed_transaction(c) {
+	    c.prisma_destroy
+	  }
+	}
+	message.prisma_destroy if message
+      end
+      destroy
+    end
+
+    # Traverse all childern recursively and returns
+    # a array of them (breath-first)
+    def children_recursive
+      a = []
+      arr = children_a
+      arr.each {|c|
+	a += c.children_recursive
+      }
+      a + arr
+    end
+
+
+    # Build a query that will return all
+    # records that have the same "path" as 
+    # the current one. Used for building queries
+    # in the UI
+    def join_query(query=nil)
+      query = "#{self.class.table_name}" unless query
+      p = parent
+      if p then
+	query = "#{query} LEFT JOIN #{p.class.table_name} ON #{self.class.table_name}.#{p.class.foreign_column_name} = #{p.class.table_name}.id"
+	query = p.join_query(query)
+      end
+      return query
+    end  
+
+    # TODO: document this
+    def get_join
+      ret = "LEFT JOIN messages ON messages.meta_id = #{table_name}.id " +
+	"AND messages.meta_type_name = '#{self.class.name}' "
+      if self.class.respond_to?(:additional_columns)
+	previous_table_name = self.class.table_name
+	for table_class, table_columns in self.class.additional_columns
+            ret += "LEFT JOIN #{table_class.table_name} "
+	  ret += " ON #{table_class.table_name}.id = "
+	  ret += "    #{previous_table_name}.#{table_class.foreign_column} "
+	  previous_table_name = table_class.table_name
+	end
+      end
+      return ret
+    end
+    
+    # Count childern (one level only)
+    def children_count
+      i = 0
+      children { |c| i+=1}
+      return i
+    end
+
+    # return parent meta, works also for views by using
+    # class in source_table_class as parent class
+    def parent
+      if self.class.respond_to?(:sources)
+	self.class.sources.each { |klass_name|
+	  p = self.send(eval(klass_name).table_name.singularize)
+	  return p if p	
+	}
+      end
+      if self.class.name != "Message" and self.class.respond_to?(:parent_classes)
+	self.class.parent_classes.each { |klass|
+	  if self.respond_to?(klass.foreign_column_name) and (mid = self.send(klass.foreign_column_name))
+	    return klass.find(mid)
+	  end
+	}
+      end
+      view = View.get_class_from_tablename(self.class.table_name)
+      source_table_class = view.source_table_class if view
+      if source_table_class
+	return source_table_class.table.find(self.id)
+      end
+      return nil	
+    end
+
+    # Sets the parent instance
+    def parent=(val)
+      return nil unless self.class.respond_to?(:sources)
+      self.class.sources.each { |klass_name|
+	self.send("#{eval(klass_name).table_name}_id=",nil)
+      }
+      self.send("#{val.class.table_name}_id=",val.id) if val
+    end
+
+    # Finds the original message for this meta, default behaviour
+    # is to return parent.original. For every raw class there sould
+    # be a corresponding meta class which then implements the original
+    # function in detail
+    def original
+      p = parent
+      return [] unless p
+      p.original
+    end
+    # Returns a text describing original message (not the object)
+    def original_text
+      o = original[0]
+      case o
+      when String
+	"ALOIS_ERROR: #{$!}"
+      when NilClass
+	"NO ORIGINAL FOUND"
+      else
+	o.msg
+      end
+    end
+    
+    # returns a time string of the time field,
+    # thin this string is used for charting in that form
+    def time
+      begin
+	t = super
+	t.strftime("%H:%M:%S")
+      rescue ActiveRecord::Transactions::TransactionError
+	raise $!
+      rescue
+	super
+      end
+    end
+  end
+
+  # Include instance and class methods
+  def self::included other
+    other.module_eval  { include InstanceMethods }
+    other.extend ClassMethods
+    other
+  end
+
+end

Added: incubator/alois/trunk/prisma/lib/prisma/database.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/database.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/database.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/database.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,474 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Prisma
+    # based on .gem/ruby/1.8/gems/rails-2.3.2/lib/tasks/databases.rake 
+
+  class Database
+    PRISMA_DBS = ["prisma","pumpy"]
+    def self.load_all(options = {})
+      PRISMA_DBS.each {|db_name| 
+        Prisma::Database.load_classes(db_name, options)
+      }
+    end
+
+    def self.config_path; PRISMA_CONFIG_PATH; end
+    def self.data_path; PRISMA_DATA_PATH; end
+    
+    def self.normalize_config(db_root, config)
+      ret = {}
+      config.each {|key,val|
+        ret[key.to_s] = val.to_s
+      }
+      if ret["adapter"] =~ /sqlite/ and !Pathname.new(ret["database"]).absolute?
+        ret["database"] = (db_root + config["database"])
+        ret["database"] = (ret["database"].dirname.realpath + ret["database"].basename).to_s
+      end
+      ret
+    end
+    
+    def self.db_config(db_name, options = {})
+      @@configs ||= {}
+      return @@configs[db_name] if @@configs[db_name] and !options[:reload]
+
+      config_file = config_path + "#{db_name}_database.yml"
+      $log.debug("Loading connection #{db_name}[#{PRISMA_ENV.inspect}] from #{config_file}")
+      config = YAML::load(config_file.open("r"))[PRISMA_ENV]
+
+      if user = options[:force_user]
+        $log.info("Forcing user #{user}")
+        config["password"] = $ui.password("Overriding accessing user.\nPlease enter password of #{user} for db #{db_name}:")
+        config["username"] = options[:force_user]
+      end
+
+      $log.debug("Connection #{db_name}: #{config.inspect}")
+
+      @@configs[db_name] = normalize_config(data_path, config)
+      return @@configs[db_name]
+    rescue
+      raise "Error loading db config #{db_name} from #{config_file}: #{$!}"
+    end
+    
+    def self.connection_name(klass)
+      base_config = normalize_config(Pathname.pwd, klass.connection_pool.spec.config)
+      
+      ActiveRecord::Base.configurations.each {|name, config|
+        config = normalize_config(Pathname.pwd, config)
+        return name.to_s if base_config == config
+      }
+      return nil
+    end
+    
+    def self.with_db(db_name = nil, options = {})
+      load_connection(db_name, options) if db_name
+      
+      old_pool = ActiveRecord::Base.connection_handler.connection_pools["ActiveRecord::Base"]
+              
+      begin
+        ActiveRecord::Base.establish_connection(db_name) if db_name
+        yield        
+      ensure
+        # use the origin connetion again
+        ActiveRecord::Base.connection_handler.connection_pools["ActiveRecord::Base"] = old_pool
+      end
+    end
+
+    def self.connection_pool(db_name)
+      ActiveRecord::Base.connection_handler.connection_pools[db_name]
+    end
+    def self.connection(db_name)
+      pool = connection_pool(db_name)
+      pool.connection if pool
+    end
+
+    def self.reconnect
+      PRISMA_DBS.each {|db|
+        $log.debug("Reconnecting #{db}")
+        p = self.connection_pool(db)
+        unless p
+          $log.debug("No pool for#{db} found")
+          next 
+        end
+        p.disconnect!
+        p.with_connection {}
+      }
+    end
+
+    def self.check_connections(options = {})
+      PRISMA_DBS.reject {|db| check_connection(db)}.length == 0
+    end
+
+    def self.check_connection(db_name, options = {})
+      c = connection(db_name)
+      return true unless c
+      c.verify!
+      return true
+    rescue
+      $log.warn("Connection #{db_name} does not work")
+      return false
+    end
+    
+    def self.migrate_path(db_name)
+      self.data_path + "#{db_name}_database/migrate"
+    end
+    def self.migrate(db_name, version = nil, options = {})
+      with_db(db_name, options) {
+        #system("ls data/prisma/#{db_name}_database/migrate")
+        ActiveRecord::Migrator.migrate(migrate_path(db_name), version )
+      }
+    end
+    
+    def self.schema(db_name)
+      self.data_path + "#{db_name}_database/schema.rb"
+    end
+    def self.schema_load(db_name)
+      with_db(db_name) {
+        load(schema(db_name).to_s)
+      }
+    end
+    def self.schema_dump(db_name)
+      with_db(db_name) {
+        require 'active_record/schema_dumper'
+        File.open(schema(db_name).to_s, "w") do |file|
+          ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+        end
+      }
+    end
+    def self.check(db_name, options = {})
+      with_db(db_name, options) {
+        pending_migrations = ActiveRecord::Migrator.new(:up, migrate_path(db_name)).pending_migrations
+        
+        if pending_migrations.any?
+          puts "You have #{pending_migrations.size} pending migrations:"
+          pending_migrations.each do |pending_migration|
+            puts '  %4d %s' % [pending_migration.version, pending_migration.name]
+          end
+          raise "Pending migrations"
+        else
+          $log.info("No pending migrations for #{db_name}")
+        end        
+      }
+    end
+    
+    def self.load_connection(db_name, options = {})
+      return if ActiveRecord::Base.configurations[db_name] 
+      spec = self.db_config(db_name, options)
+      ActiveRecord::Base.configurations[db_name] = spec
+      
+      with_db {
+        ActiveRecord::Base.establish_connection(db_name)
+        new_pool = ActiveRecord::Base.connection_pool        
+        ActiveRecord::Base.connection_handler.connection_pools[db_name] = new_pool
+      }
+      
+      # if alois is available register this connection
+      if defined?(RAILS_ENV)
+        spec = {"name" => db_name}.update(spec).symbolize_keys
+        spec.delete(:reconnect)
+        conn = Connection.new(spec)
+        conn.register
+      end
+    end
+    
+    def self.load_classes(db_name, options = {})
+      load_connection(db_name, options)
+      #      migrate(db_name)
+      @@classes ||= {}
+      @@classes[db_name] = []
+      
+      Pathname.glob(data_path + "#{db_name}_database/model/*.rb").each {|file|
+        $log.debug("Requiring #{file.to_s}")
+        require file.to_s        
+        
+        class_name = file.basename.to_s[0..-4].camelize
+        
+        
+        $log.debug("Loading #{class_name}")
+        klass = eval(class_name)
+        
+        ActiveRecord::Base.connection_handler.connection_pools[klass.name] =
+          ActiveRecord::Base.connection_handler.connection_pools[db_name]
+        
+        @@classes[db_name] << klass unless @@classes[db_name].include?(klass)
+        #establish_connection "db1"    
+      }
+    end
+    
+    def self.get_classes(type = nil)
+      return @@classes.values.flatten if type.nil?
+      case type
+      when String
+        @@classes[type]        
+      when :raw
+        return [] unless @@classes["pumpy"]
+        @@classes["pumpy"].select {|c| c.name =~ /Raw$/ }
+      when :meta
+        return [] unless @@classes["prisma"]
+        @@classes["prisma"].select {|c| c.name =~ /Meta$/ }
+      when :message
+        return [] unless @@classes["prisma"]
+        @@classes["prisma"].select {|c| c.name =~ /Message$/ }
+      else
+        raise "Called get_classes with unexpected argument #{type.inspect}"
+      end
+    end
+    
+    # Returns all possible datasources. (Views + Alois Classes)
+    def Database.data_sources
+      (View.find(:all) + get_classes)
+    end
+    
+    # Returns the class from tablename, it the table
+    # is not one of alois base classes return nil.
+    def Database.get_class_from_tablename(tablename)
+      return nil unless tablename
+      name = tablename.singularize.camelize
+      for klass in get_classes
+        return klass if klass.name == "#{name}"
+      end
+      return nil
+    end
+
+    # Delete SourceMeta, record is a sourcemeta object,
+    # dryrun lists all records that sould be deleted.
+    def self.delete_source(record,dryrun)
+      if dryrun
+        childr = record.children_recursive
+        $log.info("Would delete: #{record.to_s} with #{childr.length} children")
+        if $log.debug?
+          childr.map{|c| "#{c.id}.#{c.class.name}"}.each {|c|
+            $log.debug("  with child: #{c}")
+          }
+        end
+      else
+        $log.info("Deleting: #{record.to_s}")
+        Prisma::Database.speed_transaction(record) {
+          record.prisma_destroy
+        }
+        #	record.children_recursive.each {|c|
+        #	  c.destroy
+        #	}
+        #	record.destroy
+        #      end
+      end
+    end
+    
+    # Remove logs before the given date. Action may be :all, :database or :archive.
+    # See delete-old-logs script for more information.
+    def self.delete_logs_before(action, date, dryrun = false)
+      # disable transaction because too much
+      # data concerned
+      old_adt = $alois_disabled_transaction
+      $alois_disabled_transaction = true
+      $log.info("Commiting #{$transaction_bundle_amount} operations together")
+      
+      if action == :all or action == :database
+        ArchiveMeta.find(:all, :conditions => [ "created_at < :date", {
+                                                  :date => date}]).each {|record|
+          delete_source(record,dryrun)
+        }
+        
+        SourceDbMeta.find(:all, :conditions => [ "created_at < :date", {
+                                                   :date => date}]).each {|record|
+	  delete_source(record,dryrun)
+        }
+      end
+      
+      if action == :all or action == :archive
+        # old archive
+        Dir.glob("/var/lib/prisma/archive/old*/*").each {|file|
+          mtime = File.mtime(file).strftime("%F")
+          if mtime  < date then
+            if dryrun
+              $log.info("Would delete: '#{file}' with mtime #{mtime}.")
+            else
+              $log.info("Deleting file '#{file}' with mtime #{mtime}.")
+              begin
+                File.delete(file)
+              rescue ActiveRecord::Transactions::TransactionError
+                raise $!
+              rescue
+                $log.error("Could not delete '#{file}': #{$!.to_s}")
+              end
+            end
+          end	
+        }
+        
+        # new archive
+        files = Dir.glob($archive_pattern.to_s.gsub(/\%./,"*"))
+        files += Dir.glob($archive_pattern.to_s.gsub(/\%./,"*") + ".gz")
+        
+        archive_root = nil
+        if $archive_pattern.to_s =~ /^([^\%]*\/)\%/
+          archive_root = Pathname.new($1).realpath.to_s
+        else
+          throw "Could not get root of archive_path."
+        end
+        
+        regex =  Regexp.new(Regexp.escape($archive_pattern.to_s).gsub(/\%i/,"([^\/]*)").gsub(/\%./,"[^\/]*") + "(\.gz)?")
+        files.each {|file|
+          if file =~ regex
+            if $1 < date then
+              if dryrun
+                $log.info("Would delete: '#{file}' with incoming date #{$1}.")
+              else
+                $log.info("Deleting: '#{file}' with incoming date #{$1}.")
+                begin
+                  File.delete(file)
+                  
+                  dir = Pathname.new(file).parent.realpath.to_s	      
+                  while Dir.glob(dir + "/*").length == 0 and not dir == archive_root
+                    # directory empty delete
+                    $log.info("Removing emtpy dir '#{dir}'.")
+                    Dir.rmdir(dir)
+                    dir = Pathname.new(dir).parent.realpath.to_s
+                  end
+                rescue ActiveRecord::Transactions::TransactionError
+                  raise $!
+                rescue
+                  $log.error("Could not delete '#{file}': #{$!.to_s}")
+                end
+                
+              end	    
+            end
+          else
+            $log.error("Could not determine incoming date of file '#{file}'.")
+          end
+        }
+
+        #	cmd = "/usr/bin/find #{archive_root} -type d -empty -exec rmdir {} \\;"
+        #	if dryrun
+        #	  $log.info("Cleaning up. Would delete emtpy directories in '#{archive_root}'")
+        #	  $log.info("Would execute: '#{cmd}'")
+        #	else
+        #	  $log.info("Cleaning up. Deleting emtpy directories in '#{archive_root}'")
+        #	  $log.info("Exec '#{cmd}'")
+        #	  exec(cmd)
+        #	end
+        #      else
+        #	$log.error("Could not get root archive path.")
+        #      end
+        
+      end
+      $alois_disabled_transaction = old_adt    
+    end
+    
+    def Database.transaction(klass = nil)
+      klass ||= ActiveRecord::Base
+      if $alois_disabled_transaction
+        yield
+      else
+        $log.debug("Begin transaction on #{klass.name}")
+        klass.transaction do 
+          yield
+        end
+        $log.debug("End transaction on #{klass.name}")
+      end
+    end
+    
+  # Returns all classes that can transform messages (classes with parsing functionality)
+  def Database.transformer_classes(klass)
+    classes = get_classes(:meta).select { |meta_klass|
+      meta_klass.can_transform?(klass)
+    }
+    $log.debug{"Transformer classes for class #{klass}: #{classes.map {|c| c.name}.inspect}"}
+    classes
+  end
+  
+  def Database.speed_transaction_enabled
+    # disabled, becaus does not work with sqlite
+    false # $alois_disabled_transaction
+  end
+  # the idea here is to pack some sql calls
+  # together with transactions and call commit
+  # after eg 500 calls, so the cost of the
+  # individual commits are "packed together"
+  def Database.speed_transaction(klass = nil)
+    unless Prisma::Database.speed_transaction_enabled
+      # other transactions are used, this
+      # implementation cannot handle that
+      yield
+      return
+    end
+
+    # default settings
+    klass ||= ActiveRecord::Base
+    $transaction_bundle_amount ||= 500
+    
+    # for easier writing
+    tb = $transaction_bundle
+    unless tb
+      $log.debug{"Initialize transaction bundle"}
+      # this is the first call of speed_transaction
+      $transaction_bundle = {
+	:connection => klass.connection,
+	:count => 0,
+	:yields => 0,
+	:open_transaction => false
+      }
+      tb = $transaction_bundle
+    else
+      $log.debug{"Transaction bundle connections are equal: #{tb[:connection] == klass.connection}"}
+      # check if the transaction is on the same 
+      # connection, if not, return
+      unless tb[:connection] == klass.connection
+ 	yield
+	return
+      end
+    end
+    
+    unless tb[:open_transaction]
+      $log.debug("No transaction open for bundle, open one")
+      # start transaction if none is open
+      tb[:connection].send(:begin_db_transaction) 
+      tb[:count] = 0
+      tb[:start_time] = Time.now
+      tb[:open_transaction] = true
+    end
+    
+    tb[:count] += 1
+    tb[:yields] += 1
+    yield
+    tb[:yields] -= 1
+    
+    if tb[:open_transaction] and (tb[:count] >= $transaction_bundle_amount or 
+	tb[:yields] == 0)
+      tb[:connection].commit_db_transaction
+      
+      if $destroy_calls_speed
+	$destroy_calls ||= 0
+	$destroy_calls += $destroy_calls_speed 
+	$destroy_calls_speed = 0
+      end	
+      
+      tb[:open_transaction] = false
+      duration = Time.now - tb[:start_time]
+      Prisma::Util.perf{"speed_transaction: Had #{duration}s for #{tb[:count]} (#{tb[:count].to_f/duration} per sec)"}
+    end
+    
+    if tb[:yields] == 0
+      # this is the last call of speed
+      # transactions, reset global var
+      # and tidy up
+      # ps: transaction sould already be
+      #     commited above
+      raise "Transaction not yet commited" if
+	tb[:open_transaction] 
+      $transaction_bundle = nil
+    end
+  end
+
+    
+  end
+end

Added: incubator/alois/trunk/prisma/lib/prisma/prisma.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/prisma.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/prisma.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/prisma.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,21 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Prisma
+
+  def init
+    
+  end
+
+end

Added: incubator/alois/trunk/prisma/lib/prisma/prisma_orig.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/prisma_orig.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/prisma_orig.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/prisma_orig.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,104 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This class contains basic, static functionalities that
+# can be run from other places in alois or from scripts.
+class PrismaOLD
+  require 'zlib'
+
+  def Prisma.filter_class
+    # workaround for controllers that cannot access
+    # class Filter because of the class ActionController::Filters::Filter
+    Filter
+  end
+
+    
+  
+  # Return all alois base classes. Raw/Meta/Message and Alois Models
+  def Prisma.get_classes(type = :all)
+    unless $all_classes_loaded
+      Dir.glob(RAILS_ROOT + "/app/models/*.rb").map {|c|
+	require c
+      }
+      $all_classes_loaded = true
+    end
+    @klasses = subclasses_of(ActiveRecord::Base).select {|klass| klass.table_exists? } unless @klasses
+
+    ret = @klasses.reject { |klass|
+      !((klass.name =~ /(.*)Raw$/ and type == :raw) or
+        (klass.name =~ /(.*)Meta$/ and type == :meta) or
+        (klass.name =~ /(.*)Message$/ and type == :message) or
+         type == :all) or
+        klass.name =~/GenericRecord/ or
+        klass.name =~/BaseSource/ or
+	!klass.respond_to?(:table)
+    }
+    return ret
+  end
+
+  # Common function to check if instance is productive
+  def Prisma.is_productive?
+    RAILS_ENV=="production"
+  end
+
+
+
+  # Return all configured connections.
+  def Prisma.connections
+    Prisma.get_classes(:all).map{ |klass| klass.connection }.uniq
+  end
+  # Reconnect all connetions, use this if you for or daemonize
+  # processes.
+  def Prisma.reconnect
+return
+    Prisma.connections.each {|c| c.disconnect!}
+    Prisma.check_connections
+  end
+  
+  # Check if all classes can be accessed
+  def Prisma.check_connections
+    # IS THIS NOW OBSOLETE??
+    # do only check connection in production
+    # mode, eg in testing the transactional
+    # fixtures would be messed up
+    #    return true unless RAILS_ENV == "production"
+
+    $log.info "Connection check"
+    ret = Prisma.get_classes(:all).map{ |klass| 
+      begin
+	c = klass.connection
+	c.verify!
+	nil
+      rescue ActiveRecord::Transactions::TransactionError
+	raise $!
+      rescue  
+	"  #{$!.message}"
+      end
+    }.uniq.compact
+    if ret.length == 0
+      $log.info{"  All connections active."}
+      true
+    else
+      for msg in ret 
+	$log.warn(msg)
+      end
+      false
+    end
+  end
+  
+  # Load default views, this can be done after a fresh installation
+  def Prisma.load_default_working_items
+    iew.load_from_yaml("#{RAILS_ROOT}/config/default_working_items/views.yaml") rescue [$!.to_s]
+  end
+end

Added: incubator/alois/trunk/prisma/lib/prisma/transform.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/lib/prisma/transform.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/lib/prisma/transform.rb (added)
+++ incubator/alois/trunk/prisma/lib/prisma/transform.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,185 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Prisma
+
+  class Transform
+    # Transform all raws from the given raw class
+    def self.transform_all_raws(raw_class)
+      source = SourceDbMeta.new.prisma_initialize(:all, raw_class)
+      source.transform
+    end
+    
+    # Creates a new FileRaw and transform content (use this to import
+    # log files)
+    def self.transform_file(filename,options = nil)
+      source = FileRaw.create(filename,options)
+    end
+    
+
+  
+  def Prisma.transform_messages(limit = 100)
+
+#    [ 'HUP' , 'INT' , 'TERM', 'USR1', 'USR2'].each {|s|
+#      Signal.trap(s) {
+#	$log.error("SIGNAL---#{s}") rescue $log.error("aaaaaaaaaasldkfj")
+#      }
+#    }
+#    Signal.trap("TERM") do
+#      # End current tranform.
+#      $log.error "Caught #{Process.pid} signal TERM." if $log.info?
+#      $log.error "Going to stop..." if $log.info?
+#      $terminate = true
+#    end
+    $terminate = false
+
+    last_id = 0
+    total = 0
+    iter = Message.find(:all,:conditions => "ID>#{last_id}" ,:limit => limit,:order=>"id")
+    while iter.length > 0 and not $terminate
+      main_cost = Benchmark.measure {
+	iter.each { |message|
+	  $log.info("Transforming message #{message.id}") if $log.info?
+	  meta = message.parent
+	  if meta == nil then
+	    $log.warn("Message has no meta entry.") if $log.warn?
+	  else
+	    Prisma.transaction(meta) do
+	      meta.transform	    
+	    end
+	  end
+	  last_id = message.id
+	}
+      }.real
+      
+      total = total + iter.length
+
+      Prisma.perf {"Done #{limit} in #{main_cost}s (#{limit/ main_cost}/s)."}
+      Prisma.perf {"Current message is #{last_id} done #{total}."}
+
+      iter = Message.find(:all,:conditions => "ID>#{last_id}" ,:limit => limit,:order=>"id")
+    end
+  end
+  # TODO: comment transform_queues function
+  # transform all available dbsources
+  def self.transform_queues(type = :fifo, count = nil, waiting_time=nil)
+    if $do_not_run_prisma
+      $log.error{ "Will not start primsa queues cause option do_not_run_prisma is defined"}
+      return
+    end
+    pids = []
+    for klass in get_classes(:raw)
+      $log.warn("Starting queue for class #{klass.name}.")
+      pid = fork do
+	define_new_logger(klass.name)
+        Prisma.reconnect
+
+#	Signal.trap("USR1") do
+#	  # End current transform and begin new transform
+#	  $log.info "Child #{Process.pid} caught signal USR1" if $log.info?
+#	  $log.info "Going to restart..." if $log.info?
+#	  $restart = true
+#	  $terminate = true
+#	end
+	Signal.trap("TERM") do
+	  # End current tranform.
+	  $log.warn{"Caught #{Process.pid} signal TERM."}
+	  $log.warn{"Going to stop..."}
+	  $restart = false
+	  $terminate = true
+	end
+
+	source = nil
+	$restart = true
+	while $restart
+	  $terminate = false
+	  $restart = type != :all
+	  begin
+	    $log.info "Process last records of class #{klass.name}, #{count} per step" if $log.info?
+	    if source 
+	      source.finished = true
+	      source.save	      
+	    end
+	    source = SourceDbMeta.new.prisma_initialize(type, klass, count,nil, false, waiting_time)
+	    $enable_dublette_recognition = source.may_contain_dublettes
+	    source.transform
+	  rescue ActiveRecord::Transactions::TransactionError
+	    raise $!
+	  rescue	    
+	    $log.error{ "Processing class #{klass.name} threw error #{$!}!" }
+	    for line in $!.backtrace
+	      $log.error{"#{line}"}
+	    end
+	    if Prisma::Database.check_connections
+	      $log.fatal{"Connections are good, so something bad happened. Will not risk to restart queue #{klass.name}."}
+	      $terminate = true
+	      $restart = false
+	    else
+	      $log.info{"At least one prisma connection is down."}
+	      connection_wait_count = 1
+	      while not (Prisma::Database.check_connections or $terminate)
+		wait_time = connection_wait_count
+		wait_time = 30 if wait_time > 30 
+		$log.warn{"#{connection_wait_count} Waiting #{wait_time} seconds."}
+		Prisma.save_sleep(wait_time)
+		connection_wait_count += 1
+	      end
+	      if !$terminate
+		$log.info{"Connection are good again. Restarting queue #{klass.name}."}		
+	      end
+	    end
+	  end
+	end
+	$log.info "Child #{Process.pid} end." if $log.info?
+        $log.info "Stopped processing class #{klass.name}." if $log.info?
+      end
+      pids.push(pid)
+    end
+#    Signal.trap("USR1") do
+#      # End current transform and begin new transform
+#      $log.info "Caught signal USR1" if $log.info?
+#      $log.info "Going to restart prismas." if $log.info?
+#      for pid in pids
+#	Process.kill("USR1",pid)
+#      end
+#    end
+    Signal.trap("TERM") do
+      # End current tranform.
+      $log.warn{"Caught signal TERM."}
+      $log.warn{"Going to stop prismas."}
+      $terminate = true
+      $log.error{"No pids found."} if pids.nil? or pids.length == 0
+      for pid in pids
+	$log.warn{"Sending term to #{pid}."}
+	Process.kill("TERM",pid)
+      end
+    end
+
+    $log.debug "Parent process waiting." if $log.debug?
+    while !$terminate
+      sleep 1
+    end
+    $log.warn{"Going to wait for children."}
+    for pid in pids
+      ret = Process.wait
+      $log.warn{"Child returned with:#{ret}"}
+      $log.error{"Child #{ret} returned without shutdown!"} if not $terminate and not type==:all
+    end
+    $log.warn{"Prisma main process ended."}
+  end
+
+
+
+  end
+end

Added: incubator/alois/trunk/prisma/pumpy_database.sqlite3
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/pumpy_database.sqlite3?rev=1031127&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/alois/trunk/prisma/pumpy_database.sqlite3
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream



Mime
View raw message