incubator-alois-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fla...@apache.org
Subject svn commit: r1031127 [4/22] - in /incubator/alois/trunk: ./ bin/ debian/ doc/ etc/ etc/alois/ etc/alois/apache2/ etc/alois/environments/ etc/alois/prisma/ etc/cron.d/ etc/default/ etc/logrotate.d/ prisma/ prisma/bin/ prisma/conf/ prisma/conf/prisma/ pr...
Date Thu, 04 Nov 2010 18:27:42 GMT
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,54 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class PostfixDetailMeta < ActiveRecord::Base
+
+    description "Parsed postfix detail messages."
+    sources ["PostfixMeta"]
+
+    def self.may_have_messages?; false; end
+
+    def self.expressions
+      ret = []
+
+      ret.push({ :regex => /^size=(\d*), nrcpt=(\d*) \(([^\)]*)\)( *)$/,
+        :fields => [:size, :nrcpt, :status, nil]})
+
+      #to=<test@mail.example.com>, orig_to=<postmaster@example.com>, relay=local, delay=0, status=sent (delivered to command: /usr/bin/procmail -a "$EXTENSION" DEFAULT=$HOME/Maildir/ MAILDIR=$HOME/Maildir)
+      #to=<test@mail.example.com>, orig_to=<admin@example.com>, relay=mail2.example.com[192.168.123.123], delay=1, status=sent (250 Ok: queued as 12AF7B902B8)
+
+      #to=<archive@mail.logintas.ch>, orig_to=<archive_mail@mail.logintas.ch>, relay=local, delay=0.41, delays=0.24/0/0/0.17, dsn=2.0.0, status=sent (delivered to command: /usr/bin/procmail -a "$EXTENSION" DEFAULT=$HOME/Maildir/ MAILDIR=$HOME/Maildir)
+      ret.push({ :regex => /(orig_to=<([^>]*)>, )?relay=([^\[,]*)(\[([^\]]*)\])?(\:(\d+))?, delay=(\d+\.?\d*), (delays=(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*), )?(dsn=([^\,]*), )?status=([^ ]*) \((.* ([^ ]+) as ([A-F0-9]{11}))\) *$/,
+        :fields => [nil, :orig_to, :relay_host, nil, :relay_ip, nil, :relay_port, :delay,nil,:delay_before_qmgr, :delay_in_qmgr ,:delay_conn_setup, :delay_transmission, nil, :dsn, :status, :result_text, :result, :result_mail_id]})
+
+      # this is the same as above without the last .. as .. regexp (result_text only)
+      ret.push({ :regex => /(orig_to=<([^>]*)>, )?relay=([^\[,]*)(\[([^\]]*)\])?(\:(\d+))?, delay=(\d+\.?\d*), (delays=(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*), )?(dsn=([^\,]*), )?status=([^ ]*) \((.*)\) *$/,
+        :fields => [nil, :orig_to, :relay_host, nil,:relay_ip, nil,:relay_port, :delay,nil,:delay_before_qmgr, :delay_in_qmgr ,:delay_conn_setup, :delay_transmission, nil, :dsn, :status, :result_text]})
+
+# to=<archive_mail@mail.logintas.ch>, relay=127.0.0.1[127.0.0.1]:10024, delay=5, delays=0.03/0/0/4.9, dsn=2.6.0, status=sent (250 2.6.0 Ok, id=31764-06, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as C1C991F0FA9)=> nil
+
+
+#      ret.push({ :regex => /^to=<([^>]*)>, (orig_to=<([^>]*)>, )?relay=([^\[,]*)(\[[^\]]*\])?, delay=(\d+\.?\d*), (delays=(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*), )?(dsn=([^\,]*, )?status=([^ ]*) \(.*\)$/,
+#        :fields => [:to, nil, :orig_to, :relay_host, :relay_ip, :delay,nil,:delay_before_qmgr, :delay_in_qmgr ,:delay_conn_setup, :delay_transmission, nil, :dsn, :status, :result_text]})
+
+
+      #to=<archive_mail@mail.logintas.ch>, relay=127.0.0.1[127.0.0.1]:10024, delay=5.7, delays=0.1/0/0/5.6, dsn=2.6.0, status=sent (250 2.6.0 Ok, id=15500-09, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 76B191F10AA)
+
+
+# (delivered to command: /usr/bin/procmail -a "$EXTENSION" DEFAULT=$HOME/Maildir/ MAILDIR=$HOME/Maildir)
+# (250 2.6.0 Ok, id=15500-09, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 76B191F10AA)
+      return ret
+    end
+
+  end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,45 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class PostfixMeta < ActiveRecord::Base
+
+    description "Postfix Logs"
+    sources ["PureMeta", "LogMeta"]
+    
+    preseed_expression /^postfix\//
+
+    def self.expressions
+      ret = []
+
+      ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: (connect|disconnect) from ([^\[]*)\[([^\]]*)\]( *)$/,
+        :fields => [:program, :process_id, :action, :host, :ip, nil]})
+
+      ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (removed)( *)$/,
+        :fields => [:program, :process_id, :mail_message_id, :action, nil]})
+
+      ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ((warning):.*)$/,
+        :fields => [:program, :process_id, :message, :action]})
+
+      ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (client)=([^\[]*)\[([^\]]*)\]( *)$/,
+        :fields => [:program, :process_id, :mail_message_id, :action, :host, :ip, nil]})
+
+      ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (from|to)=<([^@]+)@([^\>]+)>, (.*)$/,
+        :fields => [:program, :process_id, :mail_message_id,:action, :email_local_part, :email_domain, :message]})
+
+      ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (message-id)=<([^@]+)@([^\>]+)>$/,
+        :fields => [:program, :process_id, :mail_message_id, :action, :message_id, :email_domain]})
+      return ret
+    end
+
+  end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,46 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class PureMeta < ActiveRecord::Base
+
+    description  "Meta record for a single message line"
+    sources ["FileMeta","CompressionMeta"]
+    
+
+    def self.create_meta( meta_message, message)
+      return nil if CompressionMeta.applyable?(meta_message,message)
+
+      case meta_message
+      when FileMeta, CompressionMeta
+	$log.debug{"Filetype of file meta is '#{meta_message.filetype}'."}
+	if meta_message.filetype == "log" or meta_message.filetype == "syslog" then
+	  
+	  if meta_message == FileMeta
+	    # check if the message has correct size
+	    throw "Message length '#{message.msg.length}' not equal to original file size '#{meta_message.size}'!" if
+	      message.msg.length != meta_message.size
+	  end
+	  for line in message.msg
+	    $log.debug("Pure Meta created:'#{line}'") if $log.debug?
+	    pure = PureMeta.new.prisma_initialize(meta_message, {:message=>line})
+	    pure.transform
+	  end      
+	end
+ 	meta_message.message = nil
+	return nil
+      end
+      return nil
+    end
+  end
+

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,220 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class SourceDbMeta < ActiveRecord::Base
+    description "Metadatas for database sources (pumpy raw table imports)."
+    
+    def raw_class
+      return @raw_class if @raw_class
+      @raw_class = eval "#{self.raw_class_name}"
+      return @raw_class
+    end
+    
+    def prisma_initialize( p_type, r_class, count=nil, current = nil, execute_once = false, waiting_time = nil )
+      if not waiting_time 
+	if r_class.respond_to?(:default_waiting_time) then
+	  waiting_time = r_class.default_waiting_time
+	else
+	  waiting_time = $default_waiting_time || 5
+	end
+      end
+
+      if not count
+	if r_class.respond_to?(:default_count) then
+	  count = r_class.default_count
+	else
+	  count = $default_count || 100
+	end
+      end
+      
+      self.process_type = p_type
+      self.start = current
+      self.total = 0
+      self.current = current
+      self.count = count
+      @raw_class = r_class
+      self.raw_class_name = r_class.name
+      self.execute_once = execute_once
+      self.waiting_time = waiting_time
+      self.save
+
+      @archivator = nil
+      throw "No archive pattern defined! Please specify in the config." unless $archive_pattern
+      
+      @archivator = Archivator.new($archive_pattern, r_class)      
+      self
+    end
+    
+    def may_contain_dublettes
+      return true if raw_class.respond_to?(:may_contain_dublettes) and
+	raw_class.may_contain_dublettes
+      return false
+    end
+
+    def messages
+      skip_count = 0
+      max_count_time_per_iteration = 0.1
+
+      return nil if self.finished
+      raise LocalJumpError unless block_given?
+
+      profile = false
+      require 'ruby-prof' if profile
+      begin
+	m_table = eval "#{raw_class}.table_name"
+	
+	while not $terminate
+          RubyProf.start if profile
+	  case self.process_type
+	  when :fifo, :firsts
+	    query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id ASC LIMIT #{self.count}"
+	  when :lasts
+	    query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id DESC LIMIT #{self.count}"
+	    # don't know if this is still correct (id > self.ucrrent?)
+	    #when :from_id
+	    #if self.current == nil then
+	    #  query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id ASC LIMIT 1"
+	    #else
+	    #  query = "SELECT #{m_table}.* FROM #{m_table} WHERE id > #{self.current} ORDER BY ID ASC LIMIT #{self.count}"
+	    #end
+	  when :all
+	    if self.current == nil then
+	      query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id ASC LIMIT #{self.count}"
+	    else
+	      query = "SELECT #{m_table}.* FROM #{m_table} WHERE id > #{self.current} ORDER BY id ASC LIMIT #{self.count}"
+	    end
+	  end
+	  
+	  $log.debug("Getting messages with #{query}") if $log.debug?
+	  iter = raw_class.send(:find_by_sql,query)
+	  loop_count = 0
+	  main_start = Time.now
+
+	  first = iter.first
+	  last = iter.last
+	  
+	  begin
+	    Prisma::Database.transaction(self) do
+	      iter.each { |raw_message|
+		loop_count += 1
+#		cost = Benchmark.measure {		
+		  @archivator.archivate(raw_message) if @archivator		
+		  yield raw_message
+		  self.current = raw_message.id		  
+#		}.real
+		#p "#{cost}s: #{raw_message.inspect}"
+		if $terminate
+		  $log.debug { "db_source, going to terminate."}
+		  break
+		end
+	      }
+	    end
+	  rescue ActiveRecord::Transactions::TransactionError
+	    self.finished = true
+	    self.save
+	    raise $!
+	  end
+	  
+	  if first and last
+	    Prisma::Database.transaction(raw_class) do
+	      if self.process_type == :lasts
+		# ex: 100 <= id AND id <= 200
+		raw_class.delete_all "#{first.id} >= id AND id >= #{last.id}"
+	      else
+		# ex: 100 <= id AND id <= 200
+		raw_class.delete_all "#{first.id} <= id AND id <= #{last.id}"
+	      end
+	    end
+	  end
+	  @archivator.close_unused_files if @archivator
+
+	  main_ende = Time.now
+	  main_cost = main_ende - main_start
+
+          done_one = loop_count > 0
+          if self.process_type == :all and not done_one or self.execute_once
+            self.finished=true
+            self.save
+            return
+          end
+          if not done_one then
+            Prisma::Util.perf{"No new record in table #{m_table}. Waiting #{self.waiting_time} seconds."}
+	    count, rest = self.waiting_time.divmod(5)
+	    count.times {
+	      Prisma::Util.save_sleep(5)
+	      # keep connection alive
+	      raw_class.find(:first)
+	      self.save
+	      break if $terminate
+	    }
+	    Prisma::Util.save_sleep(rest)
+	    
+          else
+            self.start = self.current if self.start == nil
+	    case self.process_type
+	    when :lasts, :fifo, :firsts
+	      if skip_count > 0
+		skip_count -= 1
+		Prisma::Util.perf{"Slow count, skipping todo counting for another #{skip_count} iterations"}
+	      else		
+		t = self.todo
+		count_time = Benchmark.measure {
+		  case self.process_type
+		  when :fifo, :firsts
+		    self.todo = raw_class.connection.select_value("select (select id from #{raw_class.table_name} order by id desc limit 1) - (select id from #{raw_class.table_name} limit 1)").to_i
+		  else
+		    self.todo = raw_class.count()
+		  end
+		}.real
+		
+		Prisma::Util.perf{"Todo #{self.todo}, todo delta: #{self.todo - t}"} if t
+
+		if count_time > max_count_time_per_iteration
+		  skip_count = (count_time / max_count_time_per_iteration).to_i
+		  Prisma::Util.perf{"Counttime is #{count_time} > #{max_count_time_per_iteration} Skip counting for #{skip_count} iterations"}
+		end
+	      end
+	    when :from_id
+	      self.todo = raw_class.count(:conditions => "id > #{self.current}")
+	    end
+            self.total = self.total + loop_count
+            Prisma::Util.perf {"Done #{loop_count} in #{main_cost}s (#{loop_count / main_cost}/s)."}
+	    Prisma::Util.perf {"Current record is #{self.current} done #{self.total}."}
+          end
+          self.save
+	  #        end
+          if profile
+            result = RubyProf.stop
+            printer = RubyProf::FlatPrinter.new(result)	 
+            str = StringIO.new
+            printer.print(str , :min_percent => 3)
+            str.string.split("\n").each {|l| $log.perf{l}}
+          end
+	end
+	$log.debug{"Finising source_db #{self.id}"}
+	self.finished = true
+	self.save
+      ensure
+	@archivator.close_all_files if @archivator
+      end
+      $log.info("End transform source db meta with raw class: #{raw_class_name}")
+    end
+
+    def to_s
+      "<SourceDbMeta #{total} x #{raw_class_name} (#{created_at.strftime("%F %T")})>"
+    rescue
+      super
+    end
+    
+  end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,165 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class SquidMeta < ActiveRecord::Base
+  description "Parsed squid messages"
+  sources ["PureMeta", "LogMeta"]
+  def self.may_have_messages?; true; end
+
+# from: http://www.squid-cache.org/Versions/v2/2.6/cfgman/logformat.html
+#     Usage:
+#
+#        logformat <name> <format specification>
+#
+#        Defines an access log format.
+#
+#        The <format specification> is a string with embedded % format codes
+#
+#        % format codes all follow the same basic structure where all but
+#        the formatcode is optional. Output strings are automatically escaped
+#        as required according to their context and the output format
+#        modifiers are usually not needed, but can be specified if an explicit
+#        output format is desired.
+#
+#                % ["|[|'|#] [-] [[0]width] [{argument}] formatcode
+#
+#                "       output in quoted string format
+#                [       output in squid text log format as used by log_mime_hdrs
+#                #       output in URL quoted format
+#                '       output as-is
+#
+#                -       left aligned
+#                width   field width. If starting with 0 the
+#                        output is zero padded
+#                {arg}   argument such as header name etc
+#
+#        Format codes:
+#
+#                >a      Client source IP address
+#                >A      Client FQDN
+#                >p      Client source port
+#                <A      Server IP address or peer name
+#                la      Local IP address (http_port)
+#                lp      Local port number (http_port)
+#                ts      Seconds since epoch
+#                tu      subsecond time (milliseconds)
+#                tl      Local time. Optional strftime format argument
+#                        default %d/%b/%Y:%H:%M:%S %z
+#                tg      GMT time. Optional strftime format argument
+#                        default %d/%b/%Y:%H:%M:%S %z
+#                tr      Response time (milliseconds)
+#                >h      Request header. Optional header name argument
+#                        on the format header[:[separator]element]
+#                <h      Reply header. Optional header name argument
+#                        as for >h
+#                un      User name
+#                ul      User name from authentication
+#                ui      User name from ident
+#                us      User name from SSL
+#                ue      User name from external acl helper
+#                Hs      HTTP status code
+#                Ss      Squid request status (TCP_MISS etc)
+#                Sh      Squid hierarchy status (DEFAULT_PARENT etc)
+#                mt      MIME content type
+#                rm      Request method (GET/POST etc)
+#                ru      Request URL
+#                rv      Request protocol version
+#                ea      Log string returned by external acl
+#                <st     Reply size including HTTP headers
+#                >st     Request size including HTTP headers
+#                st      Request+Reply size including HTTP headers
+#                %       a literal % character
+#
+#  logformat squid      %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
+#  logformat squidmime  %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt [%>h] [%<h]
+#  logformat common     %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st %Ss:%Sh
+#  logformat combined   %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
+  
+  def self.expressions
+    ret = []
+   
+    # Jun 10 06:38:19 hill squid[2573]: 1213072699.830      0 192.168.61.210 TCP_NEGATIVE_HIT/404 628 GET http://www.ossim.net/download/debian/Release.gpg - NONE/- text/html
+
+    #                                        squid combined     combined_logintas
+    # seconds_since_epoch: %ts                 *   (local_time)
+    # subsecond_time:      %03tu               *      
+    # response_time_milliseconds: %6tr         *                    *
+    # client_source_ip:    %>a                 *      *
+    # request_status:      %Ss                 *      *
+    # http_status_code:    %03Hs               *      *
+    # reply_size:          %<st                *      *
+    # request_method:      %rm                 *      *
+    # request_url:         %ru                 *      *
+    # user_name            %un                 *      *
+    # hierarchy_status:    %Sh                 *      *
+    # server_ip:           %<A                 *                    *
+    # mime_type:           %mt                 *                    *
+    #COMBINED NEW:
+    # user_indent:         %ui                        *
+    # protocol_version:    %rv                        *
+    # referer:             %{Referer}>h               *
+    # user_agent:          %{User-Agent}>h            *
+    #LOGINTAS COMBINED NEW:
+    # user_auth:           %ul
+    # user_acl:            %ue
+    # acl_log:             %ea
+    # client_fqdn:         %>A
+
+    
+
+    #  logformat squid      %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
+    ret.push({ :regex => /^(squid\[([^\]]+)\]:\s+)?(\d+)\.(\d\d\d)\s+(\d+) ([^ ]+) ([^\/]+)\/(\d+) (\d+) ([^ ]+) ([^ ]+) ([^ ]+) ([^\/]+)\/([^ ]+) (.+)\s*$/,
+	       :fields => [nil,:process_id,:seconds_since_epoch, :subsecond_time, :response_time_milliseconds, 
+		 :client_source_ip, :request_status, :http_status_code, :reply_size, 
+		 :request_method, :request_url, :user_name, :hierarchy_status, :client_fqdn, :mime_type]
+	     })
+
+    # one of syslogs default format
+    # squid combined format:
+    # logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
+    # 192.168.61.77 - - [29/Sep/2009:06:34:35 +0200] "GET http://debian.setup.in.here/mgmt-sarge/Release.gpg HTTP/1.0" 304 271 "-" "Debian APT-HTTP/1.3" TCP_REFRESH_HIT:DIRECT
+    ret.push({ :regex => /^(\d+\.\d+\.\d+\.\d+) ([^ ]+) ([^ ]+) \[([^\]]*)\] \"([^ ]+) ([^ ]+) HTTP\/([^\"]+)\" (\d+) (\d+) \"([^\"]*)\" \"([^\"]*)\" ([^:]+):([^ ]+) ?(.*)\n?$/,
+	       :fields => [:client_source_ip, :user_indent, :user_name, :seconds_since_epoch, 
+		 :request_method, :request_url, :protocol_version, :http_status_code, :reply_size, :referer, :user_agent,
+		 :request_status, :hierarchy_status, :message],
+	       :result_filter => lambda {|results, meta_instance|
+		 results[3] = Time.parse(DateTime.strptime(results[3],"%d/%b/%Y:%H:%M:%S %Z").to_s).to_i
+		 results
+	       }
+
+	     })
+  end
+  
+  def after_filling_values(values)
+    # get protocol and host from url
+    if %r{^(\w+)\://([^/]+)(/.*|$)$} =~ values[:request_url]
+      self.request_protocol = $1
+      self.request_host = $2
+    end
+
+    if self.message
+      raise "Get values of extended format"
+      # extendes logintas format   
+      # logformat logintas_accesslog %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh [lit:] %ul %ue %ea request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+      #
+      # extends the combined format with 
+      #    [lit:] %ul %ue %ea request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+      # 
+      
+    end
+
+  end
+  
+end
+	       

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,43 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class SquidRequestHeaderMeta < ActiveRecord::Base
+  description "Extended request header of squid messages"
+  sources ["SquidMeta"]
+  def self.may_have_messages?; false; end
+  
+  def self.expressions
+    ret = []
+
+    # tear out request part of
+    # request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+    
+    #--request-header
+    # (client_fqdn:           %>A) already in squid meta
+    # authorization:       %{Authorization}>h
+    # cache_control:       %{Cache-Control}>h
+    # from:                %{From}>h
+    # host                 %{Host}>h
+    # if_modified_since:   %{If-Modified-Since}>h
+    # if_unmodified_since: %{If-Unmodified-Since}>h
+    # pragma:              %{Pragma}>h
+    # proxy_authorization: %{Proxy-Authorization}>h
+
+    ret.push({ :regex => /request-header: [^ ]+ \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" response-header:/,
+	       :fields => [:authorization, :cache_control, :from, :host, 
+		 :if_modified_since, :if_unmodified_since, :pragma, :proxy_authorization]
+	     })
+    ret
+  end
+end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,52 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class SquidResponseHeaderMeta < ActiveRecord::Base
+  description "Extended response header of squid messages"
+  sources ["SquidMeta"]
+  def self.may_have_messages?; false; end
+  
+  def self.expressions
+    ret = []
+
+    # tear out response part of
+    # request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+
+    #--response-header
+    # (server_ip:           %<A) already in squid meta
+    # (response_time:       %tr) already in squid meta
+    # (mime_type:           %mt) already in squid meta
+    # server:              %{Server}<h
+    # content_md5:         %{Content-MD5}<h
+    # age:                 %{Age}<h
+    # cache_control:       %{Cache-Control}<h
+    # content_encoding:    %{Content-Encoding}<h
+    # content_language:    %{Content-Language}<h
+    # date:                %{Date}<h
+    # last_modified:       %{Last-Modified}>h
+    # location:            %{Location}>h
+    # pragma:              %{Pragma}<h
+    # proxy_autheticate    %{Proxy-Authenticate}<h
+    # via:                 %{Via}<h
+    # www_authenticate:    %{WWW-Authenticate}<h
+    
+    
+    ret.push({ :regex => /response-header: [^ ]+ [^ ]+ [^ ]+ \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\"$/,
+	       :fields => [:server, :content_md5, :age, :cache_control, :content_encoding,
+		 :content_language, :date, :last_modified, :location, :pragma, :proxy_authenticate, :via,
+		 :www_authenticate]
+	     })
+    ret
+  end
+end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,90 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class SyslogdMeta < ActiveRecord::Base
+    
+    description "Meta information for syslogd a syslogd log."
+    sources ["SourceDbMeta","ArchiveMeta"]
+
+    def original
+      # if not log_meta record is
+      # defined, we do not know the
+      # archive that the record is
+      # saved in, so exit
+      return [] unless log_meta
+
+      log_date = log_meta.date
+
+      if self.respond_to?(:queue_id) and (qid = self.queue_id)
+	# we know the id that should be in the archive
+	Archivator.find({:log_date => log_date,
+			  :raw_class => SyslogdRaw,
+			  :id=> qid})	
+      else
+	# do the old way with regexps
+	regs = []
+	inv_classes = []
+	self.children_recursive.reverse.each {|ch|
+	  inv_classes.push(ch.class) if ch.class.respond_to?(:invers_before_filter)
+	  
+	  ch.class.columns.each {|column|
+	    next if column.name =~ /^id$/ or column.name =~ /_id$/ or column.name =~ /meta_type_name/
+	    val = ch.send(column.name)
+	    
+	    inv_classes.each {|klass|
+	      #p [val,klass.name]
+	      val = klass.invers_before_filter(val)
+	      #p val
+	    }
+            val = val.strftime("%T") if column.name == "time" and val.class.name =~ /Time/
+	    esc = Regexp.escape(val.to_s.gsub("\"","\\\\\\\\\\\""))
+	    regs.push(Regexp.new(esc))
+	  }
+	}
+	#      regs.push(Regexp.new("time: \"#{log_meta.time}\"\n  date: \"#{log_meta.date}\"\n"))
+	Archivator.find({:log_date => log_date,
+			  :raw_class => SyslogdRaw,
+			  :regexps => regs})
+      end
+    rescue 
+      $!.message
+    end
+
+    def log_meta
+      LogMeta.find_by_syslogd_metas_id(self.id)
+    end
+    
+    def SyslogdMeta.create_meta( source_meta, msg)
+      if msg.class == SyslogdRaw then
+ 	new_meta = SyslogdMeta.new.prisma_initialize(source_meta, 
+				   { :ip => msg.ip,
+				     :facility => msg.facility,
+				     :priority => msg.priority,
+				     :level => msg.level,
+				     :tag => msg.tag,
+				     :program => msg.program,
+				     :queue_id => msg.id
+				   }
+				   )
+
+	log_meta = LogMeta.new.prisma_initialize(new_meta, {
+				 :date => msg.date,
+				 :time => msg.time,
+				 :host => msg.host,
+				 :message => msg.msg} )
+	return log_meta
+      end
+      return nil
+    end
+  end  

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,32 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class SyslogdSmallBaseMeta < ActiveRecord::Base
+    
+    description "Small syslogd meta"
+    sources ["PureMeta", "LogMeta"]
+
+    def self.expressions
+      ret = []
+      
+      ret.push({ :regex => /(.*?)\|(.*?)\|(.*?)\|(.*?)\|(.*)\|(.*)/,
+		 :fields => [:date, :time, :level, :ip, :message,nil]})
+    end
+
+    def get_hash
+      return nil unless message
+      return "#{date} #{time} #{ip} #{message.msg}".hash
+    end
+    
+  end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,36 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class TestMeta < ActiveRecord::Base
+    
+    description "Testing class"
+    sources ["PureMeta", "LogMeta"]
+
+    preseed_expression /^(PRISMA THROW EXCEPTION)$/
+
+    def initialize(parent, values)
+      super()
+      self.message = values[:test_msg]
+      self.save
+      throw "#{values[:test_msg]} This error is ok. Its for stability testing."
+    end
+    
+    def self.expressions
+      ret = []
+
+      ret.push({ :regex => /^(PRISMA THROW EXCEPTION)$/,
+        :fields => [:test_msg]})
+    end
+
+  end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,58 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  class UlogdNfctMeta < ActiveRecord::Base
+    
+    description "Ulogd Netfilter Connection Tracking Logs"
+    sources ["PureMeta", "LogMeta"]
+    
+    preseed_expression /^ulogd\[/
+
+    def self.may_have_messages?; false; end
+    
+    def self.expressions
+      ret = []
+
+      # ICMP:
+      # ulogd[9441]: [NEW]     ORIG: SRC=192.168.130.1 DST=192.168.130.33 PROTO=ICMP TYPE=0 CODE=8
+      #                              PKTS=1 BYTES=84 , REPLY: SRC=192.168.130.33 DST=192.168.130.1
+      #                              PROTO=ICMP TYPE=0 CODE=8 PKTS=0 BYTES=0
+      # ulogd[9441]: [DESTROY] ORIG: SRC=192.168.130.1 DST=192.168.130.60 PROTO=ICMP TYPE=0 CODE=8
+      #                              PKTS=1 BYTES=84 , REPLY: SRC=192.168.130.60 DST=192.168.130.1
+      #                              PROTO=ICMP TYPE=0 CODE=8 PKTS=0 BYTES=0
+      ret.push({ :regex =>  /^ulogd\[([^\]]*)\]:\s+\[([^\]]+)\]\s+ORIG:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(ICMP)\s+TYPE=(\d+)\s+CODE=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)\s+,\s+REPLY:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(ICMP)\s+TYPE=(\d+)\s+CODE=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)/,
+        :fields => [:process_id, :event, :orig_saddr, :orig_daddr, :orig_protocol, :icmp_type, :icmp_code, :orig_pktcount, :orig_pktlen, :reply_saddr, :reply_daddr, :reply_protocol, nil, nil, :reply_pktcount, :reply_pktlen]})
+      
+      # UDP:
+      # ulogd[9441]: [NEW]     ORIG: SRC=192.168.61.182 DST=192.168.61.130 PROTO=UDP SPT=123 DPT=123
+      #                              PKTS=1 BYTES=76 , REPLY: SRC=192.168.61.130 DST=192.168.61.182
+      #                              PROTO=UDP SPT=123 DPT=123 PKTS=0 BYTES=0
+      # ulogd[9441]: [DESTROY] ORIG: SRC=192.168.130.60 DST=192.168.61.54 PROTO=UDP SPT=48875 DPT=53
+      #                              PKTS=8 BYTES=580 , REPLY: SRC=192.168.61.54 DST=192.168.130.60
+      #                              PROTO=UDP SPT=53 DPT=48875 PKTS=8 BYTES=1515
+      #
+      # TCP:
+      # ulogd[9441]: [NEW]     ORIG: SRC=192.168.130.64 DST=192.168.61.58 PROTO=TCP SPT=4506 DPT=8080
+      #                              PKTS=1 BYTES=60 , REPLY: SRC=192.168.61.58 DST=192.168.130.64
+      #                              PROTO=TCP SPT=8080 DPT=4506 PKTS=0 BYTES=0
+      # ulogd[9441]: [DESTROY] ORIG: SRC=192.168.130.60 DST=192.168.61.58 PROTO=TCP SPT=37971 DPT=8080
+      #                              PKTS=5 BYTES=1377 , REPLY: SRC=192.168.61.58 DST=192.168.130.60
+      #                              PROTO=TCP SPT=8080 DPT=37971 PKTS=5 BYTES=966
+      ret.push({ :regex => /^ulogd\[([^\]]*)\]:\s+\[([^\]]+)\]\s+ORIG:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(\S+)\s+SPT=(\d+)\s+DPT=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)\s+,\s+REPLY:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(\S+)\s+SPT=(\d+)\s+DPT=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)/,
+		 :fields => [:process_id, :event, :orig_saddr, :orig_daddr, :orig_protocol, :orig_sport, :orig_dport, :orig_pktcount, :orig_pktlen, :reply_saddr, :reply_daddr, :reply_protocol, :reply_sport, :reply_dport, :reply_pktcount, :reply_pktlen]})
+      
+      return ret
+    end
+    
+  end

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,98 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+  # references:
+  #  http://support.microsoft.com/kb/308427: How to view and manage event logs in Event Viewer in Windows XP
+  #  http://community.netscape.com/n/pfx/forum.aspx?tsn=3&nav=messages&webtag=ws-winprohelp&tid=191184&redirCnt=1: Msg # 191184.3
+
+  class WindowsEventMeta < ActiveRecord::Base
+    
+    description "Meta base information for windows event logs."
+    sources ["PureMeta", "LogMeta"]
+    
+    preseed_expression /^(Master MSWinEventLog|Replika MSWinEventLog)/
+
+    def WindowsEventMeta.convert_to_unicode(s)
+      while s =~ /377777777(\d\d)/
+	s.sub!("377777777#{$1}", eval("\"\\303\\2#{$1}\""))
+      end
+      return s
+    end
+
+    def WindowsEventMeta.convert_to_wincode(s)
+      ret = ""
+      two_byte = false
+      s.each_byte {|b|
+	if two_byte and b.to_s(8) =~ /2(\d\d)/ then	  
+	  ret += "377777777#{$1}"
+	else
+	  if b.to_s(8) == "303"
+	    two_byte = true
+	  else
+	    ret += eval "\"\\#{b.to_s(8)}\""
+	  end
+	end
+      }
+
+      return ret
+    end
+
+    def WindowsEventMeta.create_meta( source_meta, message)
+      $log.debug("WIN Got message: #{message}")
+      unless message.msg =~ self.preseed_expression
+	$log.warn("Cannot create windows meta out of: #{message.msg}")
+	return nil
+      end
+      
+      splited = message.msg.split("\00011")
+      unless splited.length == 15
+	$log.info{"Windows message has not 15 parts separated by \\00011 trying \\t."}
+	splited = message.msg.split("\t")
+      end
+
+      unless splited.length == 15
+	$log.info{"Windows message has not 15 parts separated by \\00011 trying to separate by at least doublspace."}
+	splited = message.msg.split("  ").map {|m| m.strip}.reject {|m| m == ""}
+      end
+
+      unless splited.length == 15
+	$log.warn{"Cannot split message by \\00011 nor by tab nor by double spacing algorithm. giving up. (#{splited.inspect})"}
+	return nil
+      end
+
+      splited.each_with_index{|m,i| splited[i] = convert_to_unicode(m)}
+      
+      datetime = DateTime.strptime(splited[4],"%a %b %d %T %Y")
+      
+      WindowsEventMeta.new.prisma_initialize(source_meta,
+			   { :log_name => splited[0], # ["Master MSWinEventLog", "Replika MSWinEventLog"]
+			     :field1 => splited[1], # ["1"]
+			     :event_type => splited[2], # ["Application", "System", "Security"]
+			     :field3 => splited[3], #  num
+			     :date => datetime,:time => Time.parse(datetime.to_s),  #4 date Wed Aug 08 10:07:16 2007
+			     :event_id => splited[5], # ["1011", "15224", "15223", "452", "453", "40960", "146", "11166", "6013", "17101", "528", "680", "538", "15221", "7", "7035"]
+			     :source => splited[6], # ["ACESERVER6.1", "SDSERV_PROGRESS", "LSASRV", "DnsApi", "EventLog", "ACECLIENT", "Security", "Norton AntiVirus", "Service Control Manager"]
+			     :user => splited[7], # ["Unknown User", "SDesk", "SYSTEM"]
+			     :category => splited[8], # ["N/A", "User"]
+			     :level => splited[9], ## ["Information", "Warning", "Success Audit"]
+			     :computer => splited[10], # ["MSP43", "MSP44"]
+			     :facility => splited[11], #11 ["Devices", "Shell", "Printers", "None", "Disk", "Logon/Logoff", "Account Logon"]
+			     :data => splited[12], #12 trace_binary
+			     :field14=>  splited[14], #14 num
+			     
+			     :message => splited[13] #13 message
+			   })
+    end
+  end
+

Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,747 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated from the current state of the database. Instead of editing this file, 
+# please use the migrations feature of Active Record to incrementally modify your database, and
+# then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your database schema. If you need
+# to create the application database on another system, you should be using db:schema:load, not running
+# all the migrations from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended to check this file into your version control system.
+
+ActiveRecord::Schema.define(:version => 0) do
+
+  create_table "ace_passcode_metas", :force => true do |t|
+    t.string  "action",                 :limit => 40
+    t.string  "login",                  :limit => 30
+    t.string  "user_name",              :limit => 100
+    t.string  "token",                  :limit => 15
+    t.string  "group_name",             :limit => 100
+    t.string  "site",                   :limit => 100
+    t.string  "agent_host",             :limit => 100
+    t.string  "server",                 :limit => 100
+    t.integer "windows_event_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "ace_passcode_metas", ["action"], :name => "ace_passcode_metas_action_index"
+  add_index "ace_passcode_metas", ["agent_host"], :name => "ace_passcode_metas_agent_host_index"
+  add_index "ace_passcode_metas", ["group_name"], :name => "ace_passcode_metas_group_index"
+  add_index "ace_passcode_metas", ["log_metas_id"], :name => "ace_passcode_metas_log_metas_id_index"
+  add_index "ace_passcode_metas", ["login"], :name => "ace_passcode_metas_login_index"
+  add_index "ace_passcode_metas", ["server"], :name => "ace_passcode_metas_server_index"
+  add_index "ace_passcode_metas", ["site"], :name => "ace_passcode_metas_site_index"
+  add_index "ace_passcode_metas", ["user_name"], :name => "ace_passcode_metas_user_name_index"
+  add_index "ace_passcode_metas", ["windows_event_metas_id"], :name => "ace_passcode_metas_windows_event_metas_id_index"
+
+  create_table "amavis_metas", :force => true do |t|
+    t.integer "process_id"
+    t.string  "amavis_id",     :limit => 20
+    t.string  "action",        :limit => 20
+    t.string  "status",        :limit => 20
+    t.string  "from_field",    :limit => 50
+    t.string  "to_field"
+    t.string  "message_id",    :limit => 50
+    t.string  "hits",          :limit => 10
+    t.integer "process_time"
+    t.string  "ip",            :limit => 50
+    t.string  "signature",     :limit => 50
+    t.string  "quarantine",    :limit => 50
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "amavis_metas", ["action"], :name => "amavis_metas_action_index"
+  add_index "amavis_metas", ["from_field"], :name => "amavis_metas_from_field_index"
+  add_index "amavis_metas", ["ip"], :name => "amavis_metas_ip_index"
+  add_index "amavis_metas", ["log_metas_id"], :name => "amavis_metas_log_metas_id_index"
+  add_index "amavis_metas", ["message_id"], :name => "amavis_metas_message_id_index"
+  add_index "amavis_metas", ["pure_metas_id"], :name => "amavis_metas_pure_metas_id_index"
+  add_index "amavis_metas", ["signature"], :name => "amavis_metas_signature_index"
+  add_index "amavis_metas", ["status"], :name => "amavis_metas_status_index"
+
+  create_table "apache_file_metas", :force => true do |t|
+    t.string  "virtual_host",  :limit => 100
+    t.integer "file_metas_id"
+  end
+
+  add_index "apache_file_metas", ["file_metas_id"], :name => "apache_file_metas_file_metas_id_index"
+
+  create_table "apache_log_metas", :force => true do |t|
+    t.string  "forensic_id",  :limit => 30
+    t.integer "serve_time"
+    t.string  "host",         :limit => 50
+    t.integer "log_metas_id"
+  end
+
+  add_index "apache_log_metas", ["log_metas_id"], :name => "index_apache_log_metas_on_log_metas_id"
+
+  create_table "apache_metas", :force => true do |t|
+    t.string  "remote_host",    :limit => 40
+    t.string  "remote_logname", :limit => 20
+    t.string  "remote_user",    :limit => 20
+    t.time    "time"
+    t.date    "date"
+    t.string  "first_line",     :limit => 512
+    t.integer "status"
+    t.integer "bytes"
+    t.string  "referer",        :limit => 40
+    t.string  "useragent",      :limit => 40
+    t.integer "log_metas_id"
+    t.integer "pure_metas_id"
+  end
+
+  add_index "apache_metas", ["first_line"], :name => "apache_metas_first_line_index"
+  add_index "apache_metas", ["log_metas_id"], :name => "apache_metas_log_metas_id_index"
+  add_index "apache_metas", ["pure_metas_id"], :name => "apache_metas_pure_metas_id_index"
+  add_index "apache_metas", ["remote_host"], :name => "apache_metas_remote_host_index"
+  add_index "apache_metas", ["remote_user"], :name => "apache_metas_remote_user_index"
+  add_index "apache_metas", ["status"], :name => "apache_metas_status_index"
+  add_index "apache_metas", ["useragent"], :name => "apache_metas_useragent_index"
+
+  create_table "archive_metas", :force => true do |t|
+    t.string   "filename"
+    t.integer  "current"
+    t.integer  "total"
+    t.integer  "todo"
+    t.boolean  "finished",   :default => false
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "cisco_base_metas", :force => true do |t|
+    t.string  "ip",                          :limit => 40
+    t.string  "message_type",                :limit => 10
+    t.integer "severity"
+    t.integer "message_number"
+    t.integer "log_metas_id"
+    t.integer "syslogd_small_base_metas_id"
+  end
+
+  add_index "cisco_base_metas", ["ip"], :name => "cisco_base_metas_ip_index"
+  add_index "cisco_base_metas", ["log_metas_id"], :name => "cisco_base_metas_log_metas_id_index"
+  add_index "cisco_base_metas", ["message_type"], :name => "cisco_base_metas_message_type_index"
+  add_index "cisco_base_metas", ["severity"], :name => "cisco_base_metas_severity_index"
+  add_index "cisco_base_metas", ["syslogd_small_base_metas_id"], :name => "cisco_base_metas_syslogd_small_base_metas_id_index"
+
+  create_table "cisco_firewall_connection_metas", :force => true do |t|
+    t.string  "msg",                 :limit => 30
+    t.string  "reason",              :limit => 30
+    t.integer "connection_id"
+    t.string  "connection_type",     :limit => 10
+    t.string  "foreign_name",        :limit => 30
+    t.string  "foreign_ip",          :limit => 40
+    t.string  "foreign_port",        :limit => 10
+    t.string  "local_name",          :limit => 30
+    t.string  "local_ip",            :limit => 40
+    t.string  "local_port",          :limit => 10
+    t.string  "global_to_ip",        :limit => 40
+    t.string  "global_to_port",      :limit => 10
+    t.string  "global_from_ip",      :limit => 40
+    t.string  "global_from_port",    :limit => 10
+    t.time    "duration"
+    t.integer "bytes"
+    t.integer "cisco_base_metas_id"
+    t.integer "log_metas_id"
+    t.string  "user",                :limit => 20
+  end
+
+  add_index "cisco_firewall_connection_metas", ["cisco_base_metas_id"], :name => "cisco_firewall_connection_metas_cisco_base_metas_id_index"
+  add_index "cisco_firewall_connection_metas", ["connection_id"], :name => "cisco_firewall_connection_metas_connection_id_index"
+  add_index "cisco_firewall_connection_metas", ["connection_type"], :name => "cisco_firewall_connection_metas_connection_type_index"
+  add_index "cisco_firewall_connection_metas", ["foreign_ip"], :name => "cisco_firewall_connection_metas_foreign_ip_index"
+  add_index "cisco_firewall_connection_metas", ["foreign_name"], :name => "cisco_firewall_connection_metas_foreign_name_index"
+  add_index "cisco_firewall_connection_metas", ["foreign_port"], :name => "cisco_firewall_connection_metas_foreign_port_index"
+  add_index "cisco_firewall_connection_metas", ["global_from_ip"], :name => "cisco_firewall_connection_metas_global_from_ip_index"
+  add_index "cisco_firewall_connection_metas", ["global_from_port"], :name => "cisco_firewall_connection_metas_global_from_port_index"
+  add_index "cisco_firewall_connection_metas", ["global_to_ip"], :name => "cisco_firewall_connection_metas_global_to_ip_index"
+  add_index "cisco_firewall_connection_metas", ["global_to_port"], :name => "cisco_firewall_connection_metas_global_to_port_index"
+  add_index "cisco_firewall_connection_metas", ["local_ip"], :name => "cisco_firewall_connection_metas_local_ip_index"
+  add_index "cisco_firewall_connection_metas", ["local_name"], :name => "cisco_firewall_connection_metas_local_name_index"
+  add_index "cisco_firewall_connection_metas", ["local_port"], :name => "cisco_firewall_connection_metas_local_port_index"
+  add_index "cisco_firewall_connection_metas", ["log_metas_id"], :name => "cisco_firewall_connection_metas_log_metas_id_index"
+  add_index "cisco_firewall_connection_metas", ["reason"], :name => "cisco_firewall_connection_metas_reason_index"
+
+  create_table "cisco_firewall_metas", :force => true do |t|
+    t.string  "msg",                 :limit => 100
+    t.string  "source",              :limit => 40
+    t.string  "source_port",         :limit => 10
+    t.string  "destination",         :limit => 40
+    t.string  "destination_port",    :limit => 10
+    t.string  "interface",           :limit => 20
+    t.integer "cisco_base_metas_id"
+  end
+
+  add_index "cisco_firewall_metas", ["cisco_base_metas_id"], :name => "cisco_firewall_metas_cisco_base_metas_id_index"
+  add_index "cisco_firewall_metas", ["destination"], :name => "cisco_firewall_metas_destination_index"
+  add_index "cisco_firewall_metas", ["destination_port"], :name => "cisco_firewall_metas_destination_port_index"
+  add_index "cisco_firewall_metas", ["interface"], :name => "cisco_firewall_metas_interface_index"
+  add_index "cisco_firewall_metas", ["source"], :name => "cisco_firewall_metas_source_index"
+  add_index "cisco_firewall_metas", ["source_port"], :name => "cisco_firewall_metas_source_port_index"
+
+  create_table "cisco_metas", :force => true do |t|
+    t.string  "msg",                 :limit => 100
+    t.string  "server",              :limit => 40
+    t.string  "server_port",         :limit => 10
+    t.string  "name",                :limit => 40
+    t.string  "ip",                  :limit => 40
+    t.string  "port",                :limit => 10
+    t.string  "user",                :limit => 20
+    t.string  "group_name",          :limit => 20
+    t.string  "reason",              :limit => 100
+    t.integer "cisco_base_metas_id"
+  end
+
+  add_index "cisco_metas", ["cisco_base_metas_id"], :name => "cisco_metas_cisco_base_metas_id_index"
+  add_index "cisco_metas", ["group_name"], :name => "cisco_metas_group_name_index"
+  add_index "cisco_metas", ["ip"], :name => "cisco_metas_ip_index"
+  add_index "cisco_metas", ["msg"], :name => "cisco_metas_msg_index"
+  add_index "cisco_metas", ["name"], :name => "cisco_metas_name_index"
+  add_index "cisco_metas", ["port"], :name => "cisco_metas_port_index"
+  add_index "cisco_metas", ["server"], :name => "cisco_metas_server_index"
+  add_index "cisco_metas", ["server_port"], :name => "cisco_metas_server_port_index"
+  add_index "cisco_metas", ["user"], :name => "cisco_metas_user_index"
+
+  create_table "cisco_session_metas", :force => true do |t|
+    t.string  "msg",            :limit => 100
+    t.string  "session_type",   :limit => 30
+    t.time    "duration"
+    t.integer "in_bytes"
+    t.integer "out_bytes"
+    t.integer "cisco_metas_id"
+  end
+
+  add_index "cisco_session_metas", ["cisco_metas_id"], :name => "index_cisco_session_metas_on_cisco_metas_id"
+  add_index "cisco_session_metas", ["msg"], :name => "index_cisco_session_metas_on_msg"
+  add_index "cisco_session_metas", ["session_type"], :name => "index_cisco_session_metas_on_session_type"
+
+  create_table "compression_metas", :force => true do |t|
+    t.string  "extname"
+    t.string  "inflate_command"
+    t.integer "inflated_size"
+    t.integer "deflated_size"
+    t.integer "file_metas_id"
+  end
+
+  create_table "cron_metas", :force => true do |t|
+    t.integer "process_id"
+    t.string  "user",          :limit => 20
+    t.integer "uid"
+    t.string  "program",       :limit => 20
+    t.string  "action",        :limit => 20
+    t.string  "command"
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "cron_metas", ["action"], :name => "cron_metas_action_index"
+  add_index "cron_metas", ["log_metas_id"], :name => "cron_metas_log_metas_id_index"
+  add_index "cron_metas", ["program"], :name => "cron_metas_program_index"
+  add_index "cron_metas", ["pure_metas_id"], :name => "cron_metas_pure_metas_id_index"
+  add_index "cron_metas", ["uid"], :name => "cron_metas_uid_index"
+  add_index "cron_metas", ["user"], :name => "cron_metas_user_index"
+
+  create_table "fetchmail_metas", :force => true do |t|
+    t.integer "process_id"
+    t.string  "program",       :limit => 20
+    t.string  "action",        :limit => 200
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "fetchmail_metas", ["action"], :name => "fetchmail_metas_action_index"
+  add_index "fetchmail_metas", ["log_metas_id"], :name => "fetchmail_metas_log_metas_id_index"
+  add_index "fetchmail_metas", ["program"], :name => "fetchmail_metas_program_index"
+  add_index "fetchmail_metas", ["pure_metas_id"], :name => "fetchmail_metas_pure_metas_id_index"
+
+  create_table "file_metas", :force => true do |t|
+    t.string   "dirname"
+    t.string   "basename"
+    t.string   "ftype",              :limit => 20
+    t.integer  "size"
+    t.datetime "mtime"
+    t.datetime "atime"
+    t.datetime "ctime"
+    t.integer  "umask"
+    t.integer  "uid"
+    t.integer  "gid"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.string   "options"
+    t.integer  "source_db_metas_id"
+    t.integer  "archive_metas_id"
+  end
+
+  add_index "file_metas", ["archive_metas_id"], :name => "file_metas_archive_metas_id_index"
+  add_index "file_metas", ["source_db_metas_id"], :name => "file_metas_source_db_metas_id_index"
+
+  create_table "inet_header_metas", :force => true do |t|
+    t.date    "date"
+    t.time    "time"
+    t.integer "milliseconds"
+    t.string  "severity",        :limit => 10
+    t.string  "system_id",       :limit => 30
+    t.string  "msg_id",          :limit => 30
+    t.string  "user_id",         :limit => 30
+    t.string  "eventtype",       :limit => 30
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+    t.string  "client_ip",       :limit => 32
+    t.string  "server_ip",       :limit => 32
+    t.string  "session_id",      :limit => 32
+    t.integer "hit_number"
+    t.integer "num_object_hits"
+    t.string  "text1",           :limit => 1024
+    t.string  "text2",           :limit => 1024
+  end
+
+  add_index "inet_header_metas", ["client_ip"], :name => "index_inet_header_metas_on_client_ip"
+  add_index "inet_header_metas", ["date"], :name => "index_inet_header_metas_on_date"
+  add_index "inet_header_metas", ["eventtype"], :name => "index_inet_header_metas_on_eventtype"
+  add_index "inet_header_metas", ["hit_number"], :name => "index_inet_header_metas_on_hit_number"
+  add_index "inet_header_metas", ["log_metas_id"], :name => "index_inet_header_metas_on_log_metas_id"
+  add_index "inet_header_metas", ["msg_id"], :name => "index_inet_header_metas_on_msg_id"
+  add_index "inet_header_metas", ["num_object_hits"], :name => "index_inet_header_metas_on_num_object_hits"
+  add_index "inet_header_metas", ["pure_metas_id"], :name => "index_inet_header_metas_on_pure_metas_id"
+  add_index "inet_header_metas", ["server_ip"], :name => "index_inet_header_metas_on_server_ip"
+  add_index "inet_header_metas", ["session_id"], :name => "index_inet_header_metas_on_session_id"
+  add_index "inet_header_metas", ["severity"], :name => "index_inet_header_metas_on_severity"
+  add_index "inet_header_metas", ["system_id"], :name => "index_inet_header_metas_on_system_id"
+  add_index "inet_header_metas", ["user_id"], :name => "index_inet_header_metas_on_user_id"
+
+  create_table "inet_object_metas", :force => true do |t|
+    t.string  "objecttype",           :limit => 30
+    t.string  "object_id",            :limit => 512
+    t.string  "version",              :limit => 30
+    t.string  "filename",             :limit => 256
+    t.string  "description",          :limit => 256
+    t.string  "object_hashes",        :limit => 256
+    t.string  "object_url",           :limit => 512
+    t.integer "inet_header_metas_id"
+    t.integer "inet_object_metas_id"
+    t.string  "object_location",      :limit => 512
+  end
+
+  add_index "inet_object_metas", ["filename"], :name => "index_inet_object_metas_on_filename"
+  add_index "inet_object_metas", ["inet_header_metas_id"], :name => "index_inet_object_metas_on_inet_header_metas_id"
+  add_index "inet_object_metas", ["inet_object_metas_id"], :name => "index_inet_object_metas_on_inet_object_metas_id"
+  add_index "inet_object_metas", ["object_hashes"], :name => "index_inet_object_metas_on_object_hashes"
+  add_index "inet_object_metas", ["object_id"], :name => "index_inet_object_metas_on_object_id"
+  add_index "inet_object_metas", ["object_url"], :name => "index_inet_object_metas_on_object_url"
+  add_index "inet_object_metas", ["objecttype"], :name => "index_inet_object_metas_on_objecttype"
+  add_index "inet_object_metas", ["version"], :name => "index_inet_object_metas_on_version"
+
+  create_table "iptables_firewall_metas", :force => true do |t|
+    t.string  "rule",          :limit => 10
+    t.string  "src",           :limit => 20
+    t.string  "spt",           :limit => 10
+    t.string  "dst",           :limit => 20
+    t.string  "dpt",           :limit => 10
+    t.string  "custom",        :limit => 20
+    t.string  "in",            :limit => 10
+    t.string  "out",           :limit => 10
+    t.string  "physin",        :limit => 10
+    t.string  "physout",       :limit => 10
+    t.integer "len"
+    t.string  "tos",           :limit => 10
+    t.string  "prec",          :limit => 10
+    t.integer "ttl"
+    t.integer "identifier"
+    t.string  "proto",         :limit => 10
+    t.string  "additional",    :limit => 20
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "iptables_firewall_metas", ["dpt"], :name => "iptables_firewall_metas_dpt_index"
+  add_index "iptables_firewall_metas", ["dst"], :name => "iptables_firewall_metas_dst_index"
+  add_index "iptables_firewall_metas", ["log_metas_id"], :name => "iptables_firewall_metas_log_metas_id_index"
+  add_index "iptables_firewall_metas", ["proto"], :name => "iptables_firewall_metas_proto_index"
+  add_index "iptables_firewall_metas", ["pure_metas_id"], :name => "iptables_firewall_metas_pure_metas_id_index"
+  add_index "iptables_firewall_metas", ["rule"], :name => "iptables_firewall_metas_rule_index"
+  add_index "iptables_firewall_metas", ["spt"], :name => "iptables_firewall_metas_spt_index"
+  add_index "iptables_firewall_metas", ["src"], :name => "iptables_firewall_metas_src_index"
+
+  create_table "log_metas", :force => true do |t|
+    t.date    "date"
+    t.time    "time"
+    t.string  "host"
+    t.integer "hash_value"
+    t.integer "syslogd_metas_id"
+    t.integer "pure_metas_id"
+    t.integer "file_metas_id"
+  end
+
+  add_index "log_metas", ["date"], :name => "log_metas_date_index"
+  add_index "log_metas", ["file_metas_id"], :name => "log_metas_file_metas_id_index"
+  add_index "log_metas", ["hash_value"], :name => "log_metas_hash_value_index"
+  add_index "log_metas", ["host"], :name => "log_metas_host_index"
+  add_index "log_metas", ["pure_metas_id"], :name => "log_metas_pure_metas_id_index"
+  add_index "log_metas", ["syslogd_metas_id"], :name => "log_metas_syslogd_metas_id_index"
+  add_index "log_metas", ["time"], :name => "index_log_metas_on_time"
+
+  create_table "messages", :force => true do |t|
+    t.integer "meta_id"
+    t.binary  "msg"
+    t.string  "meta_type_name", :limit => 100
+  end
+
+  add_index "messages", ["meta_id"], :name => "messages_meta_id_index"
+  add_index "messages", ["meta_type_name"], :name => "messages_meta_type_name_index"
+
+  create_table "metas", :force => true do |t|
+    t.integer "parent_id"
+    t.string  "meta_type_name", :limit => 100
+    t.integer "meta_id"
+  end
+
+  add_index "metas", ["meta_id"], :name => "metas_meta_id_index"
+  add_index "metas", ["meta_type_name"], :name => "metas_meta_type_name_index"
+  add_index "metas", ["parent_id"], :name => "metas_parent_id_index"
+
+  create_table "nagios_metas", :force => true do |t|
+    t.string  "msg_type",       :limit => 50
+    t.string  "probed_by_host", :limit => 50
+    t.string  "affected_host",  :limit => 50
+    t.string  "service",        :limit => 20
+    t.string  "status",         :limit => 20
+    t.string  "unknown_1",      :limit => 20
+    t.integer "unknown_2"
+    t.string  "output"
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "nagios_metas", ["affected_host"], :name => "nagios_metas_affected_host_index"
+  add_index "nagios_metas", ["log_metas_id"], :name => "nagios_metas_log_metas_id_index"
+  add_index "nagios_metas", ["msg_type"], :name => "nagios_metas_msg_type_index"
+  add_index "nagios_metas", ["probed_by_host"], :name => "nagios_metas_probed_by_host_index"
+  add_index "nagios_metas", ["pure_metas_id"], :name => "nagios_metas_pure_metas_id_index"
+  add_index "nagios_metas", ["service"], :name => "nagios_metas_service_index"
+  add_index "nagios_metas", ["status"], :name => "nagios_metas_status_index"
+
+  create_table "nonyms", :force => true do |t|
+    t.string "real_name", :limit => 20
+  end
+
+  add_index "nonyms", ["real_name"], :name => "nonyms_real_name_index"
+
+  create_table "ovpn_base_metas", :force => true do |t|
+    t.string  "vpn",           :limit => 20
+    t.integer "process_id"
+    t.string  "client_ip",     :limit => 50
+    t.integer "client_port"
+    t.string  "cert",          :limit => 50
+    t.string  "msg_type",      :limit => 50
+    t.string  "msg"
+    t.string  "client",        :limit => 20
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "ovpn_base_metas", ["cert"], :name => "ovpn_base_metas_cert_index"
+  add_index "ovpn_base_metas", ["client_ip"], :name => "ovpn_base_metas_client_ip_index"
+  add_index "ovpn_base_metas", ["client_port"], :name => "ovpn_base_metas_client_port_index"
+  add_index "ovpn_base_metas", ["log_metas_id"], :name => "ovpn_base_metas_log_metas_id_index"
+  add_index "ovpn_base_metas", ["msg_type"], :name => "ovpn_base_metas_msg_type_index"
+  add_index "ovpn_base_metas", ["pure_metas_id"], :name => "ovpn_base_metas_pure_metas_id_index"
+  add_index "ovpn_base_metas", ["vpn"], :name => "ovpn_base_metas_vpn_index"
+
+  create_table "postfix_detail_metas", :force => true do |t|
+    t.string  "orig_to",            :limit => 50
+    t.string  "relay_host",         :limit => 50
+    t.string  "relay_ip",           :limit => 50
+    t.float   "delay"
+    t.integer "size"
+    t.integer "nrcpt"
+    t.string  "status",             :limit => 20
+    t.string  "result_text",        :limit => 200
+    t.integer "postfix_metas_id"
+    t.integer "relay_port"
+    t.float   "delay_before_qmgr"
+    t.float   "delay_in_qmgr"
+    t.float   "delay_conn_setup"
+    t.float   "delay_transmission"
+    t.string  "dsn",                :limit => 10
+    t.string  "result",             :limit => 20
+    t.string  "result_mail_id",     :limit => 10
+  end
+
+  add_index "postfix_detail_metas", ["delay"], :name => "altered_postfix_detail_metas_delay_index"
+  add_index "postfix_detail_metas", ["postfix_metas_id"], :name => "altered_postfix_detail_metas_postfix_metas_id_index"
+  add_index "postfix_detail_metas", ["relay_host"], :name => "altered_postfix_detail_metas_relay_host_index"
+  add_index "postfix_detail_metas", ["relay_ip"], :name => "altered_postfix_detail_metas_relay_ip_index"
+  add_index "postfix_detail_metas", ["status"], :name => "altered_postfix_detail_metas_status_index"
+
+  create_table "postfix_metas", :force => true do |t|
+    t.string  "program",          :limit => 10
+    t.integer "process_id"
+    t.string  "mail_message_id",  :limit => 15
+    t.string  "action",           :limit => 40
+    t.string  "host",             :limit => 50
+    t.string  "ip",               :limit => 50
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+    t.string  "email_local_part", :limit => 50
+    t.string  "message_id",       :limit => 50
+    t.string  "email_domain",     :limit => 50
+  end
+
+  add_index "postfix_metas", ["action"], :name => "altered_postfix_metas_action_index"
+  add_index "postfix_metas", ["email_domain"], :name => "index_postfix_metas_on_email_domain"
+  add_index "postfix_metas", ["email_local_part"], :name => "index_postfix_metas_on_email_local_part"
+  add_index "postfix_metas", ["host"], :name => "altered_postfix_metas_host_index"
+  add_index "postfix_metas", ["ip"], :name => "altered_postfix_metas_ip_index"
+  add_index "postfix_metas", ["log_metas_id"], :name => "altered_postfix_metas_log_metas_id_index"
+  add_index "postfix_metas", ["mail_message_id"], :name => "altered_postfix_metas_mail_message_id_index"
+  add_index "postfix_metas", ["message_id"], :name => "index_postfix_metas_on_message_id"
+  add_index "postfix_metas", ["program"], :name => "altered_postfix_metas_program_index"
+  add_index "postfix_metas", ["pure_metas_id"], :name => "altered_postfix_metas_pure_metas_id_index"
+
+  create_table "pure_metas", :force => true do |t|
+    t.integer "file_metas_id"
+    t.integer "compression_metas_id"
+  end
+
+  add_index "pure_metas", ["compression_metas_id"], :name => "index_pure_metas_on_compression_metas_id"
+  add_index "pure_metas", ["file_metas_id"], :name => "pure_metas_file_metas_id_index"
+
+  create_table "source_db_metas", :force => true do |t|
+    t.string   "process_type",   :limit => 10
+    t.integer  "start"
+    t.integer  "current"
+    t.integer  "total"
+    t.integer  "todo"
+    t.integer  "count"
+    t.string   "raw_class_name", :limit => 20
+    t.boolean  "execute_once"
+    t.integer  "waiting_time"
+    t.boolean  "finished",                     :default => false
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "squid_metas", :force => true do |t|
+    t.integer "process_id"
+    t.integer "seconds_since_epoch"
+    t.integer "subsecond_time"
+    t.integer "response_time_milliseconds"
+    t.string  "client_source_ip",           :limit => 40
+    t.string  "request_status",             :limit => 30
+    t.integer "http_status_code"
+    t.integer "reply_size"
+    t.string  "request_method",             :limit => 10
+    t.text    "request_url"
+    t.string  "user_name",                  :limit => 40
+    t.string  "hierarchy_status",           :limit => 30
+    t.string  "server_ip",                  :limit => 50
+    t.string  "mime_type",                  :limit => 60
+    t.string  "request_protocol",           :limit => 10
+    t.string  "request_host",               :limit => 50
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+    t.string  "protocol_version",           :limit => 5
+    t.string  "client_fqdn",                :limit => 50
+    t.text    "referer"
+    t.string  "user_indent",                :limit => 40
+    t.string  "user_agent",                 :limit => 40
+    t.string  "user_auth",                  :limit => 40
+    t.string  "user_acl",                   :limit => 40
+    t.text    "acl_log"
+  end
+
+  add_index "squid_metas", ["client_fqdn"], :name => "new_client_fqdn_index"
+  add_index "squid_metas", ["client_source_ip"], :name => "index_squid_metas_on_client_source_ip"
+  add_index "squid_metas", ["hierarchy_status"], :name => "index_squid_metas_on_hierarchy_status"
+  add_index "squid_metas", ["http_status_code"], :name => "index_squid_metas_on_http_status_code"
+  add_index "squid_metas", ["log_metas_id"], :name => "index_squid_metas_on_log_metas_id"
+  add_index "squid_metas", ["mime_type"], :name => "index_squid_metas_on_mime_type"
+  add_index "squid_metas", ["process_id"], :name => "index_squid_metas_on_process_id"
+  add_index "squid_metas", ["protocol_version"], :name => "index_squid_metas_on_protocol_version"
+  add_index "squid_metas", ["pure_metas_id"], :name => "index_squid_metas_on_pure_metas_id"
+  add_index "squid_metas", ["reply_size"], :name => "index_squid_metas_on_reply_size"
+  add_index "squid_metas", ["request_host"], :name => "index_squid_metas_on_request_host"
+  add_index "squid_metas", ["request_method"], :name => "index_squid_metas_on_request_method"
+  add_index "squid_metas", ["request_protocol"], :name => "index_squid_metas_on_request_protocol"
+  add_index "squid_metas", ["request_status"], :name => "index_squid_metas_on_request_status"
+  add_index "squid_metas", ["response_time_milliseconds"], :name => "index_squid_metas_on_response_time_milliseconds"
+  add_index "squid_metas", ["server_ip"], :name => "index_squid_metas_on_client_fqdn"
+  add_index "squid_metas", ["user_acl"], :name => "index_squid_metas_on_user_acl"
+  add_index "squid_metas", ["user_agent"], :name => "index_squid_metas_on_user_agent"
+  add_index "squid_metas", ["user_auth"], :name => "index_squid_metas_on_user_auth"
+  add_index "squid_metas", ["user_indent"], :name => "index_squid_metas_on_user_indent"
+  add_index "squid_metas", ["user_name"], :name => "index_squid_metas_on_user_name"
+
+  create_table "squid_request_header_metas", :force => true do |t|
+    t.string  "authorization",       :limit => 50
+    t.string  "cache_control",       :limit => 50
+    t.string  "from",                :limit => 50
+    t.string  "host",                :limit => 50
+    t.string  "if_modified_since",   :limit => 50
+    t.string  "if_unmodified_since", :limit => 50
+    t.string  "pragma",              :limit => 50
+    t.string  "proxy_authorization", :limit => 50
+    t.integer "squid_metas_id"
+  end
+
+  add_index "squid_request_header_metas", ["authorization"], :name => "index_squid_request_header_metas_on_authorization"
+  add_index "squid_request_header_metas", ["from"], :name => "index_squid_request_header_metas_on_from"
+  add_index "squid_request_header_metas", ["host"], :name => "index_squid_request_header_metas_on_host"
+  add_index "squid_request_header_metas", ["squid_metas_id"], :name => "index_squid_request_header_metas_on_squid_metas_id"
+
+  create_table "squid_response_header_metas", :force => true do |t|
+    t.string   "server",             :limit => 50
+    t.string   "content_md5",        :limit => 50
+    t.string   "age",                :limit => 50
+    t.string   "cache_control",      :limit => 50
+    t.string   "content_encoding",   :limit => 50
+    t.string   "content_language",   :limit => 50
+    t.date     "date"
+    t.datetime "last_modified"
+    t.string   "location",           :limit => 50
+    t.string   "pragma",             :limit => 50
+    t.string   "proxy_authenticate", :limit => 50
+    t.string   "via",                :limit => 50
+    t.string   "www_authenticate",   :limit => 50
+    t.integer  "squid_metas_id"
+  end
+
+  add_index "squid_response_header_metas", ["age"], :name => "index_squid_response_header_metas_on_age"
+  add_index "squid_response_header_metas", ["content_encoding"], :name => "index_squid_response_header_metas_on_content_encoding"
+  add_index "squid_response_header_metas", ["content_language"], :name => "index_squid_response_header_metas_on_content_language"
+  add_index "squid_response_header_metas", ["date"], :name => "index_squid_response_header_metas_on_date"
+  add_index "squid_response_header_metas", ["server"], :name => "index_squid_response_header_metas_on_server"
+  add_index "squid_response_header_metas", ["squid_metas_id"], :name => "index_squid_response_header_metas_on_squid_metas_id"
+
+  create_table "syslogd_metas", :force => true do |t|
+    t.string  "ip",                 :limit => 40
+    t.string  "facility",           :limit => 10
+    t.string  "priority",           :limit => 10
+    t.string  "level",              :limit => 10
+    t.string  "tag",                :limit => 10
+    t.integer "program"
+    t.integer "source_db_metas_id"
+    t.integer "archive_metas_id"
+    t.integer "queue_id"
+  end
+
+  add_index "syslogd_metas", ["archive_metas_id"], :name => "syslogd_metas_archive_metas_id_index"
+  add_index "syslogd_metas", ["ip"], :name => "syslogd_metas_ip_index"
+  add_index "syslogd_metas", ["program"], :name => "syslogd_metas_program_index"
+  add_index "syslogd_metas", ["queue_id"], :name => "index_syslogd_metas_on_queue_id"
+  add_index "syslogd_metas", ["source_db_metas_id"], :name => "syslogd_metas_source_db_metas_id_index"
+
+  create_table "syslogd_small_base_metas", :force => true do |t|
+    t.date    "date"
+    t.time    "time"
+    t.string  "level",         :limit => 10
+    t.string  "ip",            :limit => 40
+    t.integer "hash_value"
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "syslogd_small_base_metas", ["date"], :name => "syslogd_small_base_metas_date_index"
+  add_index "syslogd_small_base_metas", ["hash_value"], :name => "syslogd_small_base_metas_hash_value_index"
+  add_index "syslogd_small_base_metas", ["ip"], :name => "syslogd_small_base_metas_ip_index"
+  add_index "syslogd_small_base_metas", ["level"], :name => "syslogd_small_base_metas_level_index"
+  add_index "syslogd_small_base_metas", ["log_metas_id"], :name => "syslogd_small_base_metas_log_metas_id_index"
+  add_index "syslogd_small_base_metas", ["pure_metas_id"], :name => "syslogd_small_base_metas_pure_metas_id_index"
+
+  create_table "test_metas", :force => true do |t|
+    t.string  "message"
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "test_metas", ["log_metas_id"], :name => "test_metas_log_metas_id_index"
+  add_index "test_metas", ["pure_metas_id"], :name => "test_metas_pure_metas_id_index"
+
+  create_table "ulogd_nfct_metas", :force => true do |t|
+    t.integer "process_id"
+    t.string  "event",          :limit => 16
+    t.string  "orig_saddr",     :limit => 50
+    t.string  "orig_daddr",     :limit => 50
+    t.string  "orig_protocol",  :limit => 10
+    t.integer "orig_sport"
+    t.integer "orig_dport"
+    t.integer "orig_pktlen"
+    t.integer "orig_pktcount"
+    t.string  "reply_saddr",    :limit => 50
+    t.string  "reply_daddr",    :limit => 50
+    t.string  "reply_protocol", :limit => 10
+    t.integer "reply_sport"
+    t.integer "reply_dport"
+    t.integer "reply_pktlen"
+    t.integer "reply_pktcount"
+    t.integer "icmp_code"
+    t.integer "icmp_type"
+    t.integer "log_metas_id"
+    t.integer "pure_metas_id"
+  end
+
+  add_index "ulogd_nfct_metas", ["event"], :name => "index_ulogd_nfct_metas_on_event"
+  add_index "ulogd_nfct_metas", ["icmp_code"], :name => "index_ulogd_nfct_metas_on_icmp_code"
+  add_index "ulogd_nfct_metas", ["icmp_type"], :name => "index_ulogd_nfct_metas_on_icmp_type"
+  add_index "ulogd_nfct_metas", ["log_metas_id"], :name => "index_ulogd_nfct_metas_on_log_metas_id"
+  add_index "ulogd_nfct_metas", ["orig_daddr"], :name => "index_ulogd_nfct_metas_on_orig_daddr"
+  add_index "ulogd_nfct_metas", ["orig_dport"], :name => "index_ulogd_nfct_metas_on_orig_dport"
+  add_index "ulogd_nfct_metas", ["orig_pktcount"], :name => "index_ulogd_nfct_metas_on_orig_pktcount"
+  add_index "ulogd_nfct_metas", ["orig_protocol"], :name => "index_ulogd_nfct_metas_on_orig_protocol"
+  add_index "ulogd_nfct_metas", ["orig_saddr"], :name => "index_ulogd_nfct_metas_on_orig_saddr"
+  add_index "ulogd_nfct_metas", ["orig_sport"], :name => "index_ulogd_nfct_metas_on_orig_sport"
+  add_index "ulogd_nfct_metas", ["reply_daddr"], :name => "index_ulogd_nfct_metas_on_reply_daddr"
+  add_index "ulogd_nfct_metas", ["reply_dport"], :name => "index_ulogd_nfct_metas_on_reply_dport"
+  add_index "ulogd_nfct_metas", ["reply_pktcount"], :name => "index_ulogd_nfct_metas_on_reply_pktcount"
+  add_index "ulogd_nfct_metas", ["reply_pktlen"], :name => "index_ulogd_nfct_metas_on_reply_pktlen"
+  add_index "ulogd_nfct_metas", ["reply_protocol"], :name => "index_ulogd_nfct_metas_on_reply_protocol"
+  add_index "ulogd_nfct_metas", ["reply_saddr"], :name => "index_ulogd_nfct_metas_on_reply_saddr"
+  add_index "ulogd_nfct_metas", ["reply_sport"], :name => "index_ulogd_nfct_metas_on_reply_sport"
+
+  create_table "windows_event_metas", :force => true do |t|
+    t.string  "log_name",      :limit => 30
+    t.integer "field1"
+    t.string  "event_type",    :limit => 20
+    t.integer "field3"
+    t.date    "date"
+    t.time    "time"
+    t.integer "event_id"
+    t.string  "source",        :limit => 30
+    t.string  "user"
+    t.string  "category",      :limit => 30
+    t.string  "level",         :limit => 30
+    t.string  "computer"
+    t.string  "facility",      :limit => 30
+    t.binary  "data"
+    t.integer "field14"
+    t.integer "pure_metas_id"
+    t.integer "log_metas_id"
+  end
+
+  add_index "windows_event_metas", ["category"], :name => "windows_event_metas_category_index"
+  add_index "windows_event_metas", ["computer"], :name => "windows_event_metas_computer_index"
+  add_index "windows_event_metas", ["date"], :name => "windows_event_metas_date_index"
+  add_index "windows_event_metas", ["event_type"], :name => "windows_event_metas_event_type_index"
+  add_index "windows_event_metas", ["facility"], :name => "windows_event_metas_facility_index"
+  add_index "windows_event_metas", ["level"], :name => "windows_event_metas_message_level_index"
+  add_index "windows_event_metas", ["log_metas_id"], :name => "windows_event_metas_log_metasid_index"
+  add_index "windows_event_metas", ["log_name"], :name => "windows_event_metas_log_name_index"
+  add_index "windows_event_metas", ["pure_metas_id"], :name => "windows_event_metas_pure_metas_id_index"
+  add_index "windows_event_metas", ["source"], :name => "windows_event_metas_source_index"
+  add_index "windows_event_metas", ["user"], :name => "windows_event_metas_user_index"
+
+end

Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb Thu Nov  4 18:27:22 2010
@@ -0,0 +1,67 @@
+# Copyright 2010 The Apache Software Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class InitializeDatabase < ActiveRecord::Migration
+  def self.up
+    
+    create_table "file_raws" do |t|
+      t.column "dirname", :string
+      t.column "basename", :string
+      t.column "ftype", :string, :limit => 20
+      t.column "size", :integer
+      t.column "mtime", :datetime
+      t.column "atime", :datetime
+      t.column "ctime", :datetime
+      t.column "umask", :integer
+      t.column "uid", :integer
+      t.column "gid", :integer
+      t.column "created_at", :datetime
+      t.column "updated_at", :datetime
+      t.column "options", :string
+      t.column "msg", :binary
+    end
+
+    create_table "syslogd_raws" do |t|
+      t.column "ip", :string, :limit => 40
+      t.column "host", :string
+      t.column "facility", :string, :limit => 10
+      t.column "priority", :string, :limit => 10
+      t.column "level", :string, :limit => 10
+      t.column "tag", :string, :limit => 10
+      t.column "date", :date
+      t.column "time", :time
+      t.column "program", :integer, :limit => 15
+      t.column "created_at", :datetime
+      t.column "msg", :string
+    end
+
+    create_table "raws_states" do |t|
+      t.column "table_name", :string, :limit => 20
+      t.column "count_limit", :integer
+      t.column "count_time", :float
+      t.column "count_value", :integer
+      t.column "delete_time", :float
+      t.column "delete_value", :integer
+      t.column "created_at", :datetime
+      t.column "updated_at", :datetime
+    end
+
+    add_index "raws_states", ["table_name"], :name => "raws_states_table_name_index"
+  end
+
+  def self.down
+    raise IrreversibleMigration
+  end
+  
+end



Mime
View raw message