logging-log4j-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sde...@apache.org
Subject cvs commit: logging-log4j/src/java/org/apache/log4j/varia LogFilePatternReceiver.java
Date Thu, 09 Sep 2004 08:34:15 GMT
sdeboy      2004/09/09 01:34:14

  Modified:    src/java/org/apache/log4j/chainsaw/help release-notes.html
               src/java/org/apache/log4j/varia LogFilePatternReceiver.java
  Log:
  Rewrote LogFilePatternReceiver to add support for multi-line messages and improved parsing
logic.  To use this receiver you must now have the ORO library in your classpath.
  
  Revision  Changes    Path
  1.42      +5 -0      logging-log4j/src/java/org/apache/log4j/chainsaw/help/release-notes.html
  
  Index: release-notes.html
  ===================================================================
  RCS file: /home/cvs/logging-log4j/src/java/org/apache/log4j/chainsaw/help/release-notes.html,v
  retrieving revision 1.41
  retrieving revision 1.42
  diff -u -r1.41 -r1.42
  --- release-notes.html	2 Sep 2004 00:30:19 -0000	1.41
  +++ release-notes.html	9 Sep 2004 08:34:14 -0000	1.42
  @@ -9,6 +9,11 @@
   
   <h1>1.99.99</h2>
   
  +<h2>9 September 2004</h2>
  +<ul>
  +<li>Rewrote LogFilePatternReceiver to add support for multi-line messages and improved
parsing logic.  To use this receiver you must now have the ORO library in your classpath (ORO
is included in the WebStart version).</li>
  +</ul>
  +
   <h2>2 September 2004</h2>
   <ul>
   <li>Added Drag & Drop (rudimentary) support to Chainsaw.   You can drag any File
to Chainsaw's 
  
  
  
  1.16      +469 -507  logging-log4j/src/java/org/apache/log4j/varia/LogFilePatternReceiver.java
  
  Index: LogFilePatternReceiver.java
  ===================================================================
  RCS file: /home/cvs/logging-log4j/src/java/org/apache/log4j/varia/LogFilePatternReceiver.java,v
  retrieving revision 1.15
  retrieving revision 1.16
  diff -u -r1.15 -r1.16
  --- LogFilePatternReceiver.java	7 Sep 2004 17:16:05 -0000	1.15
  +++ LogFilePatternReceiver.java	9 Sep 2004 08:34:14 -0000	1.16
  @@ -1,17 +1,17 @@
   /*
    * Copyright 1999,2004 The Apache Software Foundation.
  - *
  - * Licensed under the Apache License, Version 2.0 (the "License");
  - * you may not use this file except in compliance with the License.
  - * You may obtain a copy of the License at
  - *
  + * 
  + * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  + * use this file except in compliance with the License. You may obtain a copy of
  + * the License at
  + * 
    *      http://www.apache.org/licenses/LICENSE-2.0
  - *
  + * 
    * Unless required by applicable law or agreed to in writing, software
  - * distributed under the License is distributed on an "AS IS" BASIS,
  - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  - * See the License for the specific language governing permissions and
  - * limitations under the License.
  + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  + * License for the specific language governing permissions and limitations under
  + * the License.
    */
   
   package org.apache.log4j.varia;
  @@ -20,16 +20,17 @@
   import java.io.IOException;
   import java.io.InputStreamReader;
   import java.io.Reader;
  -import java.net.MalformedURLException;
   import java.net.URL;
   import java.text.SimpleDateFormat;
   import java.util.ArrayList;
   import java.util.HashMap;
  +import java.util.HashSet;
   import java.util.Hashtable;
   import java.util.Iterator;
  -import java.util.LinkedList;
   import java.util.List;
   import java.util.Map;
  +import java.util.Set;
  +import java.util.TreeMap;
   
   import org.apache.log4j.Level;
   import org.apache.log4j.Logger;
  @@ -41,137 +42,143 @@
   import org.apache.log4j.spi.LoggingEvent;
   import org.apache.log4j.spi.ThrowableInformation;
   import org.apache.log4j.spi.location.LocationInfo;
  -
  +import org.apache.oro.text.perl.Perl5Util;
  +import org.apache.oro.text.regex.MalformedPatternException;
  +import org.apache.oro.text.regex.MatchResult;
  +import org.apache.oro.text.regex.Pattern;
  +import org.apache.oro.text.regex.Perl5Compiler;
  +import org.apache.oro.text.regex.Perl5Matcher;
   
   /**
  - * A receiver which supports the definition of the log format using keywords, the
  - * contained timestamp using SimpleDateFormat's format support, and the file URL
  - *
  - * FEATURES:
  - * specify the URL of the log file to be processed
  - * specify the timestamp format (if one exists)
  - * specify the layout used in the log file
  - * define your file's layout using these keywords along with any text being added as delimiters
  - * supports the conversion of exceptions found in the file
  - *
  - * TIMESTAMP
  - * LOGGER
  - * LEVEL
  - * THREAD
  - * CLASS
  - * FILE
  - * LINE
  - * METHOD
  - * RELATIVETIME
  - * MESSAGE
  - * *
  - * PROP(key)
  - * 
  - * For example,
  - *
  - * If your file's patternlayout is this:
  - * %d %-5p [%t] %C{2} (%F:%L) - %m%n
  - *
  - * specify this as the log format:
  - * TIMESTAMP LEVEL [THREAD] CLASS (FILE:LINE) - MESSAGE
  - *
  - * Only PROPERTIES are supported directly, but an MDC entry (or delimited NDC) can be set
as a property
  + * LogFilePatternReceiver can parse and tail log files, converting entries into
  + * LoggingEvents.
  + * <p>
  + * This receiver relies on ORO Perl5 features to perform the parsing of text in the 
  + * log file, however the only regular expression field explicitly supported is 
  + * a glob-style wildcard used to ignore fields in the log file if needed.  All other
  + * fields are parsed by using the supplied keywords.
  + * <p>
  + * <b>Features:</b><br>
  + * - specify the URL of the log file to be processed<br>
  + * - specify the timestamp format in the file (if one exists)<br>
  + * - specify the pattern (logFormat) used in the log file using keywords, a wildcard character
(*) and fixed text<br>
  + * - 'tail' the file (allows the contents of the file to be continually read and new events
processed)<br>
  + * - supports the parsing of multi-line messages and exceptions
  + *<p>
  + * <b>Keywords:</b><br>
  + * TIMESTAMP<br>
  + * LOGGER<br>
  + * LEVEL<br>
  + * THREAD<br>
  + * CLASS<br>
  + * FILE<br>
  + * LINE<br>
  + * METHOD<br>
  + * RELATIVETIME<br>
  + * MESSAGE<br>
  + * NDC<br>
  + * PROP(key)<br>
  + * <p>
  + * Use a * to ignore portions of the log format that should be ignored
  + * <p>
  + * Example:<br>
  + * If your file's patternlayout is this:<br>
  + * <b>%d %-5p [%t] %C{2} (%F:%L) - %m%n</b>
  + *<p>
  + * specify this as the log format:<br>
  + * <b>TIMESTAMP LEVEL [THREAD] CLASS (FILE:LINE) - MESSAGE</b>
  + *<p>
    * To define a PROPERTY field, use PROP(key)
  - * 
  - * For example, 
  - * 
  - * If you specified a userID field as an MDC entry in the file, 
  - * you can use PROP(userID) in the log format definition to access the userID MDC entry
as a property
  - *  
  - * If your file's patternlayout is this:
  - * %r [%t] %-5p %c %x - %m%n
  - *
  - * specify this as the log format:
  - * RELATIVETIME [THREAD] LEVEL LOGGER * - MESSAGE
  - *
  + * <p>
  + * Example:<br> 
  + * If you used the RELATIVETIME pattern layout character in the file, 
  + * you can use PROP(RELATIVETIME) in the logFormat definition to assign 
  + * the RELATIVETIME field as a property on the event.
  + * <p>
  + * If your file's patternlayout is this:<br>
  + * <b>%r [%t] %-5p %c %x - %m%n</b>
  + *<p>
  + * specify this as the log format:<br>
  + * <b>PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE</b>
  + * <p>
    * Note the * - it can be used to ignore a single word or sequence of words in the log
file
    * (in order for the wildcard to ignore a sequence of words, the text being ignored must
be
  - *  followed by some delimiter, like '-' or '[')
  - *
  - * Note how keywords may be surrounded by delimiters, and in the second example,
  - * ndc is ignored (even multiple words in the ndc in this case, since the keyword
  - * is followed by a delimiter (-)
  - * 
  + *  followed by some delimiter, like '-' or '[') - ndc is being ignored in this example.
  + * <p>
    * Assign a filterExpression in order to only process events which match a filter.
    * If a filterExpression is not assigned, all events are processed.
  - *
  - * LIMITATIONS:
  - * - specify delimited NDC or MDC entries using support for properties
  - * - no support for the single-line version of throwable supported by patternlayout
  - * - relativetime is set as a property
  - * - loggers with spaces in their names are not supported (but may work if followed by
a delimiter,
  - *   similar to wildcard example above)
  - * - messages must appear at the end of the line
  - * - note the explanation above describing the rules for ignoring text using the wildcard
  - *   keyword
  - * - exceptions will be converted if the exception stack trace lines (other than the first
line)
  - *   are stored in the log file with a tab as the first character
  - *
  - * EXAMPLE RECEIVER CONFIGURATION (add these as params, specifying a LogFilePatternReceiver
'plugin'
  - *
  - * param: "timestampFormat" value="yyyy-MM-d HH:mm:ss,SSS"
  - * param: "logFormat" value="RELATIVETIME [THREAD] LEVEL LOGGER * - MESSAGE"
  - * param: "fileURL" value="file:///c:/events.log"
  + *<p>
  + * <b>Limitations:</b><br>
  + * - no support for the single-line version of throwable supported by patternlayout<br>
  + *   (this version of throwable will be included as the last line of the message)<br>
  + * - the relativetime patternLayout character must be set as a property: PROP(RELATIVETIME)<br>
  + * - messages should appear as the last field of the logFormat because the variability
in message content<br>
  + * - exceptions are converted if the exception stack trace (other than the first line of
the exception)<br>
  + *   is stored in the log file with a tab followed by the word 'at' as the first characters
in the line<br>
  + * - tailing may fail if the file rolls over. 
  + *<p>
  + * <b>Example receiver configuration settings</b> (add these as params, specifying
a LogFilePatternReceiver 'plugin'):<br>
  + * param: "timestampFormat" value="yyyy-MM-d HH:mm:ss,SSS"<br>
  + * param: "logFormat" value="RELATIVETIME [THREAD] LEVEL LOGGER * - MESSAGE"<br>
  + * param: "fileURL" value="file:///c:/events.log"<br>
    * param: "tailing" value="true"
  + *<p>
  + * This configuration will be able to process these sample events:<br>
  + * 710    [       Thread-0] DEBUG                   first.logger first - <test> 
 <test2>something here</test2>   <test3 blah=something/>   <test4>
      <test5>something else</test5>   </test4></test><br>
  + * 880    [       Thread-2] DEBUG                   first.logger third - <test> 
 <test2>something here</test2>   <test3 blah=something/>   <test4>
      <test5>something else</test5>   </test4></test><br>
  + * 880    [       Thread-0] INFO                    first.logger first - infomsg-0<br>
  + * java.lang.Exception: someexception-first<br>
  + *     at Generator2.run(Generator2.java:102)<br>
    *
  - * The 'tailing' parameter allows the contents of the file to be continually read and new
events processed.
  - * 
  - * NOTE: in our example file content below, the timestampFormat entry defined above
  - * is not required, but included as an example of how to specify the format.  See SimpleDateFormat
  - * for more information.
  - *
  - * This configuration will be able to process these sample events:
  - * 710    [       Thread-0] DEBUG                   first.logger first - <test> 
 <test2>something here</test2>   <test3 blah=something/>   <test4>
      <test5>something else</test5>   </test4></test>
  - * 880    [       Thread-2] DEBUG                   first.logger third - <test> 
 <test2>something here</test2>   <test3 blah=something/>   <test4>
      <test5>something else</test5>   </test4></test>
  - * 880    [       Thread-0] INFO                    first.logger first - infomsg-0
  - * java.lang.Exception: someexception-first
  - *     at Generator2.run(Generator2.java:102)
  - *
  + *@author Scott Deboy
    */
   public class LogFilePatternReceiver extends Receiver {
  -  public static final String TIMESTAMP = "TIMESTAMP";
  -  public static final String LOGGER = "LOGGER";
  -  public static final String LEVEL = "LEVEL";
  -  public static final String THREAD = "THREAD";
  -  public static final String CLASS = "CLASS";
  -  public static final String FILE = "FILE";
  -  public static final String LINE = "LINE";
  -  public static final String METHOD = "METHOD";
  -  public static final String RELATIVETIME = "RELATIVETIME";
  -  public static final String MESSAGE = "MESSAGE";
  -  public static final String WILDCARD = "*";
  -  private static final String TAB = "\t";
  -  private static final String PROP_PREFIX = "PROP(";
  -  private static final String PROP_SUFFIX = ")";
  -  private final List keywords = new ArrayList();
  -  private final List logFormatFields = new ArrayList();
  -  private final Map defaultMap = new HashMap();
  +  private final Set keywords = new HashSet();
  +
  +  private static final String PROP_START = "PROP(";
  +  private static final String PROP_END = ")";
  +
  +  private static final String LOGGER = "LOGGER";
  +  private static final String MESSAGE = "MESSAGE";
  +  private static final String TIMESTAMP = "TIMESTAMP";
  +  private static final String NDC = "NDC";
  +  private static final String LEVEL = "LEVEL";
  +  private static final String THREAD = "THREAD";
  +  private static final String CLASS = "CLASS";
  +  private static final String FILE = "FILE";
  +  private static final String LINE = "LINE";
  +  private static final String METHOD = "METHOD";
  +
  +  private static final String EXCEPTION_PATTERN = "\t.*";
  +  private static final String REGEXP_WILDCARD = ".+";
  +  private static final String PATTERN_WILDCARD = "*";
  +  private static final String GROUP = "(" + REGEXP_WILDCARD + ")";
  +
  +  private static final String HOSTNAME_PROPERTY_VALUE = "file";
  +
  +  private final List matchingKeywords = new ArrayList();
  +  private final String newLine = System.getProperty("line.separator");
  +
  +  private final String[] emptyException = new String[] { "" };
  +
     private SimpleDateFormat dateFormat;
     private String timestampFormat = "yyyy-MM-d HH:mm:ss,SSS";
     private String logFormat;
     private String fileURL;
  -  private String shortFileName;
     private boolean tailing;
     private String filterExpression;
  +
  +  private final Perl5Util util = new Perl5Util();
  +  private final Perl5Compiler compiler = new Perl5Compiler();
  +  private final Perl5Matcher matcher = new Perl5Matcher();
  +
     private Rule expressionRule;
  +  private final Map currentMap = new HashMap();
  +  private final List additionalLines = new ArrayList();
  +  private String regexp;
  +  private Reader reader;
   
  -  /**
  -   * Creates a new LogFilePatternReceiver object.
  -   */
     public LogFilePatternReceiver() {
  -    //define defaults which may not be provided by log file but are required by log4j
  -    defaultMap.put(LOGGER, "Unknown");
  -
  -    //supported keyword replacements are expected to be single words, except for the MESSAGE
keyword,
  -    //which is expected to appear at the end of each entry in the log file
  -    //since throwable, ndc and properties can all have spaces, they're not yet supported
  -    //while loggers may containspaces, only loggers without spaces are currently supported
  -    //fullinfo pattern is not supported directly - build from individual keywords instead
       keywords.add(TIMESTAMP);
       keywords.add(LOGGER);
       keywords.add(LEVEL);
  @@ -180,62 +187,13 @@
       keywords.add(FILE);
       keywords.add(LINE);
       keywords.add(METHOD);
  -
  -    //wildcard removes any single word 
  -    //will also remove any number of words if the next entry in the log file format
  -    //is not found in the wildcard text itself 
  -    //(for example, 'LOGGER * [THREAD]' would successfully resolve keywords for 
  -    //'MYLOGGER any text here [Thread-0]'
  -    //but, 'LOGGER * THREAD' would not successfully resolve keywords for
  -    //'MYLOGGER any text here Thread-0'
  -    //but would successfully resolve
  -    //'MYLOGGER singleword Thread-0'
  -    keywords.add(WILDCARD);
  -
  -    //supported as a property on the event
  -    keywords.add(RELATIVETIME);
  -
  -    //keywords.add("NDC");
  -    //keywords.add("MDC");
  -    //keywords.add("THROWABLE");
  -    //keywords.add("PROPERTIES");
       keywords.add(MESSAGE);
  -  }
  -
  -  /**
  -   * Test
  -   *
  -   * @param args file url to parse
  -   */
  -  public static void main(String[] args) {
  -    LogFilePatternReceiver parser = new LogFilePatternReceiver();
  -    parser.setTimestampFormat("yyyy-MM-d HH:mm:ss,SSS");
  -
  -    //parser.parse(args[0]);
  -    //%d %-5p [%t] %C{2} (%F:%L) - %m%n -- a2
  -    //parser.setLogFormat("TIMESTAMP LEVEL * [THREAD] CLASS (FILE:LINE) - MESSAGE");
  -    //parser.initialize();
  -    //System.out.println("Created event: " + parser.convertToEvent("2004-12-13 22:49:22,820
DEBUG SOME VALUE [Thread-0] Generator2 (Generator2.java:100) - <test>   <test2>something
here</test2>   <test3 blah=something/>   <test4>       <test5>something
else</test5>   </test4></test>"));
  -    //parser.initialize("THREAD LEVEL LOGGER - MESSAGE");
  -    //parser.setLogFormat("RELATIVETIME PROP(USERID) [THREAD] LEVEL LOGGER * - MESSAGE");
  -    parser.setFileURL(args[0]);
  -    parser.initialize();
  -
  -    try {
  -      //expects an unbuffered reader
  -      parser.process(new InputStreamReader(new URL(parser.getFileURL()).openStream()));
  -
  -      //parser.process(new StringReader("2004-12-13 22:49:22,820 DEBUG SOME VALUE [Thread-0]
Generator2 (Generator2.java:100) - <test>   <test2>something here</test2>
  <test3 blah=something/>   <test4>       <test5>something else</test5>
  </test4></test>\nException blah\n\tat someplace.java:555\n\tat nextline blah:55"));
  -    } catch (IOException ioe) {
  -      ioe.printStackTrace();
  -    }
  -
  -    //comment out doPost calls to run via main
  +    keywords.add(NDC);
     }
   
     /**
      * Accessor
  -   *
  +   * 
      * @return file URL
      */
     public String getFileURL() {
  @@ -244,21 +202,16 @@
   
     /**
      * Mutator
  -   *
  +   * 
      * @param fileURL
      */
     public void setFileURL(String fileURL) {
       this.fileURL = fileURL;
  -    try {
  -    	shortFileName = new URL(fileURL).getFile();
  -    } catch (MalformedURLException mue) {
  -    	shortFileName = fileURL;
  -    }
     }
   
     /**
      * Accessor
  -   *
  +   * 
      * @return filter expression
      */
     public String getFilterExpression() {
  @@ -267,7 +220,7 @@
   
     /**
      * Mutator
  -   *
  +   * 
      * @param filterExpression
      */
     public void setFilterExpression(String filterExpression) {
  @@ -276,7 +229,7 @@
   
     /**
      * Accessor
  -   *
  +   * 
      * @return tailing
      */
     public boolean isTailing() {
  @@ -285,7 +238,7 @@
   
     /**
      * Mutator
  -   *
  +   * 
      * @param tailing
      */
     public void setTailing(boolean tailing) {
  @@ -294,7 +247,7 @@
   
     /**
      * Accessor
  -   *
  +   * 
      * @return log format
      */
     public String getLogFormat() {
  @@ -303,8 +256,9 @@
   
     /**
      * Mutator
  -   *
  -   * @param logFormat the format
  +   * 
  +   * @param logFormat
  +   *          the format
      */
     public void setLogFormat(String logFormat) {
       this.logFormat = logFormat;
  @@ -312,7 +266,7 @@
   
     /**
      * Mutator
  -   *
  +   * 
      * @param timestampFormat
      */
     public void setTimestampFormat(String timestampFormat) {
  @@ -321,7 +275,7 @@
   
     /**
      * Accessor
  -   *
  +   * 
      * @return timestamp format
      */
     public String getTimestampFormat() {
  @@ -329,124 +283,153 @@
     }
   
     /**
  -   * Convert a reader into a buffered reader, reading lines and converting lines into log
events
  -   *
  -   * @param unbufferedReader
  -   *
  -   * @throws IOException
  +   * Walk the additionalLines list, looking for the EXCEPTION_PATTERN.
  +   * <p>
  +   * Return the index of the first matched line minus 1 
  +   * (the match is the 2nd line of an exception)
  +   * <p>
  +   * Assumptions: <br>
  +   * - the additionalLines list may contain both message and exception lines<br>
  +   * - message lines are added to the additionalLines list and then
  +   * exception lines (all message lines occur in the list prior to all 
  +   * exception lines)
  +   * 
  +   * @return -1 if no exception line exists, line number otherwise
      */
  -  public void process(Reader unbufferedReader) throws IOException {
  -    BufferedReader reader = new BufferedReader(unbufferedReader);
  -    LinkedList list = new LinkedList();
  -    String line = null;
  -
  -    do {
  -      while ((line = reader.readLine()) != null) {
  -        //ignore blank lines in file
  -        if (line.length() == 0) {
  -          continue;
  +  private int getExceptionLine() {
  +    try {
  +      Pattern exceptionPattern = compiler.compile(EXCEPTION_PATTERN);
  +      for (int i = 0; i < additionalLines.size(); i++) {
  +        if (matcher.matches((String) additionalLines.get(i), exceptionPattern)) {
  +          return i - 1;
           }
  +      }
  +    } catch (MalformedPatternException mpe) {
  +      LogLog.warn("Bad pattern: " + EXCEPTION_PATTERN);
  +    }
  +    return -1;
  +  }
   
  -        //System.out.println("Added: " + line);
  -        list.addLast(line);
  -
  -        if (list.size() > 2) {
  -          //System.out.println("size > 2 - processing");
  -          if (line.startsWith(TAB)) {
  -            String firstLine = (String) list.remove(0);
  -
  -            //System.out.println("exception line1: " + firstLine);
  -            while ((line != null) && line.startsWith(TAB)) {
  -              line = reader.readLine();
  -
  -              if ((line != null) && (line.length() != 0)) {
  -                list.addLast(line);
  -              }
  -            }
  -
  -            //hold a reference to the last entry added - while loop adds one too many
  -            String nextLine = (String) list.getLast();
  +  /**
  +   * Combine all message lines occuring in the additionalLines list, adding
  +   * a newline character between each line
  +   * <p>
  +   * the event will already have a message - combine this message
  +   * with the message lines in the additionalLines list 
  +   * (all entries prior to the exceptionLine index)
  +   * 
  +   * @param firstMessageLine primary message line
  +   * @param exceptionLine index of first exception line
  +   * @return message
  +   */
  +  private String buildMessage(String firstMessageLine, int exceptionLine) {
  +    if (additionalLines.size() == 0 || exceptionLine == 0) {
  +      return firstMessageLine;
  +    }
  +    StringBuffer message = new StringBuffer(firstMessageLine);
   
  -            if ((nextLine != null) && !(nextLine.startsWith(TAB))) {
  -              list.removeLast();
  -            }
  +    for (int i = 0; i < exceptionLine; i++) {
  +      message.append(newLine);
  +      message.append(additionalLines.get(i));
  +    }
  +    return message.toString();
  +  }
   
  -            String[] exception = new String[list.size()];
  +  /**
  +   * Combine all exception lines occuring in the additionalLines list into a 
  +   * String array
  +   * <p>
  +   * (all entries equal to or greater than the exceptionLine index)
  +   * 
  +   * @param exceptionLine index of first exception line
  +   * @return exception
  +   */
  +  private String[] buildException(int exceptionLine) {
  +    if (exceptionLine == -1) {
  +      return emptyException;
  +    }
  +    String[] exception = new String[additionalLines.size() - exceptionLine];
  +    for (int i = 0; i < additionalLines.size() - exceptionLine; i++) {
  +      exception[i] = (String) additionalLines.get(i + exceptionLine);
  +    }
  +    return exception;
  +  }
   
  -            for (int i = 0, j = list.size(); i < j; i++) {
  -              exception[i] = (String) list.remove(0);
  +  /**
  +   * Construct a logging event from currentMap and additionalLines 
  +   * (additionalLines contains multiple message lines and any exception lines)
  +   * <p>
  +   * CurrentMap and additionalLines are cleared in the process
  +   * 
  +   * @return event
  +   */
  +  private LoggingEvent buildEvent() {
  +    LoggingEvent event = null;
   
  -              //System.out.println("exception " + i + ".." + exception[i]);
  -            }
  +    if (currentMap.size() == 0) {
  +      return event;
  +    }
  +    int exceptionLine = getExceptionLine();
  +    String[] exception = buildException(exceptionLine);
   
  -            //now that exceptions have been taken from list, re-add last entry if not an

  -            //exception line
  -            if ((nextLine != null) && !(nextLine.startsWith(TAB))) {
  -              list.addLast(nextLine);
  -            }
  +    //messages are listed before exceptions in additionallines
  +    if (additionalLines.size() > 0 && exceptionLine != 0) {
  +      currentMap.put(MESSAGE, buildMessage((String) currentMap.get(MESSAGE),
  +          exceptionLine));
  +    }
   
  -            //GENERATE EXCEPTION EVENT
  -            LoggingEvent event = convertToEvent(firstLine, exception);
  +    if (currentMap.size() > 0) {
  +      event = convertToEvent(currentMap, exception);
  +    }
  +    currentMap.clear();
  +    additionalLines.clear();
   
  -            //System.out.println(
  -            //  "created event with exception " + event.getLoggerName() + ".."
  -            //  + event.getMessage());
  -            if (event != null) {
  -              if (expressionRule != null) {
  -              	if (expressionRule.evaluate(event)) {
  -              		doPost(event);
  -              	}
  -              } else {
  -              	doPost(event);
  -              }
  -            }
  -          } else {
  -            //GENERATE NON-EXCEPTION EVENT
  -            LoggingEvent event = convertToEvent((String) list.remove(0));
  -
  -            if (event != null) {
  -                if (expressionRule != null) {
  -                  	if (expressionRule.evaluate(event)) {
  -                  		doPost(event);
  -                  	}
  -                } else {
  -                	doPost(event);
  -                }
  -            }
  +    return event;
  +  }
   
  -            //System.out.println(
  -            //  "Created event " + event.getLoggerName() + ".."
  -            //  + event.getMessage());
  -          }
  -        }
  -      }
  +  /**
  +   * Read, parse and optionally tail the log file, converting entries into logging events
  +   * <p>
  +   * A runtimeException is thrown if the logFormat pattern is malformed 
  +   * according to ORO's Perl5Compiler.
  +   * 
  +   * @param unbufferedReader
  +   * @throws IOException
  +   */
  +  private void process(Reader unbufferedReader) throws IOException {
  +    BufferedReader bufferedReader = new BufferedReader(unbufferedReader);
   
  -      //System.out.println(
  -      //  "outside loop - processing remaining entries: " + list.size());
  -      //clean up remaining - should not be an exception
  -      for (int k = 0, l = list.size(); k < l; k++) {
  -        String s = (String) list.remove(0);
  -
  -        if ((s != null) && (s.length() > 0)) {
  -          //GENERATE NON-EXCEPTION EVENT
  -          LoggingEvent event = convertToEvent(s);
  +    Pattern regexpPattern = null;
  +    try {
  +      regexpPattern = compiler.compile(regexp);
  +    } catch (MalformedPatternException mpe) {
  +      throw new RuntimeException("Bad pattern: " + regexp);
  +    }
   
  +    String line = null;
  +    do {
  +      while ((line = bufferedReader.readLine()) != null) {
  +        if (matcher.matches(line, regexpPattern)) {
  +          LoggingEvent event = buildEvent();
             if (event != null) {
  -            if (expressionRule != null) {
  -              	if (expressionRule.evaluate(event)) {
  -              		doPost(event);
  -              	}
  -            } else {
  -            	doPost(event);
  +            if (passesExpression(event)) {
  +              doPost(event);
               }
             }
  -
  -          //System.out.println(
  -          //  "cleanup - Created non-exception event " + event.getLoggerName()
  -          //  + ".." + event.getMessage());
  +          currentMap.putAll(processEvent(matcher.getMatch()));
  +        } else {
  +          //may be an exception or additional message lines
  +          additionalLines.add(line);
           }
         }
   
  +      //process last event if one exists
  +      LoggingEvent event = buildEvent();
  +      if (event != null) {
  +        if (passesExpression(event)) {
  +          doPost(event);
  +        }
  +      }
         try {
           synchronized (this) {
             wait(2000);
  @@ -454,141 +437,172 @@
         } catch (InterruptedException ie) {
         }
       } while (tailing);
  -    try {
  -      if (reader != null) {
  -          reader.close();
  +
  +    shutdown();
  +  }
  +
  +  /**
  +   * Helper method that supports the evaluation of the expression
  +   * 
  +   * @param event
  +   * @return true if expression isn't set, or the result of the evaluation otherwise 
  +   */
  +  private boolean passesExpression(LoggingEvent event) {
  +    if (event != null) {
  +      if (expressionRule != null) {
  +        return (expressionRule.evaluate(event));
         }
  -    } catch (IOException ioe) {
  -      ioe.printStackTrace();
       }
  +    return true;
     }
   
     /**
  -   * Walk the entries in the log format fields, adding keyword matches to a
  -   * map
  -   *
  -   * @param logEntry
  -   *
  -   * @return map of keywords to values in the log entry
  -   */
  -  private Map extractEventFields(String logEntry) {
  -    //System.out.println("extracting from " + logEntry);
  -    Map fieldMap = new HashMap(defaultMap);
  -
  -    for (int i = 0; i < logFormatFields.size(); i++) {
  -      String thisField = (String) logFormatFields.get(i);
  -      //LogLog.info("processing field " + thisField);
  -
  -      //ignore wildcard entries - skip to next field
  -      if (thisField.equals(WILDCARD)) {
  -        String nextField = (String) logFormatFields.get(i + 1);
  -
  -        if (logEntry.indexOf(nextField) == -1) {
  -          LogLog.info(nextField + " does not exist in line - ignoring: " + logEntry);
  -
  -          return null;
  -        }
  -
  -        logEntry = logEntry.substring(logEntry.indexOf(nextField));
  -      } else if (thisField.equals(TIMESTAMP)) {
  -        String nextField = (String) logFormatFields.get(i + 1);
  +   * Convert the ORO match into a map.
  +   * <p>
  +   * Relies on the fact that the matchingKeywords list is in the same
  +   * order as the groups in the regular expression
  +   *  
  +   * @param result
  +   * @return map
  +   */
  +  private Map processEvent(MatchResult result) {
  +    Map map = new HashMap();
  +    //group zero is the entire match - process all other groups
  +    for (int i = 1; i < result.groups(); i++) {
  +      map.put(matchingKeywords.get(i - 1), result.group(i));
  +    }
  +    return map;
  +  }
   
  -        if (logEntry.indexOf(nextField) == -1) {
  -          LogLog.info(nextField + " does not exist in line - ignoring: " + logEntry);
  +  /**
  +   * Build the regular expression needed to parse log entries
  +   *  
  +   */
  +  private void initialize() {
  +    if (timestampFormat != null) {
  +      dateFormat = new SimpleDateFormat(timestampFormat);
  +    }
   
  -          return null;
  -        }
  +    try {
  +      if (filterExpression != null) {
  +        expressionRule = ExpressionRule.getRule(filterExpression);
  +      }
  +    } catch (Exception e) {
  +      LogLog.warn("Invalid filter expression: " + filterExpression, e);
  +    }
   
  -        //uses nextfield and length of format to guess at timestamp field
  -        //luckily, SimpleDateFormat is very flexible and forgiving of trailing text
  -        int firstLength =
  -          logEntry.substring(0, timestampFormat.length()).length() - 1;
  -        int nextLength =
  -          (firstLength
  -          + logEntry.substring(firstLength - 1).indexOf(nextField)) - 1;
  -
  -        int length = Math.max(firstLength, nextLength);
  -
  -        //        System.out.println("parsing timestamp: " + logEntry.substring(0, length));
  -        fieldMap.put(thisField, logEntry.substring(0, length));
  -
  -        //System.out.println(
  -        //  "added " + thisField + ":" + fieldMap.get(thisField));
  -        logEntry = logEntry.substring(length);
  -      } else if (keywords.contains(thisField) || thisField.startsWith(PROP_PREFIX)) {
  -        logEntry = logEntry.trim();
  -
  -        if (i < (logFormatFields.size() - 1)) {
  -          //expects no two keywords to be butted up directly against eachother 
  -          String nextField = (String) logFormatFields.get(i + 1);
  +    Map keywordMap = new TreeMap();
   
  -          if (logEntry.indexOf(nextField) == -1) {
  -          	LogLog.info(nextField + " does not exist in line - ignoring: " + logEntry);
  +    String newPattern = logFormat;
   
  -            return null;
  -          }
  +    /*
  +     * examine pattern, adding properties to an index-based map.
  +     * 
  +     * Replaces PROP(X) definitions in the pattern with the short version X, so 
  +     * that the name can be used as the event property later 
  +     */
  +    int index = 0;
  +    String current = newPattern;
  +    while (index > -1) {
  +      index = current.indexOf(PROP_START);
  +      if (index > -1) {
  +        String currentProp = current.substring(current.indexOf(PROP_START));
  +        String prop = currentProp.substring(0,
  +            currentProp.indexOf(PROP_END) + 1);
  +        current = current.substring(current.indexOf(currentProp) + 1);
  +        String shortProp = prop.substring(PROP_START.length(),
  +            prop.length() - 1);
  +        keywordMap.put(new Integer(index), shortProp);
  +        newPattern = replace(prop, shortProp, newPattern);
  +      }
  +    }
   
  -          fieldMap.put(
  -            thisField,
  -            logEntry.substring(0, logEntry.indexOf(nextField)).trim());
  -
  -          //System.out.println(
  -          //  "added " + thisField + ":" + fieldMap.get(thisField));
  -          logEntry = logEntry.substring(logEntry.indexOf(nextField));
  -        } else {
  -          fieldMap.put(thisField, logEntry.trim());
  +    newPattern = replaceMetaChars(newPattern);
  +    newPattern = replace(PATTERN_WILDCARD, REGEXP_WILDCARD, newPattern);
   
  -          //System.out.println(
  -          //  "added " + thisField + ":" + fieldMap.get(thisField));
  -        }
  -      } else {
  -        //ignore non-fields
  -        //System.out.println("text: " + thisField);
  -        logEntry = logEntry.substring(thisField.length());
  +    /*
  +     * we're using a treemap, so the index will be used as the key to ensure
  +     * keywords are ordered correctly
  +     * 
  +     * examine pattern, adding keywords to an index-based map patterns can
  +     * contain only one of these per entry...properties are the only 'keyword'
  +     * that can occur multiple times in an entry
  +     */
  +    Iterator iter = keywords.iterator();
  +    while (iter.hasNext()) {
  +      String keyword = (String) iter.next();
  +      int index2 = newPattern.indexOf(keyword);
  +      if (index2 > -1) {
  +        keywordMap.put(new Integer(index2), keyword);
         }
       }
   
  -    return fieldMap;
  +    //keywordMap should be ordered by index..add all values to a list
  +    matchingKeywords.addAll(keywordMap.values());
  +
  +    /*
  +     * iterate over the keywords found in the pattern and replace with regexp
  +     * group
  +     */
  +    String currentPattern = newPattern;
  +    Iterator iter2 = matchingKeywords.iterator();
  +    while (iter2.hasNext()) {
  +      String keyword = (String) iter2.next();
  +      currentPattern = replace(keyword, GROUP, currentPattern);
  +    }
  +    regexp = currentPattern;
     }
   
     /**
  -   * Helper convert method that doesn't support exceptions
  -   *
  -   * @param logEntry
  -   *
  -   * @return logging event
  +   * Helper method that will globally replace a section of text
  +   * 
  +   * @param pattern
  +   * @param replacement
  +   * @param input 
  +   * 
  +   * @return string
      */
  -  private LoggingEvent convertToEvent(String logEntry) {
  -    return convertToEvent(logEntry, null);
  +  private String replace(String pattern, String replacement, String input) {
  +    return util.substitute("s/" + Perl5Compiler.quotemeta(pattern) + "/"
  +        + Perl5Compiler.quotemeta(replacement) + "/g", input);
     }
   
     /**
  -   * Convert log entry with exceptions
  -   *
  -   * @param logEntry
  -   * @param exception string array of exception lines
  -   *
  -   * @return logging event
  +   * Some perl5 characters may occur in the log file format.  
  +   * Escape these characters to prevent parsing errors.
  +   * 
  +   * @param input
  +   * @return string
      */
  -  private LoggingEvent convertToEvent(String logEntry, String[] exception) {
  -    return convertToEvent(extractEventFields(logEntry), exception);
  +  private String replaceMetaChars(String input) {
  +    input = replace("(", "\\(", input);
  +    input = replace(")", "\\)", input);
  +    input = replace("[", "\\[", input);
  +    input = replace("]", "\\]", input);
  +    input = replace("{", "\\{", input);
  +    input = replace("}", "\\}", input);
  +    return input;
     }
   
     /**
  -   * Convert entries in the map of keywords to values in a log entry into a
  -   * loggingEvent
  -   *
  +   * Convert a keyword-to-values map to a LoggingEvent
  +   * 
      * @param fieldMap
      * @param exception
  -   *
  +   * 
      * @return logging event
      */
     private LoggingEvent convertToEvent(Map fieldMap, String[] exception) {
       if (fieldMap == null) {
         return null;
       }
  +
  +    //a logger must exist at a minimum for the event to be processed
  +    if (!fieldMap.containsKey(LOGGER)) {
  +      fieldMap.put(LOGGER, "Unknown");
  +    }
       if (exception == null) {
  -      exception = new String[]{""};
  +      exception = emptyException;
       }
   
       Logger logger = null;
  @@ -596,68 +610,59 @@
       String level = null;
       String threadName = null;
       Object message = null;
  -
  -    //    String ndc = null;
  -    //    Hashtable mdc = null;
  +    String ndc = null;
       String className = null;
       String methodName = null;
       String eventFileName = null;
       String lineNumber = null;
       Hashtable properties = new Hashtable();
   
  +    logger = Logger.getLogger((String) fieldMap.remove(LOGGER));
  +
       if ((dateFormat != null) && fieldMap.containsKey(TIMESTAMP)) {
         try {
  -        timeStamp =
  -          dateFormat.parse((String) fieldMap.get(TIMESTAMP)).getTime();
  +        timeStamp = dateFormat.parse((String) fieldMap.remove(TIMESTAMP))
  +            .getTime();
         } catch (Exception e) {
           e.printStackTrace();
         }
       }
  -
  +    //use current time if timestamp not parseable
       if (timeStamp == 0L) {
         timeStamp = System.currentTimeMillis();
       }
   
  -    logger = Logger.getLogger((String) fieldMap.get(LOGGER));
  +    level = (String) fieldMap.remove(LEVEL);
  +    Level levelImpl = Level.toLevel(level);
   
  -    level = (String) fieldMap.get(LEVEL);
  -    threadName = (String) fieldMap.get(THREAD);
  +    threadName = (String) fieldMap.remove(THREAD);
   
  -    message = (String) fieldMap.get(MESSAGE);
  +    message = (String) fieldMap.remove(MESSAGE);
   
  -    className = (String) fieldMap.get(CLASS);
  -    methodName = (String) fieldMap.get(METHOD);
  -    eventFileName = (String) fieldMap.get(FILE);
  -    lineNumber = (String) fieldMap.get(LINE);
  +    ndc = (String) fieldMap.remove(NDC);
   
  -    if (fieldMap.get(RELATIVETIME) != null) {
  -      properties.put(RELATIVETIME, fieldMap.get(RELATIVETIME));
  -    }
  +    className = (String) fieldMap.remove(CLASS);
   
  -    Level levelImpl = Level.toLevel(level);
  +    methodName = (String) fieldMap.remove(METHOD);
  +
  +    eventFileName = (String) fieldMap.remove(FILE);
  +
  +    lineNumber = (String) fieldMap.remove(LINE);
  +
  +    properties.put(Constants.HOSTNAME_KEY, HOSTNAME_PROPERTY_VALUE);
  +    properties.put(Constants.APPLICATION_KEY, fileURL);
  +
  +    //all remaining entries in fieldmap are properties
  +    properties.putAll(fieldMap);
   
       LocationInfo info = null;
   
  -    if (
  -      (eventFileName != null) || (className != null) || (methodName != null)
  +    if ((eventFileName != null) || (className != null) || (methodName != null)
           || (lineNumber != null)) {
  -      info =
  -        new LocationInfo(eventFileName, className, methodName, lineNumber);
  +      info = new LocationInfo(eventFileName, className, methodName, lineNumber);
       } else {
         info = LocationInfo.NA_LOCATION_INFO;
       }
  -    properties.put(Constants.HOSTNAME_KEY, "file");
  -    properties.put(Constants.APPLICATION_KEY, shortFileName);
  -
  -    //add properties 
  -    Iterator iter = fieldMap.keySet().iterator();
  -    while (iter.hasNext()) {
  -    	String key = (String)iter.next();
  -    	if (key.startsWith(PROP_PREFIX)) {
  -    		String prop = key.substring(PROP_PREFIX.length(), key.length() - 1).trim();
  -    		properties.put(prop, fieldMap.get(key));
  -    	}
  -    }
   
       LoggingEvent event = new LoggingEvent();
       event.setLogger(logger);
  @@ -667,104 +672,61 @@
       event.setMessage(message);
       event.setThrowableInformation(new ThrowableInformation(exception));
       event.setLocationInformation(info);
  +    event.setNDC(ndc);
       event.setProperties(properties);
  -   
  +
       return event;
     }
   
  -  /**
  -   * Initialize and post log entries to framework
  -   */
  -  public void activateOptions() {
  -  	try {
  -  		if (filterExpression != null) {
  -  			expressionRule = ExpressionRule.getRule(filterExpression);
  -  		}
  -  	} catch (Exception e) {
  -  		LogLog.warn("Invalid filter expression: " + filterExpression, e);
  -  	}
  -    new Thread(
  -      new Runnable() {
  -        public void run() {
  -          initialize();
  -
  -          try {
  -          	process(new InputStreamReader(new URL(getFileURL()).openStream()));
  -          } catch (IOException ioe) {
  -            ioe.printStackTrace();
  -          }
  -        }
  -      }).start();
  +  public static void main(String[] args) {
  +    /*
  +    LogFilePatternReceiver test = new LogFilePatternReceiver();
  +    test.setLogFormat("TIMESTAMP LEVEL [THREAD] LOGGER (FILE:LINE) - MESSAGE");
  +    test.setTailing(true);
  +    test.setFileURL("file:///C:/downloads/workspace/test/test2.log");
  +    test.initialize();
  +    try {
  +      test.process(new InputStreamReader(new URL(test.getFileURL())
  +          .openStream()));
  +    } catch (IOException ioe) {
  +      ioe.printStackTrace();
  +    }
  +    */
     }
   
     /**
  -   * Walk the passed-in log format, building a list of entries which are (from
  -   * left to right), either a keyword, or any string of characters not a
  -   * keyword.
  +   * Close the reader. 
      */
  -  private void initialize() {
  -    if (timestampFormat != null) {
  -      dateFormat = new SimpleDateFormat(timestampFormat);
  -    }
  -
  -    int i = 0;
  -    int j = 0;
  -
  -    while (i < logFormat.length()) {
  -      String keyword = endsInKeyword(logFormat.substring(i, j));
  -
  -      //System.out.println("current arg is " + arg.substring(i,j));
  -      //fragment ends in keyword.  
  -      //add everything before the keyword as an entry in the list
  -      //then add the keyword
  -      if (keyword != null) {
  -        String firstPart = logFormat.substring(i, j - keyword.length());
  -
  -        if (firstPart.length() > 0) {
  -          //System.out.println("ADDED FIRSTPART: " + firstPart);
  -          logFormatFields.add(firstPart);
  -          
  -        }
  -
  -        logFormatFields.add(keyword);
  -
  -        //System.out.println("ADDED: " + keyword);
  -        i = j;
  -        j = j + 1;
  -      } else {
  -        j++;
  +  public void shutdown() {
  +    try {
  +      if (reader != null) {
  +        reader.close();
         }
  +    } catch (IOException ioe) {
  +      ioe.printStackTrace();
       }
     }
   
     /**
  -   * Check to see if the log format fragment passed in ends with a keyword.
  -   *
  -   * @param logFormatFragment
  -   *
  -   * @return keyword or null
  +   * Read and process the log file.
      */
  -  private String endsInKeyword(String logFormatFragment) {
  -  	String trimmedFragment = logFormatFragment.trim();
  -    Iterator iter = keywords.iterator();
  -
  -    while (iter.hasNext()) {
  -      String keyword = (String) iter.next();
  -
  -      if (trimmedFragment.endsWith(keyword)) {
  -        return keyword;
  +  public void activateOptions() {
  +    new Thread(new Runnable() {
  +      public void run() {
  +        initialize();
  +        try {
  +          reader = new InputStreamReader(new URL(getFileURL()).openStream());
  +        } catch (IOException ioe) {
  +          LogLog.debug("exception", ioe);
  +          return;
  +        }
  +        try {
  +          process(reader);
  +        } catch (IOException ioe) {
  +          //io exception - probably shut down
  +          LogLog.debug("stream closed");
  +        }
         }
  -    }
  -    if ((trimmedFragment.indexOf(PROP_PREFIX)> -1) && trimmedFragment.endsWith(PROP_SUFFIX))
{
  -    	return trimmedFragment.substring(trimmedFragment.indexOf(PROP_PREFIX));
  -    }
  -
  -    return null;
  -  }
  -
  -  /* (non-Javadoc)
  -   * @see org.apache.log4j.plugins.Plugin#shutdown()
  -   */
  -  public void shutdown() {
  +    }).start();
     }
  -}
  +}
  \ No newline at end of file
  
  
  

---------------------------------------------------------------------
To unsubscribe, e-mail: log4j-dev-unsubscribe@logging.apache.org
For additional commands, e-mail: log4j-dev-help@logging.apache.org


Mime
View raw message