chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1411813 - in /incubator/chukwa/trunk: ./ src/main/java/org/apache/hadoop/chukwa/database/ src/main/java/org/apache/hadoop/chukwa/datacollection/writer/ src/main/java/org/apache/hadoop/chukwa/dataloader/ src/main/java/org/apache/hadoop/chuk...
Date Tue, 20 Nov 2012 19:39:04 GMT
Author: eyang
Date: Tue Nov 20 19:39:03 2012
New Revision: 1411813

URL: http://svn.apache.org/viewvc?rev=1411813&view=rev
Log:
CHUKWA-642. Added regular expression validation. (Eric Spishak via Eric Yang)

Modified:
    incubator/chukwa/trunk/CHANGES.txt
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java
    incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java
    incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java
    incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java
    incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java

Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Tue Nov 20 19:39:03 2012
@@ -24,6 +24,8 @@ Trunk (unreleased changes)
 
   BUGS
 
+    CHUKWA-642. Added regular expression validation. (Eric Spishak via Eric Yang)
+
     CHUKWA-632. Added getter method for startOffset in AbstractProcessor. (Abhijit Dhar via
Eric Yang)
 
     CHUKWA-637. Fixed default cluster selection in hicc. (Eric Yang)

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java
(original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java
Tue Nov 20 19:39:03 2012
@@ -27,6 +27,7 @@ import java.util.Iterator;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.chukwa.util.DatabaseWriter;
+import org.apache.hadoop.chukwa.util.RegexUtil;
 
 public class DataExpiration {
   private static DatabaseConfig dbc = null;
@@ -50,6 +51,12 @@ public class DataExpiration {
       while (ki.hasNext()) {
         String name = ki.next();
         String tableName = dbNames.get(name);
+        if (!RegexUtil.isRegex(tableName)) {
+          log.warn("Skipping tableName: '" + tableName
+              + "' because there was an error parsing it as a regex: "
+              + RegexUtil.regexError(tableName));
+          return;
+        }
         String[] tableList = dbc.findTableName(tableName, start, end);
         for (String tl : tableList) {
           log.debug("table name: " + tableList[0]);

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java
(original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java
Tue Nov 20 19:39:03 2012
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.chukwa.util.DatabaseWriter;
 import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil;
 
 public class TableCreator {
   private static DatabaseConfig dbc = null;
@@ -56,6 +57,12 @@ public class TableCreator {
     while (ki.hasNext()) {
       String name = ki.next();
       String tableName = dbNames.get(name);
+      if (!RegexUtil.isRegex(tableName)) {
+        log.warn("Skipping tableName: '" + tableName
+            + "' because there was an error parsing it as a regex: "
+            + RegexUtil.regexError(tableName));
+        return;
+      }
       String[] tableList = dbc.findTableName(tableName, start, end);
       log.debug("table name: " + tableList[0]);
       try {

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java
(original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java
Tue Nov 20 19:39:03 2012
@@ -23,6 +23,8 @@ import java.util.concurrent.ArrayBlockin
 import java.util.regex.PatternSyntaxException;
 import org.apache.hadoop.chukwa.Chunk;
 import org.apache.hadoop.chukwa.util.Filter;
+import org.apache.hadoop.chukwa.util.RegexUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.log4j.Logger;
 import java.net.ServerSocket;
@@ -167,7 +169,7 @@ public class SocketTeeWriter extends Pip
      */
     public void setup() {
       try {   //outer try catches IOExceptions
-       try { //inner try catches Pattern Syntax errors
+       try { //inner try catches bad command syntax errors
         sock.setSoTimeout(timeout);
         sock.setKeepAlive(USE_KEEPALIVE);
         in = new BufferedReader(new InputStreamReader(sock.getInputStream()));
@@ -175,8 +177,8 @@ public class SocketTeeWriter extends Pip
         String cmd = in.readLine();
         if(!cmd.contains(" ")) {
           
-          throw new PatternSyntaxException(
-              "command should be keyword pattern, but no ' ' seen", cmd, -1);
+          throw new IllegalArgumentException(
+              "command should be keyword pattern, but no ' ' seen: " + cmd);
         }
         String uppercased = cmd.substring(0, cmd.indexOf(' ')).toUpperCase();
         if(RAW.equals(uppercased))
@@ -186,16 +188,27 @@ public class SocketTeeWriter extends Pip
         else if(ASCII_HEADER.equals(uppercased))
           fmt = DataFormat.Header;
         else {
-          throw new PatternSyntaxException("bad command '" + uppercased+
+          throw new IllegalArgumentException("bad command '" + uppercased+
               "' -- starts with neither '"+ RAW+ "' nor '"+ WRITABLE + " nor " 
-              + ASCII_HEADER+"'.", cmd, -1);
+              + ASCII_HEADER+"':" + cmd);
         }
         
         String cmdAfterSpace = cmd.substring(cmd.indexOf(' ')+1);
         if(cmdAfterSpace.toLowerCase().equals("all"))
           rules = Filter.ALL;
         else
-          rules = new Filter(cmdAfterSpace);
+          try {
+            rules = new Filter(cmdAfterSpace);
+          } catch (CheckedPatternSyntaxException pse) {
+            out.write("Error parsing command as a regex: ".getBytes());
+            out.write(pse.getMessage().getBytes());
+            out.writeByte('\n');
+            out.close();
+            in.close();
+            sock.close();
+            log.warn(pse);
+            return;
+          }
 
           //now that we read everything OK we can add ourselves to list, and return.
         synchronized(tees) {
@@ -203,7 +216,7 @@ public class SocketTeeWriter extends Pip
         }
         out.write("OK\n".getBytes());
         log.info("tee to " + sock.getInetAddress() + " established");
-      } catch(PatternSyntaxException e) {
+      } catch(IllegalArgumentException e) {
           out.write(e.toString().getBytes());
           out.writeByte('\n');
           out.close();

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java
(original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java
Tue Nov 20 19:39:03 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.chukwa.extracti
 import org.apache.hadoop.chukwa.util.ClusterConfig;
 import org.apache.hadoop.chukwa.util.DatabaseWriter;
 import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.SequenceFile;
@@ -220,7 +221,13 @@ public class MetricDataLoader implements
         String dbKey = "report.db.name." + recordType;
         Matcher m = p.matcher(recordType);
         if (dbTables.containsKey(dbKey)) {
-          String[] tmp = mdlConfig.findTableName(mdlConfig.get(dbKey), record
+          String tableName = mdlConfig.get(dbKey);
+          if (!RegexUtil.isRegex(tableName)) {
+            log.error("Error parsing 'tableName' as a regex: "
+                + RegexUtil.regexError(tableName));
+            return false;
+          }
+          String[] tmp = mdlConfig.findTableName(tableName, record
               .getTime(), record.getTime());
           table = tmp[0];
         } else if(m.matches()) {

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
(original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
Tue Nov 20 19:39:03 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.chukwa.datacoll
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
 import org.apache.hadoop.chukwa.extraction.demux.Demux;
+import org.apache.hadoop.chukwa.util.RegexUtil;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.JobConf;
@@ -65,6 +66,8 @@ public class TsProcessor extends Abstrac
   static Logger log = Logger.getLogger(TsProcessor.class);
 
   public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
+  public static final String DEFAULT_TIME_REGEX = "TsProcessor.default.time.regex";
+  public static final String TIME_REGEX = "TsProcessor.time.regex.";
 
   private Map<String, Pattern> datePatternMap;
   private Map<String, SimpleDateFormat> dateFormatMap;
@@ -155,14 +158,24 @@ public class TsProcessor extends Abstrac
 
     JobConf jobConf = Demux.jobConf;
     String datePattern = null;
+    Pattern pattern = null;
 
     if (jobConf != null) {
-      datePattern = jobConf.get("TsProcessor.default.time.regex", null);
-      datePattern = jobConf.get("TsProcessor.time.regex." + chunk.getDataType(),
-                               datePattern);
+      String timeRegexProperty = TIME_REGEX + chunk.getDataType();
+      datePattern = jobConf.get(DEFAULT_TIME_REGEX, null);
+      datePattern = jobConf.get(timeRegexProperty, datePattern);
+      if (datePattern != null) {
+        if (!RegexUtil.isRegex(datePattern, 1)) {
+          log.warn("Error parsing '" + DEFAULT_TIME_REGEX + "' or '"
+              + timeRegexProperty + "' properties as a regex: "
+              + RegexUtil.regexError(datePattern, 1)
+              + ". This date pattern will be skipped.");
+          return null;
+        }
+        pattern = Pattern.compile(datePattern);
+      }
     }
 
-    Pattern pattern = datePattern != null ? Pattern.compile(datePattern) : null;
     datePatternMap.put(dataType, pattern);
 
     return pattern;

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java
(original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java
Tue Nov 20 19:39:03 2012
@@ -22,6 +22,7 @@ package org.apache.hadoop.chukwa.inputto
 import java.io.IOException;
 import java.util.regex.*;
 import org.apache.hadoop.chukwa.*;
+import org.apache.hadoop.chukwa.util.RegexUtil;
 import org.apache.hadoop.mapred.*;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.*;
@@ -46,6 +47,9 @@ public class ChukwaInputFormat extends
 
     static Logger LOG = Logger.getLogger(ChukwaInputFormat.class);
 
+    public static final String DATATYPE_PROPERTY = "chukwa.inputfilter.datatype";
+    public static final String DATATYPE_PROPERTY_DEFAULT = ".*";
+
     private SequenceFileRecordReader<ChukwaArchiveKey, Chunk> sfrr;
     private long lineInFile = 0;
     private Chunk curChunk = null;
@@ -58,8 +62,14 @@ public class ChukwaInputFormat extends
     public ChukwaRecordReader(Configuration conf, FileSplit split)
         throws IOException {
       sfrr = new SequenceFileRecordReader<ChukwaArchiveKey, Chunk>(conf, split);
-      dtPattern = Pattern
-          .compile(conf.get("chukwa.inputfilter.datatype", ".*"));
+      String datatype = conf.get(DATATYPE_PROPERTY, DATATYPE_PROPERTY_DEFAULT);
+      if (!RegexUtil.isRegex(datatype)) {
+        LOG.warn("Error parsing '" + DATATYPE_PROPERTY
+            + "' property as a regex: " + RegexUtil.regexError(datatype)
+            + ". Using default instead: " + DATATYPE_PROPERTY_DEFAULT);
+        datatype = DATATYPE_PROPERTY_DEFAULT;
+      }
+      dtPattern = Pattern.compile(datatype);
     }
 
     @Override

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java Tue
Nov 20 19:39:03 2012
@@ -25,6 +25,7 @@ import java.util.*;
 import java.io.*;
 import org.apache.hadoop.chukwa.*;
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileUtil;
@@ -84,13 +85,14 @@ public class DumpChunks {
     Filter patterns = null;
     if(args[filterArg].toLowerCase().equals("all"))
       patterns = Filter.ALL;
-    else
+    else {
       try {
         patterns = new Filter(args[filterArg]);
-      } catch (PatternSyntaxException pse) {
+      } catch (CheckedPatternSyntaxException pse) {
         System.err.println("Error parsing \"tags\" regular expression: " + pse.getMessage());
-        System.exit(-1);
+        return;
       }
+    }
 
     System.err.println("Patterns:" + patterns);
     ArrayList<Path> filesToSearch = new ArrayList<Path>();

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java Tue Nov
20 19:39:03 2012
@@ -24,10 +24,14 @@ import java.util.regex.PatternSyntaxExce
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.chukwa.Chunk;
 import org.apache.hadoop.chukwa.extraction.engine.RecordUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
+import org.apache.log4j.Logger;
 
 
 public class Filter {
-  
+
+  static Logger log = Logger.getLogger(Filter.class);
+
   private static final String[] SEARCH_TARGS = 
   {"datatype", "name", "host", "cluster", "content"};
   static final String SEPARATOR="&";
@@ -56,6 +60,11 @@ public class Filter {
         return p.matcher(content).matches();
       } else if(targ.startsWith("tags.")) {
         String tagName = targ.substring("tags.".length());
+        if (!RegexUtil.isRegex(tagName)) {
+          log.warn("Error parsing 'tagName' as a regex: "
+              + RegexUtil.regexError(tagName));
+          return false;
+        }
         String tagVal = chunk.getTag(tagName);
         if(tagVal == null)
           return false;
@@ -73,7 +82,7 @@ public class Filter {
 
   List<SearchRule> compiledPatterns;
     
-  public Filter(String listOfPatterns) throws  PatternSyntaxException{
+  public Filter(String listOfPatterns) throws CheckedPatternSyntaxException {
     compiledPatterns = new ArrayList<SearchRule>();
     //FIXME: could escape these
     String[] patterns = listOfPatterns.split(SEPARATOR);
@@ -81,17 +90,22 @@ public class Filter {
       int equalsPos = p.indexOf('=');
       
       if(equalsPos < 0 || equalsPos > (p.length() -2)) {
-        throw new PatternSyntaxException(
+        throw new CheckedPatternSyntaxException(
             "pattern must be of form targ=pattern", p, -1);
       }
       
       String targ = p.substring(0, equalsPos);
       if(!targ.startsWith("tags.") && !ArrayUtils.contains(SEARCH_TARGS, targ)) {
-        throw new PatternSyntaxException(
+        throw new CheckedPatternSyntaxException(
             "pattern doesn't start with recognized search target", p, -1);
       }
       
-      Pattern pat = Pattern.compile(p.substring(equalsPos+1), Pattern.DOTALL);
+      String regex = p.substring(equalsPos+1);
+      if (!RegexUtil.isRegex(regex)) {
+          throw new CheckedPatternSyntaxException(RegexUtil.regexException(regex));
+      }
+
+      Pattern pat = Pattern.compile(regex, Pattern.DOTALL);
       compiledPatterns.add(new SearchRule(pat, targ));
     }
   }
@@ -119,7 +133,7 @@ public class Filter {
   }
   
   private static final class MatchAll extends Filter {
-    public MatchAll() {
+    public MatchAll() throws CheckedPatternSyntaxException {
       super("datatype=.*");
     }
     
@@ -131,6 +145,14 @@ public class Filter {
       return "ALL";
     }
   } 
-  public static final Filter ALL = new MatchAll();
+
+  public static final Filter ALL = newMatchAll();
+  private static Filter newMatchAll() {
+    try {
+      return new MatchAll();
+    } catch (CheckedPatternSyntaxException e) {
+      throw new RuntimeException("Illegal MatchAll regular expression.", e);
+    }
+  }
   
 }//end class

Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java
(original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java
Tue Nov 20 19:39:03 2012
@@ -165,4 +165,24 @@ public class TestTsProcessor extends Tes
     return key;
   }
 
+  public void testParseIllegalRegex() {
+    jobConf.set(TsProcessor.DEFAULT_TIME_REGEX, "(");
+
+    ChunkBuilder cb = new ChunkBuilder();
+    cb.addRecord("2012-10-25 00:18:44,818 some sample record data".getBytes());
+    Chunk chunk = cb.getChunk();
+
+    TsProcessor p = new TsProcessor();
+    p.reset(chunk);
+
+    ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output =
+            new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>();
+
+    p.process(null, chunk, output, Reporter.NULL);
+
+    assertEquals("Output data size not correct.", 1, output.data.size());
+    ChukwaRecordKey key = output.data.keySet().iterator().next();
+    ChukwaRecord record = output.data.get(key);
+    assertNull("Output should not be error.", record.getValue("cchunkData"));
+  }
 }
\ No newline at end of file

Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java
(original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java
Tue Nov 20 19:39:03 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.chukwa.inputto
 
 
 import java.io.IOException;
+import java.util.regex.PatternSyntaxException;
+
 import org.apache.hadoop.mapred.Reporter;
 import junit.framework.TestCase;
 import org.apache.hadoop.chukwa.*;
@@ -82,4 +84,25 @@ public class TestInputFormat extends Tes
     }
   }
 
+  public void testInputFormatIllegalRegex() {
+    try {
+      JobConf conf = new JobConf();
+      conf.set("chukwa.inputfilter.datatype", "(");
+      String TMP_DIR = System.getProperty("test.build.data", "/tmp");
+      Path filename = new Path("file:///" + TMP_DIR + "/tmpSeqFile");
+      long len = FileSystem.getLocal(conf).getFileStatus(filename).getLen();
+      InputSplit split = new FileSplit(filename, 0, len, (String[]) null);
+
+      ChukwaInputFormat in = new ChukwaInputFormat();
+      RecordReader<LongWritable, Text> r = in.getRecordReader(split, conf,
+          Reporter.NULL);
+
+    } catch (PatternSyntaxException e) {
+      e.printStackTrace();
+      fail("Illegal regular expression caused PatternSyntaxException: " + e);
+    } catch (IOException e) {
+      e.printStackTrace();
+      fail("IO exception " + e);
+    }
+  }
 }

Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java
(original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java
Tue Nov 20 19:39:03 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.chukwa.util;
 
 import junit.framework.TestCase;
 import java.util.*;
+import java.util.regex.PatternSyntaxException;
 import java.io.*;
 import org.apache.hadoop.chukwa.ChukwaArchiveKey;
 import org.apache.hadoop.chukwa.ChunkImpl;
@@ -86,4 +87,15 @@ public class TestDumpChunks extends Test
     
   }
 
+  public void testIllegalRegex() throws Exception {
+    String[] args = { "tags.command=(" };
+    Configuration conf = new Configuration();
+
+    try {
+      DumpChunks.dump(args, conf, System.out);
+    } catch (PatternSyntaxException e) {
+      e.printStackTrace();
+      fail("Illegal regular expression caused PatternSyntaxException: " + e);
+    }
+  }
 }

Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java (original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java Tue
Nov 20 19:39:03 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.chukwa.util;
 
 import junit.framework.TestCase;
 import org.apache.hadoop.chukwa.ChunkImpl;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
 
 public class TestFilter extends TestCase {
   
@@ -65,7 +66,13 @@ public class TestFilter extends TestCase
     ChunkImpl chunk1 = new ChunkImpl("Data", "aname", dat.length, dat, null);
     chunk1.setSource("asource");
     assertTrue(Filter.ALL.matches(chunk1));
-    Filter rule = new Filter("tags.foo=bar");
+    Filter rule = null;
+    try {
+      rule = new Filter("tags.foo=bar");
+    } catch (CheckedPatternSyntaxException e) {
+      e.printStackTrace();
+      fail("Regular expression error: " + e);
+    }
     
     assertFalse(rule.matches(chunk1));
     chunk1.addTag("foo=\"bar\"");
@@ -75,4 +82,12 @@ public class TestFilter extends TestCase
     assertTrue(Filter.ALL.matches(chunk1));
   }
   
+  public void testIllegalRegex() {
+    try {
+      new Filter("tags.foo=(");
+    } catch (CheckedPatternSyntaxException e) {
+      return;
+    }
+    fail("No CheckedPatternSyntaxException thrown for illegal regular expression.");
+  }
 }



Mime
View raw message