hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r732655 - in /hadoop/core/trunk: ./ src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/ src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ src/contrib/chukwa/src/test/org/apache/ha...
Date Thu, 08 Jan 2009 09:14:46 GMT
Author: cdouglas
Date: Thu Jan  8 01:14:46 2009
New Revision: 732655

URL: http://svn.apache.org/viewvc?rev=732655&view=rev
Log:
HADOOP-4947. Make Chukwa command parsing more forgiving of whitespace. Contributed by Ari
Rabkin.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jan  8 01:14:46 2009
@@ -330,6 +330,9 @@
     HADOOP-4980. Improve code layout of capacity scheduler to make it 
     easier to fix some blocker bugs. (Vivek Ratan via yhemanth)
 
+    HADOOP-4916. Make user/location of Chukwa installation configurable by an
+    external properties file. (Eric Yang via cdouglas)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits
@@ -555,12 +558,12 @@
     HADOOP-4884. Make tool tip date format match standard HICC format. (Eric
     Yang via cdouglas)
 
-    HADOOP-4916. Make user/location of Chukwa installation configurable by an
-    external properties file. (Eric Yang via cdouglas)
-
     HADOOP-4925. Make Chukwa sender properties configurable. (Ari Rabkin via
     cdouglas)
 
+    HADOOP-4947. Make Chukwa command parsing more forgiving of whitespace. (Ari
+    Rabkin via cdouglas)
+
 Release 0.19.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
Thu Jan  8 01:14:46 2009
@@ -29,6 +29,8 @@
 
 import java.io.*;
 import java.util.Timer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * An adaptor that repeatedly tails a specified file, sending the new bytes.
@@ -81,11 +83,12 @@
 	    this.type = type;
 	    this.dest = dest;
 	    this.attempts = 0;
-			  
-	    String[] words = params.split(" ");
-	    if(words.length > 1) {
-	        offsetOfFirstByte = Long.parseLong(words[0]);
-	        toWatch = new File(params.substring(words[0].length() + 1));
+			
+	    Pattern cmd = Pattern.compile("(\\d+)\\s+(.+)");
+	    Matcher m = cmd.matcher(params);
+	    if(m.matches()) {
+	        offsetOfFirstByte = Long.parseLong(m.group(1));
+	        toWatch = new File(m.group(2));
 	    } else {
 	        toWatch = new File(params);
 	    }

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
Thu Jan  8 01:14:46 2009
@@ -80,7 +80,7 @@
      * @throws IOException
      */
     public void processCommand(String cmd, PrintStream out) throws IOException  {
-      String[] words = cmd.split(" ");
+      String[] words = cmd.split("\\s+");
       if (log.isDebugEnabled())
   		{ log.debug("command from " + connection.getRemoteSocketAddress() + ":"+ cmd);}
       

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Thu Jan  8 01:14:46 2009
@@ -30,6 +30,8 @@
 
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.io.*;
 
 /**
@@ -231,59 +233,45 @@
     }
 
   }
-
+  // words should contain (space delimited):
+  // 0) command ("add")
+  // 1) AdaptorClassname
+  // 2) dataType (e.g. "hadoop_log")
+  // 3) params <optional>
+  // (e.g. for files, this is filename,
+  // but can be arbitrarily many space
+  // delimited agent specific params )
+  // 4) offset
+  Pattern addCmdPattern = Pattern.compile("add\\s+(\\S+)\\s+(\\S+)\\s+(.*\\S)?\\s*(\\d+)\\s*");
   // FIXME: should handle bad lines here
   public long processCommand(String cmd)
   {
-    String[] words = cmd.split(" ");
-    if (words[0].equalsIgnoreCase("add"))
-    {
-      // words should contain (space delimited):
-      // 0) command ("add")
-      // 1) AdaptorClassname
-      // 2) dataType (e.g. "hadoop_log")
-      // 3) params <optional>
-      // (e.g. for files, this is filename,
-      // but can be arbitrarily many space
-      // delimited agent specific params )
-      // 4) offset
-
-      long offset;
-      try
-      {
-        offset = Long.parseLong(words[words.length - 1]);
-      } catch (NumberFormatException e)
-      {
+    Matcher m = addCmdPattern.matcher(cmd);
+    if (m.matches()) {
+      long offset;  //check for obvious errors first
+      try {
+        offset = Long.parseLong(m.group(4));
+      } catch (NumberFormatException e)  {
         log.warn("malformed line " + cmd);
         return -1L;
       }
-      String adaptorName = words[1];
+      
+      String adaptorName = m.group(1);
+      String dataType = m.group(2);
+      String params = m.group(3);
+      if(params == null)
+        params = "";
 
       Adaptor adaptor = AdaptorFactory.createAdaptor(adaptorName);
-      if (adaptor == null)
-      {
+      if (adaptor == null)  {
         log.warn("Error creating adaptor from adaptor name " + adaptorName);
         return -1L;
       }
 
-      String dataType = words[2];
-      String streamName = "";
-      String params = "";
-      if (words.length > 4)
-      { // no argument
-        int begParams = adaptorName.length() + dataType.length() + 6;
-        // length("ADD x type ") = length(x) + 5, i.e. count letters & spaces
-        params = cmd.substring(begParams, cmd.length()
-            - words[words.length - 1].length() - 1);
-        streamName = params.substring(params.indexOf(" ") + 1, params.length());
-      }
       long adaptorID;
-      synchronized (adaptorsByNumber)
-      {
-        for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet())
-        {
-          if (streamName.intern() == a.getValue().getStreamName().intern())
-          {
+      synchronized (adaptorsByNumber) {
+        for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet()) {
+          if (params.intern() == a.getValue().getStreamName().intern()) {
             log.warn(params + " already exist, skipping.");
             return -1;
           }

Modified: hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
Thu Jan  8 01:14:46 2009
@@ -65,7 +65,7 @@
         ArrayList<Long> runningAdaptors = new ArrayList<Long>();
        
         for(int i = 1; i < 7; ++i) {
-          long l = agent.processCommand("add org.apache.hadoop.chukwa.util.ConstRateAdaptor
raw"+i+ " 2000"+i+" 0");
+          long l = agent.processCommand("add  org.apache.hadoop.chukwa.util.ConstRateAdaptor
 raw"+i+ " 2000"+i+" 0");
           assertTrue(l != -1);
           runningAdaptors.add(l);
         }



Mime
View raw message