chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r788236 - in /hadoop/chukwa/branches/chukwa-0.2: ./ src/java/org/apache/hadoop/chukwa/datacollection/agent/ src/java/org/apache/hadoop/chukwa/datacollection/controller/ src/java/org/apache/hadoop/chukwa/inputtools/log4j/ src/java/org/apache...
Date Thu, 25 Jun 2009 01:30:40 GMT
Author: eyang
Date: Thu Jun 25 01:30:39 2009
New Revision: 788236

URL: http://svn.apache.org/viewvc?rev=788236&view=rev
Log:
CHUKWA-330. Modified AgentController to add adaptor by name for uniquely identify stream.
(Ari Rabkin via Eric Yang)

Added:
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/util/AdaptorNamingUtils.java
Modified:
    hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java

Modified: hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt?rev=788236&r1=788235&r2=788236&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt (original)
+++ hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt Thu Jun 25 01:30:39 2009
@@ -72,6 +72,8 @@
 
   BUG FIXES
 
+    CHUKWA-330. Modified AgentController to add adaptor by name for uniquely identify stream.
(Ari Rabkin via Eric Yang)
+
     CHUKWA-37. Remove stale and unused bin scripts. (Ari Rabkin)
 
     CHUKWA-331. Fixed regular expression for down sampling base on record type. (Cheng Zhang
via Eric Yang)

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=788236&r1=788235&r2=788236&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Thu Jun 25 01:30:39 2009
@@ -44,6 +44,7 @@
 import org.apache.hadoop.chukwa.datacollection.connector.Connector;
 import org.apache.hadoop.chukwa.datacollection.connector.http.HttpConnector;
 import org.apache.hadoop.chukwa.datacollection.test.ConsoleOutConnector;
+import org.apache.hadoop.chukwa.util.AdaptorNamingUtils;
 import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -316,8 +317,14 @@
       if (params == null)
         params = "";
 
-      if(adaptorID == null)
-        adaptorID = synthesizeAdaptorID(adaptorClassName, dataType, params);
+      if(adaptorID == null) { //user didn't specify, so synthesize
+        try {
+         adaptorID = AdaptorNamingUtils.synthesizeAdaptorID(adaptorClassName, dataType, params);
+        } catch(NoSuchAlgorithmException e) {
+          log.fatal("MD5 apparently doesn't work on your machine; bailing", e);
+          shutdown(true);
+        }
+      }
       
       Adaptor adaptor = AdaptorFactory.createAdaptor(adaptorClassName);
       if (adaptor == null) {
@@ -358,29 +365,7 @@
     return null;
   }
 
-  private String synthesizeAdaptorID(String adaptorClassName, String dataType,
-      String params) {
-    MessageDigest md;
-    try {
-      md = MessageDigest.getInstance("MD5");
 
-      md.update(adaptorClassName.getBytes());
-      md.update(dataType.getBytes());
-      md.update(params.getBytes());
-      StringBuilder sb = new StringBuilder();
-      byte[] bytes = md.digest();
-      for(int i=0; i < bytes.length; ++i) {
-        if( (bytes[i] & 0xF0) == 0)
-          sb.append('0');
-        sb.append( Integer.toHexString(0xFF & bytes[i]) );
-      }
-      return sb.toString();
-    } catch (NoSuchAlgorithmException e) {
-      log.fatal("MD5 apparently doesn't work on your machine; bailing", e);
-      shutdown(true);//abort agent
-    }
-    return null;
-  }
 
   /**
    * Tries to restore from a checkpoint file in checkpointDir. There should

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java?rev=788236&r1=788235&r2=788236&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
Thu Jun 25 01:30:39 2009
@@ -71,7 +71,7 @@
       try {
         log.info("Trying to resend the add command [" + adaptorName + "]["
             + offset + "][" + params + "] [" + numRetries + "]");
-        add(adaptorName, type, params, offset, numRetries, retryInterval);
+        addByName(null, adaptorName, type, params, offset, numRetries, retryInterval);
       } catch (Exception e) {
         log.warn("Exception in AddAdaptorTask.run", e);
         e.printStackTrace();
@@ -129,7 +129,10 @@
       }
       PrintWriter bw = new PrintWriter(new OutputStreamWriter(s
           .getOutputStream()));
-      bw.println("ADD " + className + " " + appType + " " + params + " " + offset);
+      if(id != null)
+        bw.println("ADD " + id + " = " + className + " " + appType + " " + params + " " +
offset);
+      else
+        bw.println("ADD " + className + " " + appType + " " + params + " " + offset);
       bw.flush();
       BufferedReader br = new BufferedReader(new InputStreamReader(s
           .getInputStream()));
@@ -231,7 +234,7 @@
    * @return the id number of the adaptor, generated by the agent
    */
   public String add(String adaptorName, String type, String params, long offset) {
-    return add(adaptorName, type, params, offset, 20, 15 * 1000);// retry for
+    return addByName(null, adaptorName, type, params, offset, 20, 15 * 1000);// retry for
                                                                  // five
                                                                  // minutes,
                                                                  // every
@@ -246,11 +249,11 @@
    * 
    * @return the id number of the adaptor, generated by the agent
    */
-  public String add(String adaptorName, String type, String params, long offset,
+  public String addByName(String adaptorID, String adaptorName, String type, String params,
long offset,
       long numRetries, long retryInterval) {
     ChukwaAgentController.Adaptor adaptor = new ChukwaAgentController.Adaptor(
         adaptorName, type, params, offset);
-    String adaptorID = null;
+    adaptor.id = adaptorID;
     if (numRetries >= 0) {
       try {
         adaptorID = adaptor.register();
@@ -417,7 +420,7 @@
       }
     }
     if (!isDuplicate) {
-      return add(DEFAULT_FILE_TAILER, appType, 0L + " " + filename, 0L,
+      return addByName(null, DEFAULT_FILE_TAILER, appType, 0L + " " + filename, 0L,
           numRetries, retryInterval);
     } else {
       System.out.println("An adaptor for filename \"" + filename

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java?rev=788236&r1=788235&r2=788236&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Thu Jun 25 01:30:39 2009
@@ -22,8 +22,10 @@
 import java.util.TimeZone;
 import java.util.regex.Pattern;
 
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
 import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
+import org.apache.hadoop.chukwa.util.AdaptorNamingUtils;
 import org.apache.hadoop.chukwa.util.RecordConstants;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Layout;
@@ -541,8 +543,10 @@
               numRetries = 48;
             }
 
-
-            String adaptorID = chukwaClient.add(ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
+            String name = AdaptorNamingUtils.synthesizeAdaptorID
+              (ChukwaAgentController.CharFileTailUTF8NewLineEscaped, recordType, log4jFileName);
+            
+            String adaptorID = chukwaClient.addByName(name, ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
                 recordType,currentLength + " " + log4jFileName, currentLength,
                 numRetries, retryInterval);
 

Added: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/util/AdaptorNamingUtils.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/util/AdaptorNamingUtils.java?rev=788236&view=auto
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/util/AdaptorNamingUtils.java
(added)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/util/AdaptorNamingUtils.java
Thu Jun 25 01:30:39 2009
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.util;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class AdaptorNamingUtils {
+  
+  public static String synthesizeAdaptorID(String adaptorClassName, String dataType,
+      String params) throws NoSuchAlgorithmException {
+    MessageDigest md;
+   md = MessageDigest.getInstance("MD5");
+
+    md.update(adaptorClassName.getBytes());
+    md.update(dataType.getBytes());
+    md.update(params.getBytes());
+    StringBuilder sb = new StringBuilder();
+    byte[] bytes = md.digest();
+    for(int i=0; i < bytes.length; ++i) {
+      if( (bytes[i] & 0xF0) == 0)
+        sb.append('0');
+      sb.append( Integer.toHexString(0xFF & bytes[i]) );
+    }
+    return sb.toString();
+   
+  }
+
+}



Mime
View raw message