chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r783201 - in /hadoop/chukwa/trunk: ./ src/java/org/apache/hadoop/chukwa/datacollection/ src/java/org/apache/hadoop/chukwa/datacollection/connector/http/ src/java/org/apache/hadoop/chukwa/datacollection/sender/ src/test/org/apache/hadoop/chu...
Date Wed, 10 Jun 2009 04:33:09 GMT
Author: asrabkin
Date: Wed Jun 10 04:33:08 2009
New Revision: 783201

URL: http://svn.apache.org/viewvc?rev=783201&view=rev
Log:
CHUKWA-273. Agents use correct collector port number.

Modified:
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/sender/TestRetryListOfCollectors.java

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=783201&r1=783200&r2=783201&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Wed Jun 10 04:33:08 2009
@@ -36,6 +36,8 @@
 
   BUG FIXES
 
+    CHUKWA-273. Agents correctly use configured collector port numbers. (asrabkin)
+
     CHUKWA-216. postProcess.sh should use HADOOP_CONF_DIR to find hadoop-site.xml. (Jerome
Boulon via asrabkin)
 
     CHUKWA-261. Added X axis label for non Time Series graphs. (Eric Yang)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java?rev=783201&r1=783200&r2=783201&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
Wed Jun 10 04:33:08 2009
@@ -23,6 +23,7 @@
 import java.io.IOException;
 import java.util.Iterator;
 
+import org.apache.hadoop.conf.*;
 import org.apache.hadoop.chukwa.datacollection.agent.MemLimitQueue;
 import org.apache.hadoop.chukwa.datacollection.sender.RetryListOfCollectors;
 import org.apache.log4j.Logger;
@@ -66,7 +67,7 @@
    * @return empty list if file does not exist
    * @throws IOException on other error
    */
-  public Iterator<String> getCollectorURLs() throws IOException {
+  public Iterator<String> getCollectorURLs(Configuration conf) throws IOException {
     String chukwaHome = System.getenv("CHUKWA_HOME");
     if (chukwaHome == null) {
       chukwaHome = ".";
@@ -88,7 +89,7 @@
         + COLLECTORS_FILENAME);
     File collectors = new File(chukwaConf + File.separator + "collectors");
     try {
-      return new RetryListOfCollectors(collectors, 1000 * 15);// time is ms
+      return new RetryListOfCollectors(collectors, 1000 * 15, conf);// time is ms
                                                               // between tries
     } catch (java.io.IOException e) {
       log.error("failed to read collectors file: ", e);

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java?rev=783201&r1=783200&r2=783201&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
Wed Jun 10 04:33:08 2009
@@ -107,7 +107,7 @@
 
     // build a list of our destinations from collectors
     try {
-      destinations = DataFactory.getInstance().getCollectorURLs();
+      destinations = DataFactory.getInstance().getCollectorURLs(agent.getConfiguration());
     } catch (IOException e) {
       log.error("Failed to retreive list of collectors from "
           + "conf/collectors file", e);
@@ -190,7 +190,7 @@
 
     // build a list of our destinations from collectors
     try {
-      destinations = DataFactory.getInstance().getCollectorURLs();
+      destinations = DataFactory.getInstance().getCollectorURLs(agent.getConfiguration());
     } catch (IOException e) {
       log.error(
           "Failed to retreive list of collectors from conf/collectors file", e);

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java?rev=783201&r1=783200&r2=783201&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java
Wed Jun 10 04:33:08 2009
@@ -43,12 +43,12 @@
   private String portNo;
   Configuration conf;
 
-  public RetryListOfCollectors(File collectorFile, int maxRetryRateMs)
+  public RetryListOfCollectors(File collectorFile, int maxRetryRateMs, Configuration conf)
       throws IOException {
     this.maxRetryRateMs = maxRetryRateMs;
     lastLookAtFirstNode = 0;
     collectors = new ArrayList<String>();
-    conf = new Configuration();
+    this.conf = conf;
     portNo = conf.get("chukwaCollector.http.port", "8080");
 
     try {
@@ -57,18 +57,17 @@
       while ((line = br.readLine()) != null) {
         if (!line.contains("://")) {
           // no protocol, assume http
-          if (line.matches(":\\d+")) {
+          if (line.matches(".*:\\d+")) {
             collectors.add("http://" + line);
           } else {
-            collectors.add("http://" + line + ":" + portNo + "/");
+            collectors.add("http://" + line + ":" + portNo);
           }
         } else {
-          if (line.matches(":\\d+")) {
+          if (line.matches(".*:\\d+")) {
             collectors.add(line);
           } else {
-            collectors.add(line + ":" + portNo + "/");
+            collectors.add(line + ":" + portNo);
           }
-          collectors.add(line);
         }
       }
       br.close();
@@ -83,6 +82,12 @@
     shuffleList();
   }
 
+  /**
+   * This is only used for debugging. Possibly it should sanitize urls the same way the other
+   * constructor does.
+   * @param collectors
+   * @param maxRetryRateMs
+   */
   public RetryListOfCollectors(final List<String> collectors, int maxRetryRateMs) {
     this.maxRetryRateMs = maxRetryRateMs;
     lastLookAtFirstNode = 0;

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/sender/TestRetryListOfCollectors.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/sender/TestRetryListOfCollectors.java?rev=783201&r1=783200&r2=783201&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/sender/TestRetryListOfCollectors.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/sender/TestRetryListOfCollectors.java
Wed Jun 10 04:33:08 2009
@@ -19,8 +19,10 @@
 
 
 import junit.framework.TestCase;
+import java.io.*;
 import java.util.*;
 import org.apache.hadoop.chukwa.datacollection.sender.RetryListOfCollectors;
+import org.apache.hadoop.conf.Configuration;
 
 public class TestRetryListOfCollectors extends TestCase {
 
@@ -45,7 +47,54 @@
       System.out.println("saw unexpected collector " + s);
       fail();
     }
+  }
+  
+  public void testCollectorsFile() {
+    
+    try {
+    File tmpOutput = new File(System.getProperty("test.build.data", "/tmp"),
+        "collectors_test");
+    PrintWriter out = new PrintWriter(new FileOutputStream(tmpOutput));
+    
+    HashSet<String> validHosts = new HashSet<String>();
+    validHosts.add("http://host1:5052");
+    validHosts.add("http://host2:5050");
+    validHosts.add("http://host3:5052");
+    validHosts.add("http://host4:5050");
+    validHosts.add("http://host5:5052");
+    validHosts.add("http://host6:5052");
+    
+    out.println("host1");
+    out.println("host2:5050");
+    out.println("http://host3");
+    out.println("http://host4:5050");
+    out.println("http://host5:5052");
+    out.println("host6:5052");
+    out.close();
+    
+    Configuration conf = new Configuration();
+    conf.setInt("chukwaCollector.http.port", 5052);
+    RetryListOfCollectors rloc = new RetryListOfCollectors(tmpOutput, 2000, conf);
+    for (int i = 0; i < validHosts.size(); ++i) {
+      assertTrue(rloc.hasNext());
+      String s = rloc.next();
+      assertTrue(s != null);
+      
+      System.out.println("host: " + s);
+      assertTrue(validHosts.contains(s));
+    }
+    
+    if (rloc.hasNext()) {
+      String s = rloc.next();
+      System.out.println("saw unexpected collector " + s);
+      fail();
+    }
 
+    } catch(IOException e) {
+      e.printStackTrace();
+      fail();
+    }
+    
   }
 
 }



Mime
View raw message