chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r806355 - in /hadoop/chukwa/trunk/src: java/org/apache/hadoop/chukwa/extraction/archive/ test/org/apache/hadoop/chukwa/datacollection/collector/ test/org/apache/hadoop/chukwa/datacollection/controller/
Date Thu, 20 Aug 2009 20:55:48 GMT
Author: asrabkin
Date: Thu Aug 20 20:55:48 2009
New Revision: 806355

URL: http://svn.apache.org/viewvc?rev=806355&view=rev
Log:
CHUKWA-373. Fix Backpressure test case

Modified:
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java?rev=806355&r1=806354&r2=806355&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
Thu Aug 20 20:55:48 2009
@@ -91,6 +91,9 @@
         fs.mkdirs(pMRInputDir);
       
       Path pOutputDir = new Path(archivesMROutputDir);
+      if(!fs.exists(pOutputDir))
+        fs.mkdirs(pOutputDir);
+      
       if(fs.listStatus(pOutputDir).length == 0)
         fs.delete(pOutputDir, true);
       Path archive = new Path(chukwaRootDir + "archive");

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java?rev=806355&r1=806354&r2=806355&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java
Thu Aug 20 20:55:48 2009
@@ -15,15 +15,24 @@
 public class TestBackpressure extends TestCase {
 
   int PORTNO = 9991;
-  int SLEEP_SEC = 20;
-  int WRITE_RATE = 300; //kb/sec
-  int SEND_RATE = 2500000; //bytes/sec
+  
+  /**
+   * NOTE THAT WRITE-RATE * POST SIZE MUST BE GREATER THAN TEST DURATION
+   * 
+   * Default max post size is 2 MB; need to process that several times during test.
+   */
+  int TEST_DURATION_SECS = 40;
+  int WRITE_RATE_KB = 200; //kb/sec
+  
+  
+  int SEND_RATE = 2500* 1000; //bytes/sec
+  int MIN_ACCEPTABLE_PERCENT = 60;
   
   public void testBackpressure() throws Exception {
     Configuration conf = new Configuration();
     conf.set("chukwaCollector.writerClass", NullWriter.class
         .getCanonicalName());
-    conf.set(NullWriter.RATE_OPT_NAME, ""+WRITE_RATE);//kb/sec
+    conf.set(NullWriter.RATE_OPT_NAME, ""+WRITE_RATE_KB);//kb/sec
     conf.setInt(HttpConnector.MIN_POST_INTERVAL_OPT, 100);
     conf.setInt("constAdaptor.sleepVariance", 1);
     conf.setInt("constAdaptor.minSleep", 50);
@@ -41,13 +50,13 @@
     Thread.sleep(1000);
     agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName()
+ 
         " testData "+ SEND_RATE + " 0");
-    Thread.sleep(SLEEP_SEC * 1000);
+    Thread.sleep(TEST_DURATION_SECS * 1000);
 
     String stat = agent.getAdaptorList().get("constSend");
-    long bytesPerSec = Long.valueOf(stat.split(" ")[3]) / SLEEP_SEC / 1000;
-    System.out.println("data rate was " + bytesPerSec + " kb /second");
-    assertTrue(bytesPerSec < WRITE_RATE);
-    assertTrue(bytesPerSec > 3* WRITE_RATE / 4);//an assumption, but should hold true
+    long kbytesPerSec = Long.valueOf(stat.split(" ")[3]) / TEST_DURATION_SECS / 1000;
+    System.out.println("data rate was " + kbytesPerSec + " kb /second");
+    assertTrue(kbytesPerSec < WRITE_RATE_KB); //write rate should throttle sends
+    assertTrue(kbytesPerSec > MIN_ACCEPTABLE_PERCENT* WRITE_RATE_KB / 100);//an assumption,
but should hold true
     agent.shutdown();
   }
 }

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java?rev=806355&r1=806354&r2=806355&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java
Thu Aug 20 20:55:48 2009
@@ -18,8 +18,9 @@
 package org.apache.hadoop.chukwa.datacollection.controller;
 
 
-import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.hadoop.chukwa.datacollection.connector.ChunkCatcherConnector;
 import org.apache.hadoop.chukwa.datacollection.connector.Connector;
 import org.apache.hadoop.chukwa.datacollection.connector.http.HttpConnector;
 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
@@ -28,30 +29,24 @@
 import junit.framework.TestCase;
 
 public class TestAgentClient extends TestCase {
-  ChukwaConfiguration config;
+  Configuration config;
   ChukwaAgent agent;
   ChukwaAgentController c;
-  Connector httpConnector;
+  Connector connector;
 
   // consoleConnector = new ConsoleOutConnector(agent);
 
   protected void setUp() throws ChukwaAgent.AlreadyRunningException {
-    config = new ChukwaConfiguration();
-    agent = new ChukwaAgent();
+    config = new Configuration();
+    agent = new ChukwaAgent(config);
     c = new ChukwaAgentController();
-    httpConnector = new HttpConnector(agent); // use default source for list of
-                                              // collectors (i.e.
-                                              // conf/connectors)
-
-    httpConnector.start();
-
-    // assertTrue(Integer.parseInt(config.get("chukwaAgent.control.port")) ==
-    // agent.getControlSock().getPortNumber());
+    connector = new ChunkCatcherConnector();
+    connector.start();
   }
 
   protected void tearDown() {
     System.out.println("in tearDown()");
-    ((HttpConnector) httpConnector).shutdown();
+    connector.shutdown();
   }
 
   public void testAddFile() {



Mime
View raw message