chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r793128 - in /hadoop/chukwa/trunk: ./ src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/ src/java/org/apache/hadoop/chukwa/datacollection/agent/ src/java/org/apache...
Date Fri, 10 Jul 2009 22:36:03 GMT
Author: asrabkin
Date: Fri Jul 10 22:36:02 2009
New Revision: 793128

URL: http://svn.apache.org/viewvc?rev=793128&view=rev
Log:
CHUKWA-284. Checkpoint includes last byte Acked, not sent

Modified:
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Fri Jul 10 22:36:02 2009
@@ -34,6 +34,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-284.  Checkpoint includes last byte Acked, not sent. (asrabkin)
+
     CHUKWA-25.  Archiver can group by cluster name. (asrabkin)
     
     CHUKWA-308. Added capability to start hicc without tomcat. (Eric Yang)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
Fri Jul 10 22:36:02 2009
@@ -51,8 +51,9 @@
       ChunkReceiver dest, AdaptorManager c) throws AdaptorException;
 
   /**
-   * Return the adaptor's state Should not include class name, datatype or byte
-   * offset, which are written by caller.
+   * Return the adaptor's state Should not include class name or byte
+   * offset, which are written by caller. The datatype should, however,
+   * be written by this method.
    * 
    * @return the adaptor state as a string
    * @throws AdaptorException

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/DirTailingAdaptor.java
Fri Jul 10 22:36:02 2009
@@ -92,7 +92,7 @@
   @Override
   public String getCurrentStatus() throws AdaptorException {
     try {
-      return type + " " + baseDir.getCanonicalPath()+ " " + adaptorName+ " " + lastSweepStartTime;
+      return type + " " + baseDir.getCanonicalPath()+ " " + adaptorName;
     } catch(IOException e) {
       throw new AdaptorException(e);
     }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
Fri Jul 10 22:36:02 2009
@@ -123,7 +123,7 @@
 
   @Override
   public String getCurrentStatus() throws AdaptorException {
-    return type + " " + period + " " + cmd + " " + sendOffset;
+    return type + " " + period + " " + cmd;
   }
 
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
Fri Jul 10 22:36:02 2009
@@ -313,8 +313,7 @@
 
   @Override
   public String getCurrentStatus() throws AdaptorException {
-    return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath()
-        + " " + fileReadOffset;
+    return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath();
   }
 
 }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
Fri Jul 10 22:36:02 2009
@@ -77,7 +77,6 @@
   public void start(long bytes) {
     this.attempts = 0;
 
-   
     log.info("started file tailer on file " + toWatch
         + " with first byte at offset " + offsetOfFirstByte);
 
@@ -177,8 +176,7 @@
    * @see org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor#getCurrentStatus()
    */
   public String getCurrentStatus() {
-    return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath()
-        + " " + fileReadOffset;
+    return type.trim() + " " + offsetOfFirstByte + " " + toWatch.getPath();
     // can make this more efficient using a StringBuilder
   }
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Fri Jul 10 22:36:02 2009
@@ -445,7 +445,7 @@
           new OutputStreamWriter(fos)));
 
       for (Map.Entry<String, String> stat : getAdaptorList().entrySet()) {
-          out.println("ADD "+ stat.getKey()+ " = " + stat.getValue());
+        out.println("ADD "+ stat.getKey()+ " = " + stat.getValue());
       }
 
       out.close();
@@ -490,7 +490,8 @@
 
   
   private String formatAdaptorStatus(Adaptor a) throws AdaptorException {
-    return a.getClass().getCanonicalName() + " " + a.getCurrentStatus();
+    return a.getClass().getCanonicalName() + " " + a.getCurrentStatus() + 
+   " " + adaptorPositions.get(a).offset;
   }
   
 /**

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
Fri Jul 10 22:36:02 2009
@@ -52,7 +52,7 @@
     String adaptorName;
     String type;
     String params;
-    long offset;
+    private long offset;
     long numRetries;
     long retryInterval;
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java Fri Jul
10 22:36:02 2009
@@ -40,7 +40,7 @@
   private volatile boolean stopping = false;
 
   public String getCurrentStatus() throws AdaptorException {
-    return type.trim() + " " + bytesPerSec + " " + offset;
+    return type.trim() + " " + bytesPerSec;
   }
 
   public void start(String adaptorID, String type, 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java Fri Jul
10 22:36:02 2009
@@ -37,7 +37,7 @@
   private String adaptorID;
 
   public String getCurrentStatus() throws AdaptorException {
-    return "";
+    return type;
   }
 
   public void start(String adaptorID, String type, long offset,

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
Fri Jul 10 22:36:02 2009
@@ -45,7 +45,10 @@
     agent.shutdown();
   }
   
-  
+  /*
+   * Buzz in a loop, starting ls every 100 ms.
+   * Length of loop controlled by sleep statement near bottom of function
+   */
   public void testForLeaks()  throws ChukwaAgent.AlreadyRunningException, InterruptedException
{
     Configuration conf = new Configuration();
 //    conf.set("chukwaAgent.control.port", "0");
@@ -57,7 +60,7 @@
     assertEquals(0, agent.adaptorCount());
     String lsID = agent.processAddCommand(
       "add exec= org.apache.hadoop.chukwa.datacollection.adaptor.ExecAdaptor Listing 100
/bin/sleep 1 0");
-    Thread.sleep( 60*1000);
+    Thread.sleep( 25*1000); //RAISE THIS to test longer
     System.out.println("stopped ok"); 
   }
 

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java?rev=793128&r1=793127&r2=793128&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestFileAdaptor.java
Fri Jul 10 22:36:02 2009
@@ -84,7 +84,7 @@
   
   public void testRepeatedly() throws IOException,
   ChukwaAgent.AlreadyRunningException, InterruptedException {
-    int tests = 300; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks
+    int tests = 100; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks
 
     ChukwaAgent agent = new ChukwaAgent(conf);
     for(int i=0; i < tests; ++i) {



Mime
View raw message