hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r747979 - in /hadoop/core/trunk/src/contrib/chukwa/src: java/org/apache/hadoop/chukwa/datacollection/writer/ test/org/apache/hadoop/chukwa/datacollection/collector/
Date Thu, 26 Feb 2009 00:46:46 GMT
Author: asrabkin
Date: Thu Feb 26 00:46:46 2009
New Revision: 747979

URL: http://svn.apache.org/viewvc?rev=747979&view=rev
Log:
HADOOP-5033  Simplify Chukwa writer API.
Contributed by Ari Rabkin (asrabkin) 


Modified:
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
Thu Feb 26 00:46:46 2009
@@ -26,7 +26,6 @@
 public interface ChukwaWriter
 {
 	public void init(Configuration c) throws WriterException;
-	public void add(Chunk data) throws WriterException;
 	public void add(List<Chunk> chunks) throws WriterException;
 	public void close() throws WriterException;;
 

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/Dedup.java
Thu Feb 26 00:46:46 2009
@@ -19,6 +19,7 @@
 
 import java.util.ArrayDeque;
 import java.util.HashSet;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Queue;
 
@@ -101,16 +102,14 @@
   }
 
   @Override
-  public void add(Chunk data) throws WriterException {
-    if(! cache.addAndCheck(new DedupKey(data.getStreamName(), data.getSeqID())))
-        next.add(data);
-  }
-
-  @Override
   public void add(List<Chunk> chunks) throws WriterException {
+    ArrayList<Chunk> passedThrough = new ArrayList<Chunk>();
     for(Chunk c: chunks)
-      add(c);
-
+      if(! cache.addAndCheck(new DedupKey(c.getStreamName(), c.getSeqID())))
+        passedThrough.add(c);
+    
+    if(!passedThrough.isEmpty())
+      next.add(passedThrough);
   }
 
   @Override

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
Thu Feb 26 00:46:46 2009
@@ -31,12 +31,6 @@
   ChukwaWriter writer; //head of pipeline
 
   @Override
-  public void add(Chunk data) throws WriterException {
-    writer.add(data);
-
-  }
-
-  @Override
   public void add(List<Chunk> chunks) throws WriterException {
     writer.add(chunks);
   }

Modified: hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java?rev=747979&r1=747978&r2=747979&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/collector/CaptureWriter.java
Thu Feb 26 00:46:46 2009
@@ -29,17 +29,12 @@
   static ArrayList<Chunk> outputs = new ArrayList<Chunk>();
 
   @Override
-  public void add(Chunk data) throws WriterException {
-    synchronized(outputs) {
-      outputs.add(data);
-    }
-    
-  }
-
-  @Override
   public void add(List<Chunk> chunks) throws WriterException {
+
+    synchronized(outputs) {
      for(Chunk c: chunks)
-       add(c);
+       outputs.add(c);
+    }
   }
 
   @Override



Mime
View raw message