chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r765736 - in /hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa: Chunk.java ChunkImpl.java datacollection/DataFactory.java datacollection/adaptor/FileAdaptor.java datacollection/agent/ChukwaAgent.java
Date Thu, 16 Apr 2009 20:03:25 GMT
Author: eyang
Date: Thu Apr 16 20:03:24 2009
New Revision: 765736

URL: http://svn.apache.org/viewvc?rev=765736&view=rev
Log:
CHUKWA-119. Removed dependency of ChukwaAgent from ChunkImpl for preventing multiple MetricsContext
to be initialized in the same VM. (Jerome Boulon via Eric Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/Chunk.java
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/Chunk.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/Chunk.java?rev=765736&r1=765735&r2=765736&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/Chunk.java (original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/Chunk.java Thu Apr
16 20:03:24 2009
@@ -105,16 +105,17 @@
   public int getSerializedSizeEstimate();
 
   /**
-   * @return name of cluster that this chunk comes from.
+   * @return tags.
    * 
    */
   public String getTags();
 
   /**
-   * Set the name of the cluster that this chunk comes from.
+   * Add tag.
    * 
    */
-  public void setTags(String tags);
+  public void addTag(String tag);
 
+  
   public void write(DataOutput data) throws IOException;
 }

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/ChunkImpl.java?rev=765736&r1=765735&r2=765736&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/ChunkImpl.java (original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/ChunkImpl.java Thu
Apr 16 20:03:24 2009
@@ -24,12 +24,14 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+
+import org.apache.hadoop.chukwa.datacollection.DataFactory;
 import org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor;
-import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
 
 public class ChunkImpl implements org.apache.hadoop.io.Writable, Chunk {
   public static int PROTOCOL_VERSION = 1;
 
+  protected DataFactory dataFactory = DataFactory.getInstance();
   private String source = "";
   private String application = "";
   private String dataType = "";
@@ -43,7 +45,6 @@
   long seqID;
 
   ChunkImpl() {
-    this.tags = ChukwaAgent.getTags();
   }
 
   public static ChunkImpl getBlankChunk() {
@@ -54,7 +55,7 @@
                    Adaptor source) {
     this.seqID = seq;
     this.source = localHostAddr;
-    this.tags = ChukwaAgent.getTags();
+    this.tags = dataFactory.getDefaultTags();
     this.application = streamName;
     this.dataType = dataType;
     this.data = data;
@@ -161,8 +162,8 @@
   }
 
   @Override
-  public void setTags(String tags) {
-    this.tags = tags;
+  public void addTag(String tags) {
+    this.tags += " "+ tags;
   }
 
   /**

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java?rev=765736&r1=765735&r2=765736&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
Thu Apr 16 20:03:24 2009
@@ -22,8 +22,9 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.Iterator;
+
+import org.apache.hadoop.chukwa.datacollection.agent.MemLimitQueue;
 import org.apache.hadoop.chukwa.datacollection.sender.RetryListOfCollectors;
-import org.apache.hadoop.chukwa.datacollection.agent.*;
 import org.apache.log4j.Logger;
 
 public class DataFactory {
@@ -31,8 +32,10 @@
   static final int QUEUE_SIZE_KB = 10 * 1024;
   static final String COLLECTORS_FILENAME = "collectors";
   private static DataFactory dataFactory = null;
-  private ChunkQueue chunkQueue = new MemLimitQueue(QUEUE_SIZE_KB * 1024);
+  private ChunkQueue chunkQueue = null;
 
+  private String defaultTags = "";
+  
   static {
     dataFactory = new DataFactory();
   }
@@ -44,10 +47,21 @@
     return dataFactory;
   }
 
-  public ChunkQueue getEventQueue() {
+  public synchronized ChunkQueue getEventQueue() {
+    if (chunkQueue == null) {
+      chunkQueue = new MemLimitQueue(QUEUE_SIZE_KB * 1024);
+    }
     return chunkQueue;
   }
 
+  public String getDefaultTags() {
+    return defaultTags;
+  }
+  
+  public void addDefaultTag(String tag) {
+    this.defaultTags += " " + tag.trim();
+  }
+  
   /**
    * @return empty list if file does not exist
    * @throws IOException on other error

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java?rev=765736&r1=765735&r2=765736&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
Thu Apr 16 20:03:24 2009
@@ -279,8 +279,7 @@
       byte[] buf, long fileTime) throws InterruptedException {
     final ChunkImpl chunk = new ChunkImpl(type, toWatch.getAbsolutePath(),
         buffOffsetInFile + buf.length, buf, this);
-    String tags = chunk.getTags();
-    chunk.setTags(tags + " time=\"" + fileTime + "\"");
+    chunk.addTag("time=\"" + fileTime + "\"");
     log.info("Adding " + toWatch.getAbsolutePath() + " to the queue");
     eq.add(chunk);
     log.info( toWatch.getAbsolutePath() + " added to the queue");

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=765736&r1=765735&r2=765736&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Thu Apr 16 20:03:24 2009
@@ -19,21 +19,36 @@
 package org.apache.hadoop.chukwa.datacollection.agent;
 
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import org.apache.hadoop.chukwa.datacollection.DataFactory;
-import org.apache.hadoop.chukwa.datacollection.adaptor.*;
+import org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor;
+import org.apache.hadoop.chukwa.datacollection.adaptor.AdaptorException;
 import org.apache.hadoop.chukwa.datacollection.agent.metrics.AgentMetrics;
-import org.apache.hadoop.chukwa.datacollection.connector.*;
+import org.apache.hadoop.chukwa.datacollection.connector.Connector;
 import org.apache.hadoop.chukwa.datacollection.connector.http.HttpConnector;
 import org.apache.hadoop.chukwa.datacollection.test.ConsoleOutConnector;
-import org.apache.hadoop.chukwa.util.*;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.io.*;
 
 /**
  * The local agent daemon that runs on each machine. This class is designed to
@@ -192,7 +207,8 @@
     if (checkpointDir != null && !checkpointDir.exists()) {
       checkpointDir.mkdirs();
     }
-    tags = conf.get("chukwaAgent.tags", "cluster=\"unknown\"");
+    
+    DataFactory.getInstance().addDefaultTag(conf.get("chukwaAgent.tags", "cluster=\"unknown\""));
 
     log.info("Config - CHECKPOINT_BASE_NAME: [" + CHECKPOINT_BASE_NAME + "]");
     log.info("Config - checkpointDir: [" + checkpointDir + "]");
@@ -594,9 +610,4 @@
   AgentControlSocketListener getControlSock() {
     return controlSock;
   }
-
-  public static String getTags() {
-    return tags;
-  }
-
 }



Mime
View raw message