chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r778826 - in /hadoop/chukwa/trunk: ./ conf/ src/java/org/apache/hadoop/chukwa/datacollection/collector/ src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/
Date Tue, 26 May 2009 18:22:05 GMT
Author: asrabkin
Date: Tue May 26 18:22:05 2009
New Revision: 778826

URL: http://svn.apache.org/viewvc?rev=778826&view=rev
Log:
CHUKWA-30.  Remove HDFS flush and connection holding. (Contributed by Jerome Boulon)

Modified:
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/build.xml
    hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template
    hadoop/chukwa/trunk/default.properties
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=778826&r1=778825&r2=778826&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Tue May 26 18:22:05 2009
@@ -10,6 +10,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-30.  Remove HDFS flush and connection holding in Collector. (Jerome Boulon via
asrabkin)
+
     CHUKWA-142. Remove add(Chunk) from SeqFileWriter.  (Jerome Boulon via asrabkin)
 
     CHUKWA-82.  Added min,max,average to HICC Charting widget. (Terence Kwan via Eric Yang)

Modified: hadoop/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/build.xml?rev=778826&r1=778825&r2=778826&view=diff
==============================================================================
--- hadoop/chukwa/trunk/build.xml (original)
+++ hadoop/chukwa/trunk/build.xml Tue May 26 18:22:05 2009
@@ -935,6 +935,10 @@
 
 		<copy file="${basedir}/conf/chukwa-collector-conf.xml.template" tofile="${build.dir}/conf/chukwa-collector-conf.xml">
 			<filterchain>
+		
+			    <replacetokens>
+					<token key="TODO-COLLECTORS-LOCAL-OUTPUT-DIR" value="${TODO-COLLECTORS-LOCAL-OUTPUT-DIR}"
/>
+				</replacetokens>
 				<replacetokens>
 					<token key="TODO-COLLECTORS-NAMENODE" value="${TODO-COLLECTORS-NAMENODE}" />
 				</replacetokens>

Modified: hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template?rev=778826&r1=778825&r2=778826&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template (original)
+++ hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template Tue May 26 18:22:05 2009
@@ -5,6 +5,24 @@
 
 <configuration>
 
+
+
+<!-- LocalWriter parameters
+  <property>
+    <name>chukwaCollector.localOutputDir</name>
+    <value>@TODO-COLLECTORS-LOCAL-OUTPUT-DIR@</value>
+    <description>Chukwa local data sink directory, see LocalWriter.java</description>
+  </property>
+
+  <property>
+    <name>chukwaCollector.writerClass</name>
+    <value>org.apache.hadoop.chukwa.datacollection.writer.localfs.LocalWriter</value>
+    <description>Local chukwa writer, see LocalWriter.java</description>
+  </property>
+-->
+
+
+
   <property>
     <name>writer.hdfs.filesystem</name>
     <value>@TODO-COLLECTORS-NAMENODE@</value>

Modified: hadoop/chukwa/trunk/default.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/default.properties?rev=778826&r1=778825&r2=778826&view=diff
==============================================================================
--- hadoop/chukwa/trunk/default.properties (original)
+++ hadoop/chukwa/trunk/default.properties Tue May 26 18:22:05 2009
@@ -23,6 +23,7 @@
 
 TODO-ALERT-EMAIL=user@example.com
 TODO-CLUSTER-NAME=chukwa
+TODO-COLLECTORS-LOCAL-OUTPUT-DIR=/tmp/chukwa/dataSink/
 TODO-COLLECTORS-NAMENODE=hdfs://localhost:9000/
 TODO-COLLECTORS-ROTATEINTERVAL=300000
 TODO-COLLECTORS-PORT=8080

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java?rev=778826&r1=778825&r2=778826&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
Tue May 26 18:22:05 2009
@@ -87,7 +87,7 @@
       Context root = new Context(jettyServer, "/", Context.SESSIONS);
       root.addServlet(new ServletHolder(new ServletCollector(conf)), "/*");
       jettyServer.start();
-      jettyServer.setStopAtShutdown(false);
+      jettyServer.setStopAtShutdown(true);
 
       System.out.println("started http collector on port number " + portNum);
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java?rev=778826&r1=778825&r2=778826&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java
Tue May 26 18:22:05 2009
@@ -33,6 +33,7 @@
 import org.apache.hadoop.chukwa.Chunk;
 import org.apache.hadoop.chukwa.ChunkImpl;
 import org.apache.hadoop.chukwa.datacollection.writer.*;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.log4j.Logger;
 
 public class ServletCollector extends HttpServlet {
@@ -101,11 +102,14 @@
 
     // We default to here if the pipeline construction failed or didn't happen.
     try {
-      if (writer == null)
-        writer = new SeqFileWriter();// default to SeqFileWriter
+      if (writer == null) {
+        writer =  new SeqFileWriter();
+      }
+      
       writer.init(conf);
-    } catch (WriterException e) {
-      throw new ServletException("Problem init-ing servlet", e);
+    } catch (Throwable e) {
+      log.warn("Exception during Servel init",e);
+      DaemonWatcher.bailout(-1);
     }
   }
 
@@ -133,17 +137,14 @@
       StringBuilder sb = new StringBuilder();
 
       for (int i = 0; i < numEvents; i++) {
-        // TODO: pass new data to all registered stream handler
-        // methods for this chunk's stream
-        // TODO: should really have some dynamic assignment of events to writers
-
         logEvent = ChunkImpl.read(di);
+        events.add(logEvent);
+
         sb.append("ok:");
         sb.append(logEvent.getData().length);
         sb.append(" bytes ending at offset ");
         sb.append(logEvent.getSeqID() - 1).append("\n");
 
-        events.add(logEvent);
 
         if (FANCY_DIAGNOSTICS) {
           diagnosticPage.sawChunk(logEvent, i);



Mime
View raw message