chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r767247 - in /hadoop/chukwa: branches/chukwa-0.1/conf/ branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/conf/ trunk/conf/ trunk/src/java/org/apache/hadoop/chukwa/conf/
Date Tue, 21 Apr 2009 18:44:52 GMT
Author: eyang
Date: Tue Apr 21 18:44:51 2009
New Revision: 767247

URL: http://svn.apache.org/viewvc?rev=767247&view=rev
Log:
CHUKWA-145. Tuned hadoop parameters for demux. (Jerome Boulon via Eric Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.1/conf/chukwa-demux-conf.xml
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
    hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java

Modified: hadoop/chukwa/branches/chukwa-0.1/conf/chukwa-demux-conf.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/chukwa-demux-conf.xml?rev=767247&r1=767246&r2=767247&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/conf/chukwa-demux-conf.xml (original)
+++ hadoop/chukwa/branches/chukwa-0.1/conf/chukwa-demux-conf.xml Tue Apr 21 18:44:51 2009
@@ -5,13 +5,40 @@
 
 <configuration>
 
+<!-- Chukwa Job parameters -->
+	<property>
+	  <name>io.sort.mb</name>
+	  <value>256</value>
+	  <description>The total amount of buffer memory to use while sorting
+	  files, in megabytes.  By default, gives each merge stream 1MB, which
+	  should minimize seeks.</description>
+	</property>
+
+	<property>
+	  <name>fs.inmemory.size.mb</name>
+	  <value>256</value>
+	  <description>The size of the in-memory filsystem instance in MB</description>
+	</property>
+
+	<property>
+	  <name>io.sort.factor</name>
+	  <value>100</value>
+	  <description>The number of streams to merge at once while sorting
+	  files.  This determines the number of open file handles.</description>
+	</property>
 
+<!-- -->
 
+<!-- PostProcessorManager config -->
   <property>
     <name>postProcessorManager.dbloader.ds</name>
     <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage</value>
     <description>Reduce count </description>
   </property>
+
+<!-- -->
+
+<!-- ArchiveManager config -->
   
   <property>
     <name>chukwaArchiveBuilder.reduceCount</name>
@@ -19,6 +46,17 @@
     <description>Reduce count </description>
   </property>
 
+<!-- -->
+
+
+<!-- Demux aliases -->
+
+  <property>
+    <name>HadoopMetricsProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for Hadoop Metrics </description>
+  </property>
+
   <property>
     <name>jvm</name>
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
@@ -42,7 +80,6 @@
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
     <description>Parser class for </description>
   </property>
-
   
   <property>
     <name>SysLog</name>
@@ -129,3 +166,4 @@
    </property>
 
 </configuration>
+ 
\ No newline at end of file

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java?rev=767247&r1=767246&r2=767247&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
Tue Apr 21 18:44:51 2009
@@ -50,18 +50,29 @@
 
       log.info("chukwaConf is " + chukwaConf);
 
-      super.addResource(new Path(chukwaConf + "/chukwa-collector-conf.xml"));
-      log.debug("added chukwa-collector-conf.xml to ChukwaConfiguration");
+      
+      // Hadoop config
+      String hadoopConfDir = System.getenv("HADOOP_CONF_DIR");
+      if (hadoopConfDir != null) {        
+        super.addResource(new Path(hadoopConfDir + "/hadoop-site.xml"));
+        log.debug("added " + hadoopConfDir"/hadoop-site.xml to ChukwaConfiguration");
+      } else {
+        String hadoopHome = System.getenv("HADOOP_HOME");
+        if (hadoopHome != null) {
+          super.addResource(new Path(hadoopHome + "/conf/hadoop-site.xml"));
+          log.debug("added " + hadoopHome + "/hadoop-site.xml to ChukwaConfiguration");
+        }
+      }
+
 
       super.addResource(new Path(chukwaConf + "/chukwa-agent-conf.xml"));
       log.debug("added chukwa-agent-conf.xml to ChukwaConfiguration");
 
+      super.addResource(new Path(chukwaConf + "/chukwa-collector-conf.xml"));
+      log.debug("added chukwa-collector-conf.xml to ChukwaConfiguration");
+
       super.addResource(new Path(chukwaConf + "/chukwa-demux-conf.xml"));
       log.debug("added chukwa-demux-conf.xml to ChukwaConfiguration");
-      
-      super.addResource(new Path(chukwaConf + "/hadoop-site.xml"));
-      log.debug("added hadoop-site.xml to ChukwaConfiguration");
-
     }
   }
 

Modified: hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml?rev=767247&r1=767246&r2=767247&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml (original)
+++ hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml Tue Apr 21 18:44:51 2009
@@ -5,13 +5,40 @@
 
 <configuration>
 
+<!-- Chukwa Job parameters -->
+	<property>
+	  <name>io.sort.mb</name>
+	  <value>256</value>
+	  <description>The total amount of buffer memory to use while sorting
+	  files, in megabytes.  By default, gives each merge stream 1MB, which
+	  should minimize seeks.</description>
+	</property>
+
+	<property>
+	  <name>fs.inmemory.size.mb</name>
+	  <value>256</value>
+	  <description>The size of the in-memory filsystem instance in MB</description>
+	</property>
+
+	<property>
+	  <name>io.sort.factor</name>
+	  <value>100</value>
+	  <description>The number of streams to merge at once while sorting
+	  files.  This determines the number of open file handles.</description>
+	</property>
 
+<!-- -->
 
+<!-- PostProcessorManager config -->
   <property>
     <name>postProcessorManager.dbloader.ds</name>
     <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage</value>
     <description>Reduce count </description>
   </property>
+
+<!-- -->
+
+<!-- ArchiveManager config -->
   
   <property>
     <name>chukwaArchiveBuilder.reduceCount</name>
@@ -19,6 +46,17 @@
     <description>Reduce count </description>
   </property>
 
+<!-- -->
+
+
+<!-- Demux aliases -->
+
+  <property>
+    <name>HadoopMetricsProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for Hadoop Metrics </description>
+  </property>
+
   <property>
     <name>jvm</name>
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
@@ -42,7 +80,6 @@
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
     <description>Parser class for </description>
   </property>
-
   
   <property>
     <name>SysLog</name>
@@ -129,3 +166,4 @@
    </property>
 
 </configuration>
+ 
\ No newline at end of file

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java?rev=767247&r1=767246&r2=767247&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java Tue
Apr 21 18:44:51 2009
@@ -50,18 +50,29 @@
 
       log.info("chukwaConf is " + chukwaConf);
 
-      super.addResource(new Path(chukwaConf + "/chukwa-collector-conf.xml"));
-      log.debug("added chukwa-collector-conf.xml to ChukwaConfiguration");
+      
+      // Hadoop config
+      String hadoopConfDir = System.getenv("HADOOP_CONF_DIR");
+      if (hadoopConfDir != null) {        
+        super.addResource(new Path(hadoopConfDir + "/hadoop-site.xml"));
+        log.debug("added " + hadoopConfDir"/hadoop-site.xml to ChukwaConfiguration");
+      } else {
+        String hadoopHome = System.getenv("HADOOP_HOME");
+        if (hadoopHome != null) {
+          super.addResource(new Path(hadoopHome + "/conf/hadoop-site.xml"));
+          log.debug("added " + hadoopHome + "/hadoop-site.xml to ChukwaConfiguration");
+        }
+      }
+
 
       super.addResource(new Path(chukwaConf + "/chukwa-agent-conf.xml"));
       log.debug("added chukwa-agent-conf.xml to ChukwaConfiguration");
 
+      super.addResource(new Path(chukwaConf + "/chukwa-collector-conf.xml"));
+      log.debug("added chukwa-collector-conf.xml to ChukwaConfiguration");
+
       super.addResource(new Path(chukwaConf + "/chukwa-demux-conf.xml"));
       log.debug("added chukwa-demux-conf.xml to ChukwaConfiguration");
-      
-      super.addResource(new Path(chukwaConf + "/hadoop-site.xml"));
-      log.debug("added hadoop-site.xml to ChukwaConfiguration");
-
     }
   }
 



Mime
View raw message