chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r782363 - in /hadoop/chukwa/trunk: build.xml conf/chukwa-agent-conf.xml.template conf/chukwa-collector-conf.xml.template conf/chukwa-demux-conf.xml.template conf/configuration.xsl conf/mdl.xml default.properties
Date Sun, 07 Jun 2009 10:55:03 GMT
Author: eyang
Date: Sun Jun  7 10:55:03 2009
New Revision: 782363

URL: http://svn.apache.org/viewvc?rev=782363&view=rev
Log:
CHUKWA-263. Added ability to configure demux parameters during build time. (Jerome Boulon
via Eric Yang)

Added:
    hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml.template
    hadoop/chukwa/trunk/conf/configuration.xsl
Modified:
    hadoop/chukwa/trunk/build.xml
    hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template
    hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template
    hadoop/chukwa/trunk/conf/mdl.xml
    hadoop/chukwa/trunk/default.properties

Modified: hadoop/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/build.xml?rev=782363&r1=782362&r2=782363&view=diff
==============================================================================
--- hadoop/chukwa/trunk/build.xml (original)
+++ hadoop/chukwa/trunk/build.xml Sun Jun  7 10:55:03 2009
@@ -935,6 +935,22 @@
 			</filterchain>
 		</copy>
 
+		<copy file="${basedir}/conf/chukwa-demux-conf.xml.template" tofile="${build.dir}/conf/chukwa-demux-conf.xml">
+			<filterchain>
+		
+			    <replacetokens>
+					<token key="TODO-DEMUX-IO-SORT-MB" value="${TODO-DEMUX-IO-SORT-MB}" />
+				</replacetokens>
+				<replacetokens>
+					<token key="TODO-DEMUX-FS-INMEMORY-SIZE_MB" value="${TODO-DEMUX-FS-INMEMORY-SIZE_MB}"
/>
+				</replacetokens>
+				<replacetokens>
+					<token key="TODO-DEMUX-IO-SORT-FACTOR" value="${TODO-DEMUX-IO-SORT-FACTOR}" />
+				</replacetokens>
+
+			</filterchain>
+		</copy>
+		
 		<copy file="${basedir}/conf/chukwa-env.sh.template" tofile="${build.dir}/conf/chukwa-env.sh">
 			<filterchain>
 				<replacetokens>

Modified: hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template?rev=782363&r1=782362&r2=782363&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template (original)
+++ hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template Sun Jun  7 10:55:03 2009
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
+<?xml-stylesheet type="text/xsl"  href="configuration.xsl"?>
 
 <!-- Put site-specific property overrides in this file. -->
 

Modified: hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template?rev=782363&r1=782362&r2=782363&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template (original)
+++ hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template Sun Jun  7 10:55:03 2009
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
+<?xml-stylesheet type="text/xsl"  href="configuration.xsl"?>
 
 <!-- Put site-specific property overrides in this file. -->
 

Added: hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml.template?rev=782363&view=auto
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml.template (added)
+++ hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml.template Sun Jun  7 10:55:03 2009
@@ -0,0 +1,187 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl"  href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<!-- Chukwa Job parameters -->
+	<property>
+	  <name>io.sort.mb</name>
+	  <value>@TODO-DEMUX-IO-SORT-MB@</value>
+	  <description>The total amount of buffer memory to use while sorting
+	  files, in megabytes.  By default, gives each merge stream 1MB, which
+	  should minimize seeks.</description>
+	</property>
+
+	<property>
+	  <name>fs.inmemory.size.mb</name>
+	  <value>@TODO-DEMUX-FS-INMEMORY-SIZE_MB@</value>
+	  <description>The size of the in-memory filsystem instance in MB</description>
+	</property>
+
+	<property>
+	  <name>io.sort.factor</name>
+	  <value>@TODO-DEMUX-IO-SORT-FACTOR@</value>
+	  <description>The number of streams to merge at once while sorting
+	  files.  This determines the number of open file handles.</description>
+	</property>
+
+<!-- -->
+
+<!-- PostProcessorManager config -->
+  <property>
+    <name>postProcessorManager.dbloader.ds</name>
+    <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage,chunkQueue,chukwaAgent,chukwaHttpSender,JobConfData</value>
+    <description>Reduce count </description>
+  </property>
+
+<!-- -->
+
+<!-- ArchiveManager config -->
+  
+  <property>
+    <name>chukwaArchiveBuilder.reduceCount</name>
+    <value>5</value>
+    <description>Reduce count </description>
+  </property>
+
+<!-- -->
+
+
+<!-- Demux aliases -->
+
+  <property>
+    <name>HadoopMetricsProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for Hadoop Metrics </description>
+  </property>
+
+  <property>
+    <name>jvm</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>mapred</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>rpc</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>dfs</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for </description>
+  </property>
+  
+  <property>
+    <name>SysLog</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.SysLog</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>Df</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Df</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>HadoopLog</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopLogProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>Iostat</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Iostat</value>
+    <description>Parser class for </description>
+  </property>
+ 
+   <property>
+    <name>PbsNodes</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.PbsNodes</value>
+    <description>Parser class for </description>
+  </property>
+ 
+   <property>
+    <name>Sar</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Sar</value>
+    <description>Parser class for </description>
+  </property>
+
+   <property>
+    <name>TsProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.TsProcessor</value>
+    <description>Parser class for </description>
+   </property>
+  
+   <property>
+    <name>Top</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Top</value>
+    <description>Parser class for </description>
+   </property>
+
+   <property>
+    <name>Torque</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Torque</value>
+    <description>Parser class for Parsing qstat and tracejob</description>
+   </property>
+  
+   <property>
+    <name>DbLoader</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.TsProcessor</value>
+    <description>Parser class for </description>
+   </property>
+
+   <property>
+    <name>JobHistory</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.JobLog</value>
+    <description>Parser class for Job History Log File</description>
+   </property>
+
+   <property>
+    <name>JobConf</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.JobConfProcessor</value>
+    <description>Parser class for Map reduce Job Configuration</description>
+   </property>
+    
+   <property>
+     <name>ClientTrace</name>
+     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.ClientTraceProcessor</value>
+     <description>Parser class for TaskTracker and DataNode clienttrace data</description>
+   </property>
+ 
+   <property>
+    <name>HDFSUsage</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.JPluginMapper</value>
+    <description></description>
+   </property>
+
+   <property>
+    <name>chukwaAgent</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor</value>
+    <description></description>
+   </property>
+
+   <property>
+    <name>chunkQueue</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor</value>
+    <description></description>
+   </property>
+
+   <property>
+    <name>chukwaHttpSender</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor</value>
+    <description></description>
+   </property>
+   
+</configuration>
+ 

Added: hadoop/chukwa/trunk/conf/configuration.xsl
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/configuration.xsl?rev=782363&view=auto
==============================================================================
--- hadoop/chukwa/trunk/conf/configuration.xsl (added)
+++ hadoop/chukwa/trunk/conf/configuration.xsl Sun Jun  7 10:55:03 2009
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+  <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+  <td><xsl:value-of select="value"/></td>
+  <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>

Modified: hadoop/chukwa/trunk/conf/mdl.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/mdl.xml?rev=782363&r1=782362&r2=782363&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/mdl.xml (original)
+++ hadoop/chukwa/trunk/conf/mdl.xml Sun Jun  7 10:55:03 2009
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
 <!-- Put site-specific property overrides in this file. -->
 

Modified: hadoop/chukwa/trunk/default.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/default.properties?rev=782363&r1=782362&r2=782363&view=diff
==============================================================================
--- hadoop/chukwa/trunk/default.properties (original)
+++ hadoop/chukwa/trunk/default.properties Sun Jun  7 10:55:03 2009
@@ -36,3 +36,6 @@
 TODO_CHUKWA_JDBC_DRIVER=
 TODO_CHUKWA_JDBC_URL_PREFIX=
 TODO-NAGIOS-SERVER=localhost
+TODO-DEMUX-IO-SORT-MB=64
+TODO-DEMUX-FS-INMEMORY-SIZE_MB=64
+TODO-DEMUX-IO-SORT-FACTOR=10
\ No newline at end of file



Mime
View raw message