chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r767731 - in /hadoop/chukwa/trunk: ./ conf/
Date Wed, 22 Apr 2009 23:57:43 GMT
Author: eyang
Date: Wed Apr 22 23:57:43 2009
New Revision: 767731

URL: http://svn.apache.org/viewvc?rev=767731&view=rev
Log:
CHUKWA-131. Added additional Mapred job/task metrics.  (Eric Yang)

Added:
    hadoop/chukwa/trunk/conf/hadoop-metrics.properties.template
Removed:
    hadoop/chukwa/trunk/conf/hadoop-metrics.properties
Modified:
    hadoop/chukwa/trunk/build.xml
    hadoop/chukwa/trunk/conf/alert.template
    hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template
    hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template
    hadoop/chukwa/trunk/conf/chukwa-env.sh.template
    hadoop/chukwa/trunk/default.properties

Modified: hadoop/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/build.xml?rev=767731&r1=767730&r2=767731&view=diff
==============================================================================
--- hadoop/chukwa/trunk/build.xml (original)
+++ hadoop/chukwa/trunk/build.xml Wed Apr 22 23:57:43 2009
@@ -450,16 +450,26 @@
 				<include name="late-log4j.properties" />
 			</fileset>
 		</copy>
-
+		<delete dir="${test.build.dir}/conf" />
+		<mkdir dir="${test.build.dir}/conf" />
+		<copy file="${conf.dir}/jdbc.conf.template" tofile="${test.build.dir}/conf/jdbc.conf"/>
+		<copy file="${conf.dir}/mdl.xml" tofile="${test.build.dir}/conf/mdl.xml"/>
+		<copy file="${conf.dir}/aggregator.sql" tofile="${test.build.dir}/conf/aggregator.sql"/>
+		<copy file="${conf.dir}/database_create_tables.sql" tofile="${test.build.dir}/conf/database_create_tables.sql"/>
+		<copy todir="${test.build.dir}/var">
+                        <fileset dir="${basedir}/test/samples" />
+                </copy>
 	</target>
 
 	<!--printsummary="${test.junit.printsummary}" -->
 
 	<target name="test-chukwa" depends="compile,compile-test" description="Run Chukwa unit
tests">
 		<mkdir dir="${basedir}/var" />
+		<mkdir dir="${test.build.dir}/var" />
 		<junit showoutput="${test.output}" fork="yes" printsummary="withOutAndErr" forkmode="${test.junit.fork.mode}"
maxmemory="${test.junit.maxmemory}" dir="${test.build.dir}/classes/" timeout="${test.timeout}"
errorProperty="tests.failed" failureProperty="tests.failed">
 			<classpath refid="testClasspath" />
-			<env key="DATACONFIG" value="${build.dir}/conf" />
+			<env key="CHUKWA_CONF_DIR" value="${test.build.dir}/conf" />
+			<env key="CHUKWA_DATA_DIR" value="${test.build.dir}/var" />
 			<sysproperty key="test.src.dir" value="${test.src.dir}" />
 			<formatter type="${test.junit.output.format}" />
 			<batchtest todir="${test.build.dir}" unless="testcase">
@@ -857,7 +867,85 @@
 		</copy>
 	</target>
 
-	<target name="package" depends="compress,docs,package-hadoop,package-standalone" description="Build
distribution">
+	<target name="chukwa-config" description="create real files based on templates" depends="init"
>
+	
+		<copy todir="${build.dir}/conf">
+		 <fileset dir="${basedir}/conf">
+		 	<exclude name="*.template" />
+		 </fileset>
+		</copy>
+	
+	    <copy file="${basedir}/conf/alert.template" tofile="${build.dir}/conf/alert">
+	            <filterchain>
+	                    <replacetokens>
+	                            <token key="TODO-ALERT-EMAIL" value="${TODO-ALERT-EMAIL}"
/>
+	                    </replacetokens>
+	            </filterchain>
+	    </copy>
+	
+		<copy file="${basedir}/conf/hadoop-metrics.properties.template" tofile="${build.dir}/conf/hadoop-metrics.properties">
+		        <filterchain>
+		                <replacetokens>
+		                        <token key="TODO-CHUKWA_LOG_DIR" value="${TODO-CHUKWA_LOG_DIR}"
/>
+		                </replacetokens>
+		        </filterchain>
+		</copy>
+		<copy file="${basedir}/conf/chukwa-agent-conf.xml.template" tofile="${build.dir}/conf/chukwa-agent-conf.xml">
+		        <filterchain>
+		                <replacetokens>
+		                        <token key="TODO-CLUSTER-NAME" value="${TODO-CLUSTER-NAME}"
/>
+		                </replacetokens>
+		        </filterchain>
+		</copy>
+		
+		<copy file="${basedir}/conf/chukwa-collector-conf.xml.template" tofile="${build.dir}/conf/chukwa-collector-conf.xml">
+		        <filterchain>
+		                <replacetokens>
+		                        <token key="TODO-COLLECTORS-NAMENODE" value="${TODO-COLLECTORS-NAMENODE}"
/>
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-COLLECTORS-ROTATEINTERVAL" value="${TODO-COLLECTORS-ROTATEINTERVAL}"
/>
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-COLLECTORS-PORT" value="${TODO-COLLECTORS-PORT}"
/>
+		                </replacetokens>
+		
+		        </filterchain>
+		</copy>
+		
+		<copy file="${basedir}/conf/chukwa-env.sh.template" tofile="${build.dir}/conf/chukwa-env.sh">
+		        <filterchain>
+		                <replacetokens>
+		                        <token key="TODO-JAVA-HOME" value="${TODO-JAVA-HOME}" />
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-HADOOP-HOME" value="${TODO-HADOOP-HOME}" />
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-HADOOP-CONF-DIR" value="${TODO-HADOOP-CONF-DIR}"
/>
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-CHUKWA_PID_DIR" value="${TODO-CHUKWA_PID_DIR}"
/>
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-CHUKWA_LOG_DIR" value="${TODO-CHUKWA_LOG_DIR}"
/>
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO-AGENTS-INSTANCE-NAME" value="${TODO-AGENTS-INSTANCE-NAME}"
/>
+		                </replacetokens>
+		                <replacetokens>
+		                        <token key="TODO_CHUKWA_JDBC_DRIVER" value="${TODO_CHUKWA_JDBC_DRIVER}"
/>
+		                </replacetokens>
+		              <replacetokens>
+		                        <token key="TODO_CHUKWA_JDBC_URL_PREFIX" value="${TODO_CHUKWA_JDBC_URL_PREFIX}"
/>
+		                </replacetokens>
+		
+		        </filterchain>
+		</copy>
+
+</target>
+
+	<target name="package" depends="compress,docs,chukwa-config,package-hadoop,package-standalone"
description="Build distribution">
 		<mkdir dir="${build.dir}/${final.name}" />
 		<mkdir dir="${build.dir}/${final.name}/lib" />
 		<mkdir dir="${build.dir}/${final.name}/bin" />
@@ -889,17 +977,10 @@
 		</copy>
 
 		<copy todir="${build.dir}/${final.name}/conf">
-			<fileset dir="${basedir}/conf">
+			<fileset dir="${build.dir}/conf">
 				<exclude name="hadoop-metrics.properties" />
 			</fileset>
 		</copy>
-		<copy file="${basedir}/conf/hadoop-metrics.properties" tofile="${build.dir}/${final.name}/conf/hadoop-metrics.properties">
-			<filterchain>
-				<replacetokens>
-					<token key="CHUKWA_LOG_DIR" value="${CHUKWA_LOG_DIR}" />
-				</replacetokens>
-			</filterchain>
-		</copy>
 
 		<copy todir="${build.dir}/${final.name}/opt" failonerror="false">
 			<fileset dir="${opt.dir}" />

Modified: hadoop/chukwa/trunk/conf/alert.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/alert.template?rev=767731&r1=767730&r2=767731&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/alert.template (original)
+++ hadoop/chukwa/trunk/conf/alert.template Wed Apr 22 23:57:43 2009
@@ -1 +1 @@
-user@example.com
+@TODO-ALERT-EMAIL@

Modified: hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template?rev=767731&r1=767730&r2=767731&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template (original)
+++ hadoop/chukwa/trunk/conf/chukwa-agent-conf.xml.template Wed Apr 22 23:57:43 2009
@@ -8,7 +8,7 @@
   
   <property>
     <name>chukwaAgent.tags</name>
-    <value>cluster="demo"</value>
+    <value>cluster="@TODO-CLUSTER-NAME@"</value>
     <description>The cluster's name for this agent</description>
   </property>
   

Modified: hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template?rev=767731&r1=767730&r2=767731&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template (original)
+++ hadoop/chukwa/trunk/conf/chukwa-collector-conf.xml.template Wed Apr 22 23:57:43 2009
@@ -7,7 +7,7 @@
 
   <property>
     <name>writer.hdfs.filesystem</name>
-    <value>hdfs://localhost:9000/</value>
+    <value>@TODO-COLLECTORS-NAMENODE@</value>
     <description>HDFS to dump to</description>
   </property>
   
@@ -19,13 +19,13 @@
 
   <property>
     <name>chukwaCollector.rotateInterval</name>
-    <value>300000</value>
+    <value>@TODO-COLLECTORS-ROTATEINTERVAL@</value>
     <description>Chukwa rotate interval (ms)</description>
   </property>
 
   <property>
     <name>chukwaCollector.http.port</name>
-    <value>8080</value>
+    <value>@TODO-COLLECTORS-PORT@</value>
     <description>The HTTP port number the collector will listen on</description>
   </property>
 

Modified: hadoop/chukwa/trunk/conf/chukwa-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-env.sh.template?rev=767731&r1=767730&r2=767731&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-env.sh.template (original)
+++ hadoop/chukwa/trunk/conf/chukwa-env.sh.template Wed Apr 22 23:57:43 2009
@@ -6,7 +6,7 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-#export JAVA_HOME=/usr/lib/j2sdk1.5-sun
+export JAVA_HOME=@TODO-JAVA-HOME@
 
 # Optional (.ie. will try sensible defaults)
 # The location of the Hadoop the collector should use. Chukwa uses this to 
@@ -21,7 +21,8 @@
 # if jars or classes can not be found using that default assumption, 
 # Chukwa will default to the hadoop jars that come with it
 # in the hadoopjars directory. 
-#export HADOOP_HOME="/home/user/Development/hadoop-trunk"
+export HADOOP_HOME="@TODO-HADOOP-HOME@"
+export HADOOP_CONF_DIR="@TODO-HADOOP-CONF-DIR@"
 
 # Optional (i.e. will try sensible defaults)
 # The location of a hadoop jars. Setting this will overrided the search for 
@@ -42,10 +43,10 @@
 export chukwaRecordsRepository="/chukwa/repos/"
 
 # The directory where pid files are stored. CHUKWA_HOME/var/run by default.
-#export CHUKWA_PID_DIR="/tmp/chukwa-pid-dir"
+export CHUKWA_PID_DIR=@TODO-CHUKWA_PID_DIR@
 
 # The location of chukwa logs, defaults to CHUKWA_HOME/logs
-#export CHUKWA_LOG_DIR="/tmp/chukwa-log-dir"
+export CHUKWA_LOG_DIR=@TODO-CHUKWA_LOG_DIR@
 
 # The location to store chukwa data, defaults to CHUKWA_HOME/data
 #export CHUKWA_DATA_DIR="${CHUKWA_HOME}/data"
@@ -60,10 +61,10 @@
 #export TORQUE_HOME=/usr/local/torque
 
 # Instance name for chukwa deployment
-export CHUKWA_IDENT_STRING=TODO-AGENTS-INSTANCE-NAME
+export CHUKWA_IDENT_STRING=@TODO-AGENTS-INSTANCE-NAME@
 
 # Datatbase driver name for storing Chukwa Data.
-# export JDBC_DRIVER=
+export JDBC_DRIVER=@TODO_CHUKWA_JDBC_DRIVER@
 
 # Database URL prefix for Database Loader.
-# export JDBC_URL_PREFIX=jdbc://
+export JDBC_URL_PREFIX=@TODO_CHUKWA_JDBC_URL_PREFIX@

Added: hadoop/chukwa/trunk/conf/hadoop-metrics.properties.template
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/hadoop-metrics.properties.template?rev=767731&view=auto
==============================================================================
--- hadoop/chukwa/trunk/conf/hadoop-metrics.properties.template (added)
+++ hadoop/chukwa/trunk/conf/hadoop-metrics.properties.template Wed Apr 22 23:57:43 2009
@@ -0,0 +1,31 @@
+chukwaAgent.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+chukwaAgent.period=60
+chukwaAgent.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+ 
+chukwaHttpSender.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+chukwaHttpSender.period=60
+chukwaHttpSender.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+
+chunkQueue.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+chunkQueue.period=60
+chunkQueue.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+
+dfs.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+dfs.period=60
+dfs.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+
+jvm.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+jvm.period=60
+jvm.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+jvm.uuid=true
+
+mapred.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+mapred.period=60
+mapred.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+mapred.uuid=true
+
+rpc.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+rpc.period=60
+rpc.directory=@TODO-CHUKWA_LOG_DIR@/metrics/
+rpc.uuid=true
+

Modified: hadoop/chukwa/trunk/default.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/default.properties?rev=767731&r1=767730&r2=767731&view=diff
==============================================================================
--- hadoop/chukwa/trunk/default.properties (original)
+++ hadoop/chukwa/trunk/default.properties Wed Apr 22 23:57:43 2009
@@ -19,3 +19,17 @@
 rpm.gid=users
 test.exclude=TestDatabase*
 CHUKWA_LOG_DIR=/usr/local/chukwa/var/log
+
+TODO-ALERT-EMAIL=user@example.com
+TODO-CLUSTER-NAME=chukwa
+TODO-COLLECTORS-NAMENODE=hdfs://localhost:9000/
+TODO-COLLECTORS-ROTATEINTERVAL=300000
+TODO-COLLECTORS-PORT=8080
+TODO-JAVA-HOME=/usr/lib/j2sdk1.5-sun
+TODO-HADOOP-HOME=/home/user/Development/hadoop-trunk
+TODO-HADOOP-CONF-DIR=/home/user/Development/hadoop-conf
+TODO-CHUKWA_PID_DIR=/tmp/chukwa/pidDir
+TODO-CHUKWA_LOG_DIR=/tmp/chukwa/log
+TODO-AGENTS-INSTANCE-NAME=chukwa
+TODO_CHUKWA_JDBC_DRIVER=
+TODO_CHUKWA_JDBC_URL_PREFIX=



Mime
View raw message