hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r723855 [2/23] - in /hadoop/core/trunk: ./ src/contrib/ src/contrib/chukwa/ src/contrib/chukwa/bin/ src/contrib/chukwa/conf/ src/contrib/chukwa/docs/ src/contrib/chukwa/docs/paper/ src/contrib/chukwa/hadoop-packaging/ src/contrib/chukwa/lib...
Date Fri, 05 Dec 2008 20:30:21 GMT
Modified: hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh Fri Dec  5 12:30:14 2008
@@ -41,8 +41,8 @@
   pid=`head ${pidFile}`
   ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep TorqueDataLoader | grep -v grep | wc -l`
   if [ $ChildPIDRunningStatus -lt 1 ]; then
-      ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
+      ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
   fi 
 else
-      ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
+      ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
 fi

Added: hadoop/core/trunk/src/contrib/chukwa/bin/validateDemux.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/validateDemux.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/validateDemux.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/validateDemux.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,48 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+echo "hadoop jar for agent is " ${HADOOP_JAR}
+now=`date +%Y%m%d_%H_%M%S`
+hdfsDir="/test_$now/"
+
+HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
+
+$HADOOP_CMDE dfs -mkdir ${hdfsDir}
+
+echo "Moving data to HDFS: ${hdfsDir}"
+
+$HADOOP_CMDE dfs -put ${CHUKWA_HOME}/data/demuxData ${hdfsDir}/
+
+echo "demuxTestData: "
+$HADOOP_CMDE dfs -ls ${hdfsDir}/demuxData/input
+exitCode=$?
+echo "ls ExitCode: ${exitCode} "
+ 
+$HADOOP_CMDE jar  ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.Demux -Dmapred.compress.map.output=true -Dmapred.map.output.compression.codec=org.apache.hadoop.io.compress.LzoCodec -Dmapred.output.compress=true -Dmapred.output.compression.type=BLOCK -r 4 ${hdfsDir}/demuxData/input ${hdfsDir}/demuxData/output
+exitCode=$?
+echo "Demux ExitCode: ${exitCode} "
+
+${JAVA_HOME}/bin/java -Xms10M -Xmx32M -classpath /tmp/chukwaTest.jar:${CLASSPATH}:${HADOOP_JAR}:${COMMON} org.apache.hadoop.chukwa.validationframework.DemuxDirectoryValidator -hdfs ${hdfsDir}/demuxData/gold ${hdfsDir}/demuxData/output
+exitCode=$?
+echo "Validation ExitCode: ${exitCode} "
+

Modified: hadoop/core/trunk/src/contrib/chukwa/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/build.xml?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/build.xml (original)
+++ hadoop/core/trunk/src/contrib/chukwa/build.xml Fri Dec  5 12:30:14 2008
@@ -19,42 +19,166 @@
 
 <project name="chukwa" default="main">
 
-  <import file="../build-contrib.xml"/>
-	<property name="chukwaVersion" value="0.1.0"/>
+	<property name="name" value="chukwa"/>
+        <property name="chukwaVersion" value="0.1.1"/>
+	<property name="rpm.prefix" value="/usr/local"/>
+	<property name="hadoop.conf" value="/usr/local/hadoop/conf"/>
+	<property name="final.name" value="${name}-${chukwaVersion}"/>
+
+        <import file="../build-contrib.xml" optional="true"/>
+        <available file="../build-contrib.xml" property="present"/>
+
+        <condition property="standaloneMode">
+                <and><not><available file="../build-contrib.xml" property="present"/></not></and>
+        </condition>
+
+        <condition property="hadoopMode">
+                <and><available file="../build-contrib.xml" property="present"/></and>
+        </condition>
+
+        <target name="init-standalone" if="standaloneMode">
+          <echo>Standalone Mode</echo>
+          <property environment="env"/>
+	  <property name="basedir" value="."/>
+          <!--SET UP HADOOP JARS-->
+          <property name="hadoop.home.dir" value="${env.HADOOP_HOME}"/>
+          <echo message="HADOOP_HOME set to: ${hadoop.home.dir}"/>
+          <property name="hadoop.default.jars.dir" value="${basedir}/hadoopjars"/>
+
+          <condition property="hadoop.home.jars.dir" value="${hadoop.home.dir}/build" else="${basedir}/hadoopjars">
+                <available file="${hadoop.home.dir}/build"/>
+          </condition>
+          <echo message="hadoop.home.jars.dir set to ${hadoop.home.jars.dir}"/>
+
+          <property name="build.dir" value="${basedir}/build"/>
+          <property name="build.classes" value="${basedir}/build/classes"/>
+          <property name="test.build.dir" value="${build.dir}/test"/>
+          <property name="test.build.classes" value="${test.build.dir}/classes"/>
+          <property name="lib.dir" value="${basedir}/lib"/>
+          <path id="classpath">
+                  <fileset dir="${lib.dir}">
+                          <include name="**/*.jar" />
+                          <exclude name="**/excluded/" />
+                  </fileset>
+                  <fileset dir="${hadoop.default.jars.dir}">   <!-- ASR -->
+                          <include name="**/*.jar" />
+                          <exclude name="**/*core*.jar" />
+                  </fileset>
+                  <fileset dir="${hadoop.home.jars.dir}">   <!-- ASR -->
+                          <include name="**/*core*.jar" />
+                  </fileset>
+          </path>
+          <path id="testClasspath">
+                  <pathelement location="${build.classes}"/>
+                  <pathelement location="${test.build.classes}"/>
+                  <fileset dir="${lib.dir}">
+                          <include name="**/*.jar" />
+                          <exclude name="**/excluded/" />
+                  </fileset>
+                  <fileset dir="${hadoop.default.jars.dir}">   <!-- ASR -->
+                          <include name="**/*.jar" />
+                          <exclude name="**/*core*.jar" />
+                  </fileset>
+                  <fileset dir="${hadoop.home.jars.dir}">   <!-- ASR -->
+                          <include name="**/*core*.jar" />
+                  </fileset>
+          </path>
+          <path id="testDemuxClasspath">
+                  <pathelement location="${build.classes}"/>
+                  <pathelement location="${test.build.classes}"/>
+                  <fileset dir="${hadoop.jar}">
+                          <include name="**/*.jar" />
+                          <exclude name="**/excluded/" />
+                  </fileset>
+                  <fileset dir="${lib.dir}">
+                          <include name="**/*.jar" />
+                          <exclude name="**/excluded/" />
+                  </fileset>
 
-	<property name="lib.dir" value="${basedir}/lib"/>
-	<property name="src.dir" value="${basedir}/src"/>
-        <property name="build.classes" value="${build.dir}/classes"/>
-	<property name="conf.dir" value="${basedir}/conf"/>
-	<property name="docs.dir" value="${basedir}/docs"/>
-	<property name="tools.dir" value="${basedir}/tools"/>
-	<property name="dist.dir" value="${basedir}/dist"/>
-	<property name="opt.dir" value="${basedir}/opt"/>
-        <property name="javac.debug" value="on"/>
-        <property name="javac.version" value="1.6"/>
+          </path>
+        </target>
+        
+        <target name="init-hadoop" if="hadoopMode">
+                <echo>Hadoop Mode</echo>
+	        <property name="build.dir" value="${basedir}/build"/>
+	        <property name="build.classes" value="${basedir}/build/classes"/>
+                <property name="test.build.dir" value="${build.dir}/test"/>
+                <property name="test.build.classes" value="${test.build.dir}/classes"/>
+                <delete file="${build.dir}/${final.name}/hadoop-*-core.jar" />
+	        <property name="lib.dir" value="${basedir}/lib"/>
+	        <path id="classpath">
+		        <fileset dir="${lib.dir}">
+			        <include name="**/*.jar" />
+			        <exclude name="**/excluded/" />
+		        </fileset>
+		        <fileset dir="${hadoop.root}/lib">
+			        <include name="**/*.jar" />
+			        <exclude name="**/excluded/" />
+		        </fileset>
+                        <pathelement location="${hadoop.root}/build/classes"/>
+	        </path>
+                <path id="testClasspath">
+                        <pathelement location="${build.classes}"/>
+                        <pathelement location="${test.build.classes}"/>
+                        <fileset dir="${lib.dir}">
+                                <include name="**/*.jar" />
+                                <exclude name="**/excluded/" />
+                        </fileset>
+		        <fileset dir="${hadoop.root}/lib">
+			        <include name="**/*.jar" />
+                                <exclude name="**/excluded/" />
+                        </fileset>
+                        <pathelement location="${hadoop.root}/build/classes"/>
+                        <pathelement location="${hadoop.root}/build/test/classes"/>
+                </path>
+
+                <path id="testDemuxClasspath">
+                        <pathelement location="${build.classes}"/>
+                        <pathelement location="${test.build.classes}"/>
+                        <fileset dir="${hadoop.root}/lib">
+                                <include name="**/*.jar" />
+                                <exclude name="**/excluded/" />
+                        </fileset>
+                        <fileset dir="${lib.dir}">
+                                <include name="**/*.jar" />
+                                <exclude name="**/excluded/" />
+                        </fileset>
+                        <pathelement location="${hadoop.root}/build/classes"/>
+                        <pathelement location="${hadoop.root}/build/test/classes"/>
+                </path>
        
-        <property name="test.src.dir" value="${basedir}/src/test"/>
-        <property name="test.lib.dir" value="${basedir}/src/test/lib"/>
-        <property name="test.build.dir" value="${build.dir}/test"/>
-        <property name="test.generated.dir" value="${test.build.dir}/src"/>
-        <property name="test.build.data" value="${test.build.dir}/data"/>
-        <property name="test.cache.data" value="${test.build.dir}/cache"/>
-        <property name="test.debug.data" value="${test.build.dir}/debug"/>
-        <property name="test.log.dir" value="${test.build.dir}/logs"/>
-        <property name="test.build.classes" value="${test.build.dir}/classes"/>
-        <property name="test.build.testjar" value="${test.build.dir}/testjar"/>
-        <property name="test.include" value="Test*"/>
-        <property name="test.classpath.id" value="test.classpath"/>
-        <property name="test.output" value="yes"/>
-        <property name="test.timeout" value="900000"/>
-        <property name="test.junit.output.format" value="plain"/>
-        <property name="test.junit.fork.mode" value="perTest" />
-        <property name="test.junit.printsummary" value="yes" />
-        <property name="test.junit.haltonfailure" value="yes" />
-        <property name="test.junit.maxmemory" value="256m" />
- 
 
-	<target name="init">
+        </target>
+
+	<target name="init" depends="init-standalone,init-hadoop">
+                <property name="src.dir" value="${basedir}/src"/>
+                <property name="build.classes" value="${build.dir}/classes"/>
+                <property name="conf.dir" value="${basedir}/conf"/>
+                <property name="docs.dir" value="${basedir}/docs"/>
+                <property name="tools.dir" value="${basedir}/tools"/>
+                <property name="dist.dir" value="${basedir}/dist"/>
+                <property name="opt.dir" value="${basedir}/opt"/>
+                <property name="javac.debug" value="on"/>
+                <property name="javac.version" value="1.6"/>
+                <property name="test.src.dir" value="${basedir}/src/test"/>
+                <property name="test.lib.dir" value="${basedir}/src/test/lib"/>
+                <property name="test.build.dir" value="${build.dir}/test"/>
+                <property name="test.generated.dir" value="${test.build.dir}/src"/>
+                <property name="test.build.data" value="${test.build.dir}/data"/>
+                <property name="test.cache.data" value="${test.build.dir}/cache"/>
+                <property name="test.debug.data" value="${test.build.dir}/debug"/>
+                <property name="test.log.dir" value="${test.build.dir}/logs"/>
+                <property name="test.build.classes" value="${test.build.dir}/classes"/>
+                <property name="test.build.testjar" value="${test.build.dir}/testjar"/>
+                <property name="test.include" value="Test*"/>
+                <property name="test.classpath.id" value="test.classpath"/>
+                <property name="test.output" value="yes"/>
+                <property name="test.timeout" value="900000"/>
+                <property name="test.junit.output.format" value="plain"/>
+                <property name="test.junit.fork.mode" value="perTest" />
+                <property name="test.junit.printsummary" value="yes" />
+                <property name="test.junit.haltonfailure" value="yes" />
+                <property name="test.junit.maxmemory" value="256m" />
 		<mkdir dir="${build.dir}"/>
 		<mkdir dir="${build.classes}"/>
 		<mkdir dir="${build.dir}/test"/>
@@ -63,7 +187,6 @@
 		</exec>
 
 		<mkdir dir="${dist.dir}"/>
-		<property name="final.name" value="${name}-${chukwaVersion}"/>
 
 	</target>
 
@@ -74,8 +197,11 @@
 	</target>
 
 	<target name="compile" depends="init" description="Compilation target">
+		<mkdir dir="${build.dir}"/>
+		<mkdir dir="${build.classes}"/>
+		<mkdir dir="${build.dir}/test"/>
 		<javac srcdir="src/java/org/apache/hadoop/chukwa" destdir="${build.classes}" excludes="**/ChukwaTTInstru.java" debug="${javac.debug}">
-			<classpath refid="chukwaHadoopClasspath" />
+			<classpath refid="classpath" />
 		</javac>
 	</target>
 
@@ -88,8 +214,9 @@
            <delete dir="${test.build.classes}"/>
            <mkdir dir="${test.build.classes}"/>
 
-            <javac srcdir="${test.src.dir}/org/apache/hadoop/chukwa" destdir="${test.build.dir}/classes" excludes="**/TestDemux.java" debug="${javac.debug}">
-                 <classpath refid="chukwaTestClasspath" />
+            <javac srcdir="${test.src.dir}/org/apache/hadoop/chukwa" destdir="${test.build.dir}/classes"  debug="${javac.debug}">
+                 <classpath refid="testClasspath" />
+                 <classpath refid="testDemuxClasspath" />
             </javac>
         </target>
 
@@ -104,7 +231,7 @@
             maxmemory="${test.junit.maxmemory}"
             dir="${test.build.dir}/classes/" timeout="${test.timeout}"
             errorProperty="tests.failed" failureProperty="tests.failed">
-           <classpath refid="chukwaTestClasspath"/>
+           <classpath refid="testClasspath"/>
            <sysproperty key="test.src.dir" value="${test.src.dir}"/>
            <formatter type="${test.junit.output.format}" />
            <batchtest todir="${test.build.dir}" >
@@ -167,6 +294,7 @@
 			<fileset dir="${build.dir}/collector" includes="**" />
 		</jar>
 	</target>
+
 	<target name="tools_jar" depends="compile, collector" description="Create tools jar">
 		<jar jarfile="${build.dir}/tools-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/inputtools/**/*.class">
 			<fileset dir="${basedir}/src/java">
@@ -221,7 +349,7 @@
                             </section>
                         </manifest>
 
-			<fileset dir="${build.dir}">
+			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/chukwa/**/*.class"/>
 			</fileset>
 			<fileset dir="${basedir}/src/java">
@@ -235,10 +363,13 @@
 		<jar jarfile="${build.dir}/chukwa-hadoop-${chukwaVersion}-client.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class">
 			<fileset dir="${basedir}/src/java">
 				<include name="org/apache/hadoop/chukwa/inputtools/log4j/**/*.java"/>
-		        <include name="org/apache/hadoop/chukwa/datacollection/client/**/*.java"/>
-		        <include name="org/apache/hadoop/chukwa/util/**/*.java"/>
+		                <include name="org/apache/hadoop/chukwa/datacollection/client/**/*.java"/>
+		                <include name="org/apache/hadoop/chukwa/util/**/*.java"/>
 			</fileset>
-			<fileset dir="${build.dir}">
+			<fileset dir="${basedir}/conf">
+		                <include name="chukwa-hadoop-metrics-log4j.properties"/>
+			</fileset>
+			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/chukwa/datacollection/client/**/*.class"/>
 				<include name="org/apache/hadoop/chukwa/util/**/*.class"/>
 				<include name="org/apache/hadoop/chukwa/datacollection/controller/*.class"/>
@@ -246,8 +377,90 @@
 		</jar>
 	</target>
 
-	<target name="compress" depends="compile,collector,collector_jar,tools_jar,agent_jar,chukwa_jar,chukwa-hadoop_jar" description="Compression target">
+        <target name="hicc" depends="compile, chukwa_jar, package-hadoop, package-standalone" description="Prepare hicc.war">
+                <mkdir dir="${build.dir}/hicc"/>
+                <mkdir dir="${build.dir}/hicc/WEB-INF"/>
+                <mkdir dir="${build.dir}/hicc/WEB-INF/classes"/>
+                <mkdir dir="${build.dir}/hicc/WEB-INF/lib"/>
+                <mkdir dir="${build.dir}/hicc/META-INF"/>
+                <copy todir="${build.dir}/hicc">
+                        <fileset dir="${basedir}/src/web/hicc">
+                                <include name="**" />
+                        </fileset>
+                </copy>
+                <copy todir="${build.dir}/hicc/WEB-INF/classes">
+                        <fileset dir="${build.classes}">
+                                <include name="org/apache/hadoop/chukwa/hicc/**/*.class" />
+                        </fileset>
+                        <fileset dir="${build.classes}">
+                                <include name="org/apache/hadoop/chukwa/conf/**/*.class" />
+                        </fileset>
+                        <fileset dir="${build.classes}">
+                                <include name="org/apache/hadoop/chukwa/datacollection/**/*.class" />
+                        </fileset>
+                        <fileset dir="${build.classes}">
+                                <include name="org/apache/hadoop/chukwa/extraction/engine/**/*.class" />
+                        </fileset>
+                        <fileset dir="${build.classes}">
+                                <include name="org/apache/hadoop/chukwa/inputtools/*.class" />
+                        </fileset>
+                        <fileset dir="${build.classes}">
+                                <include name="org/apache/hadoop/chukwa/util/**/*.class" />
+                        </fileset>
+
+                        <fileset dir="${basedir}/src/java">
+                                <include name="org/apache/hadoop/chukwa/hicc/**/*.java" />
+                        </fileset>
+                        <fileset dir="${basedir}/src/java">
+                                <include name="org/apache/hadoop/chukwa/conf/**/*.java" />
+                        </fileset>
+                        <fileset dir="${basedir}/src/java">
+                                <include name="org/apache/hadoop/chukwa/datacollection/**/*.java" />
+                        </fileset>
+                        <fileset dir="${basedir}/src/java">
+                                <include name="org/apache/hadoop/chukwa/extraction/engine/**/*.java" />
+                        </fileset>
+                        <fileset dir="${basedir}/src/java">
+                                <include name="org/apache/hadoop/chukwa/inputtools/*.java" />
+                        </fileset>
+                        <fileset dir="${basedir}/src/java">
+                                <include name="org/apache/hadoop/chukwa/util/**/*.java" />
+                        </fileset>
+
+                </copy>
+                <copy todir="${build.dir}/hicc/WEB-INF/lib">
+                        <fileset dir="${basedir}/lib">
+                                <include name="json.jar" />
+                                <include name="jstl.jar" />
+                                <include name="servlet.jar" />
+                                <include name="taglibs.jar" />
+                                <include name="commons-fileupload-*.jar" />
+                                <include name="commons-httpclient-*.jar" />
+                                <include name="commons-logging-adapters-*.jar" />
+                                <include name="commons-io-*.jar" />
+                                <include name="commons-logging-api-*.jar" />
+                                <include name="commons-logging.jar" />
+                                <include name="log4j-*.jar" />
+                        </fileset>
+                        <fileset dir="${build.dir}">
+                                <include name="${name}-core-${chukwaVersion}.jar" />
+                        </fileset>
+                        <fileset dir="${build.dir}/${final.name}/hadoopjars">
+                                <include name="*.jar" />
+                                <exclude name="jasper-*.jar" />
+                                <exclude name="jetty-*.jar" />
+                                <exclude name="jsp-api.jar" />
+                        </fileset>
+                </copy>
+        </target>
 
+        <target name="hicc_war" depends="compile, hicc" description="Create hicc jar">
+                <jar jarfile="${build.dir}/hicc.war" basedir="${build.dir}/hicc">
+                        <fileset dir="${build.dir}/hicc" includes="**" />
+                </jar>
+        </target>
+
+	<target name="compress" depends="compile,collector,collector_jar,hicc_war,tools_jar,agent_jar,chukwa_jar,chukwa-hadoop_jar" description="Compression target">
 		<copy todir="." includeEmptyDirs="false">
 			<fileset dir="${build.dir}">
 				<exclude name="**" />
@@ -256,7 +469,7 @@
 		</copy>
 	</target>
 
-	<target name="test" depends="compile,compile-test,test-chunk,test-input-tools" description="Automated Test Framework">
+	<target name="test" depends="compile,compile-test,test-chunk,test-input-tools,test-agent,test-database" description="Automated Test Framework">
 	</target>
 
 	<target name="test-input-tools" depends="compile,compile-test" description="Test Input Tools">
@@ -270,7 +483,7 @@
              errorProperty="tests.failed" failureProperty="tests.failed">
                <sysproperty key="CHUKWA_HOME" value="${basedir}"/>
                <sysproperty key="CHUKWA_CONF_DIR" value="${basedir}/conf"/>
-               <classpath refid="chukwaTestClasspath"/>
+               <classpath refid="testClasspath"/>
                <formatter type="${test.junit.output.format}" />
                <batchtest todir="${build.dir}/test" >
                <fileset dir="${test.src.dir}" includes="org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/*.java"/>
@@ -282,6 +495,85 @@
             <fail if="tests.failed">Tests failed!</fail>
 	</target>
 
+        <!-- ================================================================== -->
+        <!-- Clean.  Delete the build files, and their directories              -->
+        <!-- ================================================================== -->
+        <target name="clean" depends="init" description="Clean.  Delete the build files, and their directories">
+            <delete dir="${build.dir}"/>
+            <delete dir="${docs.src}/build"/>
+        </target>
+	<target name="test-agent" depends="compile,compile-test" description="Test Agent">
+            <junit showoutput="${test.output}"
+             printsummary="${test.junit.printsummary}"
+             haltonfailure="${test.junit.haltonfailure}"
+             fork="yes"
+             forkmode="${test.junit.fork.mode}"
+             maxmemory="${test.junit.maxmemory}"
+             dir="${basedir}" timeout="${test.timeout}"
+             errorProperty="tests.failed" failureProperty="tests.failed">
+               <sysproperty key="CHUKWA_HOME" value="${basedir}"/>
+               <sysproperty key="CHUKWA_CONF_DIR" value="${basedir}/conf"/>
+               <classpath refid="testClasspath"/>
+               <formatter type="${test.junit.output.format}" />
+               <batchtest todir="${build.dir}/test" >
+               <fileset dir="${test.src.dir}" includes="org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java"/>
+               </batchtest>
+            </junit> 
+            <delete>
+               <fileset dir="${basedir}/var" includes="*"/>
+            </delete>
+            <fail if="tests.failed">Tests failed!</fail>
+	</target>
+
+	<target name="test-demux" depends="compile,compile-test" description="Test Demux">
+           <copy file="${basedir}/data/demuxData.tgz" todir="${test.cache.data}"/> 
+           <junit showoutput="${test.output}"
+             printsummary="${test.junit.printsummary}"
+             haltonfailure="${test.junit.haltonfailure}"
+             fork="yes"
+             forkmode="${test.junit.fork.mode}"
+             maxmemory="${test.junit.maxmemory}"
+             dir="${basedir}" timeout="${test.timeout}"
+             errorProperty="tests.failed" failureProperty="tests.failed">
+               <sysproperty key="test.build.data" value="${test.build.data}"/>
+               <sysproperty key="test.demux.data" value="${test.cache.data}"/>
+               <sysproperty key="CHUKWA_HOME" value="${basedir}"/>
+               <sysproperty key="CHUKWA_CONF_DIR" value="${basedir}/conf"/>
+               <classpath refid="testDemuxClasspath"/>
+               <formatter type="${test.junit.output.format}" />
+               <batchtest todir="${build.dir}/test" >
+               <fileset dir="${test.src.dir}" includes="org/apache/hadoop/chukwa/validationframework/TestDemux.java"/>
+               </batchtest>
+            </junit> 
+            <delete>
+               <fileset dir="${basedir}/var" includes="*"/>
+            </delete>
+            <fail if="tests.failed">Tests failed!</fail>
+	</target>
+
+	<target name="test-database" depends="compile,compile-test" description="Test Database">
+            <junit showoutput="${test.output}"
+             printsummary="${test.junit.printsummary}"
+             haltonfailure="${test.junit.haltonfailure}"
+             fork="yes"
+             forkmode="${test.junit.fork.mode}"
+             maxmemory="${test.junit.maxmemory}"
+             dir="${basedir}" timeout="${test.timeout}"
+             errorProperty="tests.failed" failureProperty="tests.failed">
+               <sysproperty key="CHUKWA_HOME" value="${basedir}"/>
+               <sysproperty key="CHUKWA_CONF_DIR" value="${basedir}/conf"/>
+               <classpath refid="testClasspath"/>
+               <formatter type="${test.junit.output.format}" />
+               <batchtest todir="${build.dir}/test" >
+               <fileset dir="${test.src.dir}" includes="org/apache/hadoop/chukwa/*.java"/>
+               </batchtest>
+            </junit> 
+            <delete>
+               <fileset dir="${basedir}/var" includes="*"/>
+            </delete>
+            <fail if="tests.failed">Tests failed!</fail>
+	</target>
+
 	<!-- ====================================================== -->
 	<!-- Macro definitions                                      -->
 	<!-- ====================================================== -->
@@ -301,7 +593,37 @@
 	<!-- ================================================================== -->
 	<!--                                                                    -->
 	<!-- ================================================================== -->
-	<target name="package" depends="compress" description="Build distribution">
+
+        <target name="package-hadoop" description="package hadoop from hadoop source" if="hadoopMode">
+		<mkdir dir="${build.dir}/${final.name}/hadoopjars"/>
+                <copy toDir="${build.dir}/${final.name}/hadoopjars">
+                        <fileset dir="${hadoop.root}/build">
+                                <include name="hadoop-*-core.jar" />
+                        </fileset>
+                        <fileset dir="${hadoop.root}/lib">
+                                <include name="commons-cli-*.jar" />
+                                <include name="commons-codec-*.jar" />
+                                <include name="commons-el.jar" />
+                                <include name="commons-httpclient-*.jar" />
+                                <include name="commons-logging-*.jar" />
+                                <include name="commons-net-*.jar" />
+                                <include name="jasper-*.jar" />
+                                <include name="jetty-*.jar" />
+                                <include name="jsp-api.jar" />
+                        </fileset>
+                </copy>
+        </target>
+
+        <target name="package-standalone" description="package hadoop from hadoop source" if="standaloneMode">
+		<mkdir dir="${build.dir}/${final.name}/hadoopjars"/>
+		<copy todir="${build.dir}/${final.name}/hadoopjars" includeEmptyDirs="false">
+			<fileset dir="${basedir}/hadoopjars">
+				<include name="**/*.jar" />
+			</fileset>
+		</copy>
+        </target>
+
+	<target name="package" depends="compress,package-hadoop,package-standalone" description="Build distribution">
 		<mkdir dir="${build.dir}/${final.name}"/>
 		<mkdir dir="${build.dir}/${final.name}/lib"/>
 		<mkdir dir="${build.dir}/${final.name}/bin"/>
@@ -330,11 +652,11 @@
 		</exec>
 
 		<copy todir="${build.dir}/${final.name}/bin">
-			<fileset dir="bin"/>
+			<fileset dir="${basedir}/bin"/>
 		</copy>
 
 		<copy todir="${build.dir}/${final.name}/conf">
-			<fileset dir="${conf.dir}"/>
+			<fileset dir="${basedir}/conf"/>
 		</copy>
 
 		<copy todir="${build.dir}/${final.name}/docs">
@@ -367,48 +689,140 @@
 
 		<copy todir="${build.dir}/${final.name}" file="build.xml"/>
 
-		<chmod perm="ugo+x" type="file" parallel="false">
-			<fileset dir="${build.dir}/${final.name}/bin">
-				<include name="*/bin/*" />
-			</fileset>
-		</chmod>
-
 	</target>
 
 	<!-- ================================================================== -->
 	<!-- Make release tarball                                               -->
 	<!-- ================================================================== -->
 	<target name="tar" depends="package" description="Make release tarball">
-		<macro_tar param.destfile="${dist.dir}/${final.name}.tar.gz">
+		<macro_tar param.destfile="${build.dir}/${final.name}.tar.gz">
 			<param.listofitems>
 				<tarfileset dir="${build.dir}" mode="664">
+					<exclude name="${final.name}/bin/*" />
 					<exclude name="${final.name}/org/*" />
 					<exclude name="${final.name}/collector/**" />
 					<exclude name="${final.name}/${final.name}/**" />
+					<exclude name="${final.name}/tools/**" />
+					<exclude name="${final.name}/hicc/*" />
 					<include name="${final.name}/**" />
 				</tarfileset>
 				<tarfileset dir="${build.dir}" mode="755">
 					<include name="${final.name}/bin/*" />
+					<include name="${final.name}/tools/**" />
 				</tarfileset>
 			</param.listofitems>
 		</macro_tar>
 	</target>
 
 	<target name="binary" depends="package" description="Make tarball without source and documentation">
-		<macro_tar param.destfile="${dist.dir}/${final.name}-bin.tar.gz">
+		<macro_tar param.destfile="${build.dir}/${final.name}-bin.tar.gz">
 			<param.listofitems>
 				<tarfileset dir="${build.dir}" mode="664">
+					<exclude name="${final.name}/bin/*" />
+					<exclude name="${final.name}/org/*" />
+					<exclude name="${final.name}/collector/**" />
+					<exclude name="${final.name}/${final.name}/**" />
+					<exclude name="${final.name}/tools/**" />
+					<exclude name="${final.name}/hicc/*" />
 					<exclude name="${final.name}/src/**" />
 					<exclude name="${final.name}/docs/**" />
 					<include name="${final.name}/**" />
 				</tarfileset>
 				<tarfileset dir="${build.dir}" mode="755">
 					<include name="${final.name}/bin/*" />
+					<include name="${final.name}/tools/**" />
 				</tarfileset>
 			</param.listofitems>
 		</macro_tar>
 	</target>
 
+        <target name="rpm" depends="clean,tar" description="Make release rpm">
+		<mkdir dir="${build.dir}/BUILD"/>
+		<mkdir dir="${build.dir}/RPMS"/>
+		<mkdir dir="${build.dir}/SOURCES"/>
+		<mkdir dir="${build.dir}/SPECS"/>
+		<copy todir="${build.dir}/SOURCES">
+			<fileset dir="${build.dir}">
+			        <include name="${final.name}.tar.gz" />
+                        </fileset>
+                </copy>
+                <echo file="${build.dir}/SPECS/chukwa.spec" append="false">
+# RPM Spec file for Chukwa v.${chukwaVersion}
+
+%define _topdir         ${build.dir}
+%define _prefix         ${rpm.prefix}
+%define _conf           ${hadoop.conf}
+%define name            chukwa
+%define summary         Distributed Computing Monitoring Framework.
+%define version         ${chukwaVersion}
+%define release         1
+%define license         ASF 2.0
+%define group           Development/Monitoring
+%define source          %{name}-%{version}.tar.gz
+%define vendor          Apache Software Fundation
+%define packager        Eric Yang
+%define buildroot       %{_topdir}/BUILD
+
+Name:      %{name}
+Version:   %{version}
+Release:   %{release}
+Packager:  %{packager}
+Vendor:    %{vendor}
+License:   %{license}
+Summary:   %{summary}
+Group:     %{group}
+Source0:   %{source}
+Prefix:    %{_prefix}
+Buildroot: %{buildroot}
+
+%description
+Chukwa is the monitoring framework for large scale distributed
+clusters.
+
+%prep
+%setup -q
+%build
+mkdir -p %{buildroot}%{_prefix}
+if [ -d %{buildroot}%{_prefix}/%{name} ]; then
+    rm -rf %{buildroot}%{_prefix}/%{name}
+fi 
+mv %{buildroot}/%{name}-%{version} %{buildroot}%{_prefix}/%{name}
+cd %{buildroot}%{_prefix}
+mkdir -p %{buildroot}/etc/init.d
+cp %{name}/tools/init.d/chukwa-data-processors %{buildroot}/etc/init.d
+cp %{name}/tools/init.d/chukwa-collector %{buildroot}/etc/init.d
+cp %{name}/tools/init.d/chukwa-agent %{buildroot}/etc/init.d
+cp %{name}/tools/init.d/chukwa-system-metrics %{buildroot}/etc/init.d
+chmod a+x %{buildroot}/etc/init.d/chukwa-*
+rm -rf %{buildroot}%{_prefix}/%{name}/src
+rm -rf %{buildroot}%{_prefix}/%{name}/build.xml
+%post
+mkdir -p %{_prefix}/%{name}
+chown -R gmon:users %{_prefix}/%{name}
+echo "Congratulation!  You have successfully installed Chukwa."
+echo ""
+echo "To collect Data             : /etc/init.d/chukwa-system-metrics start"
+echo "To run Chukwa Agent         : /etc/init.d/chukwa-agent start"
+echo "To run Chukwa Collector     : /etc/init.d/chukwa-collector start"
+echo "To run Chukwa Data Processor: /etc/init.d/chukwa-data-processors start"
+%preun
+/etc/init.d/chukwa-data-processors stop
+/etc/init.d/chukwa-system-metrics stop
+/etc/init.d/chukwa-agent stop
+/etc/init.d/chukwa-collector stop
+echo
+%postun
+%files
+%defattr(-,root,root)
+%{_prefix}/%{name}/*
+/etc/init.d/chukwa-data-processors
+/etc/init.d/chukwa-collector
+/etc/init.d/chukwa-agent
+/etc/init.d/chukwa-system-metrics
+                </echo>
+		<rpm specFile="chukwa.spec" topDir="${build.dir}" cleanBuildDir="true" failOnError="true"/>
+        </target>
+
 	<!-- the normal classpath -->
 	<path id="classpath">
 		<pathelement location="${build.classes}"/>
@@ -419,30 +833,4 @@
 		<pathelement location="${conf.dir}"/>
 	</path>
 
-	<!-- the hadoop classpath -->
-	<path id="chukwaHadoopClasspath">
-		<pathelement location="${hadoop.root}/build/classes"/>
-		<fileset dir="${lib.dir}">
-			<include name="**/*.jar" />
-			<exclude name="**/excluded/" />
-		</fileset>
-		<fileset dir="${hadoop.root}/lib">
-			<include name="**/*.jar" />
-			<exclude name="**/excluded/" />
-		</fileset>
-	</path>
-        <path id="chukwaTestClasspath">
-                <pathelement location="${hadoop.root}/build/classes"/>
-                <pathelement location="${build.classes}"/>
-                <pathelement location="${test.build.classes}"/>
-                <fileset dir="${lib.dir}">
-                        <include name="**/*.jar" />
-                        <exclude name="**/excluded/" />
-                </fileset>
-                <fileset dir="${hadoop.root}/lib">
-                        <include name="**/*.jar" />
-                        <exclude name="**/excluded/" />
-                </fileset>
-        </path>
-
 </project>

Added: hadoop/core/trunk/src/contrib/chukwa/conf/README
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/README?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/README (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/README Fri Dec  5 12:30:14 2008
@@ -0,0 +1,18 @@
+Check for an updated copy of this README at http://wiki.apache.org/hadoop/Chukwa_Configuration
+
+Chukwa comes with templates for all configuration files. These files are in the conf directory and are named <conf file name>.template. You should be able to just make a copy of each of these files, removing the ".template" suffix.
+
+'''Required configuration files'''
+
+ * conf/collectors - new line delimited list of collectors. This file is used by startup and shutdown scripts to determine where to run Chukwa  
+ * collectors and is also used by Chukwa agents to find collectors to send their data to.
+ * conf/chukwa-agents - a list of hosts in the cluster on which to run the Chukwa agent daemon.
+ * conf/chukwa-env.sh - environment variables required to run Chukwa.
+ * conf/chukwa-collectors-conf.xml - collector specific settings (e.g. port number to listen for agents on, which file system to store data in, etc.)
+ * conf/chukwa-agents-conf.xml - agent specific settings.
+
+'''Optional configuration files'''
+
+ * conf/alert.conf - a list of e-mail addresses to which chukwa alerts should be sent.
+ * conf/initial_adaptors - a list of adaptors to add to all agents at startup time.
+

Added: hadoop/core/trunk/src/contrib/chukwa/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/aggregator.sql?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/aggregator.sql (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/aggregator.sql Fri Dec  5 12:30:14 2008
@@ -0,0 +1,12 @@
+insert into [cluster_system_metrics] (select timestamp,[avg(system_metrics)] from [system_metrics] where timestamp between '[past_hour]' and '[now]' group by timestamp);
+insert into [dfs_throughput] (select timestamp,[avg(dfs_datanode)] from [dfs_datanode] where timestamp between '[past_hour]' and '[now]' group by timestamp);
+insert into [cluster_disk] (select a.timestamp,a.mount,a.used,a.available,a.used_percent from (select from_unixtime(unix_timestamp(timestamp)-unix_timestamp(timestamp)%60)as timestamp,mount,avg(used) as used,avg(available) as available,avg(used_percent) as used_percent from [disk] where timestamp between '[past_hour]' and '[now]' group by timestamp,mount) as a group by a.timestamp, a.mount);
+insert into [hod_job_digest] (select timestamp,d.hodid,d.userid,[avg(system_metrics)] from (select a.HodID,b.host as machine,a.userid,a.starttime,a.endtime from [HodJob] a join [hod_machine] b on (a.HodID = b.HodID) where endtime between '[past_hour]' and '[now]') as d,[system_metrics] where timestamp between d.starttime and d.endtime and host=d.machine group by hodid,timestamp);
+insert into [cluster_hadoop_rpc] (select timestamp,[avg(hadoop_rpc)] from [hadoop_rpc] where timestamp between '[past_hour]' and '[now]' group by timestamp);
+#insert into [cluster_hadoop_mapred] (select timestamp,[avg(hadoop_mapred_job)] from [hadoop_mapred_job] where timestamp between '[past_hour]' and '[now]' group by timestamp);
+insert into [user_util] (select timestamp, j.UserID as user, sum(j.NumOfMachines) as node_total, sum(cpu_idle_pcnt*j.NumOfMachines) as cpu_unused, sum((cpu_user_pcnt+cpu_system_pcnt)*j.NumOfMachines) as cpu_used, avg(cpu_user_pcnt+cpu_system_pcnt) as cpu_used_pcnt, sum((100-(sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines) as disk_unused, sum(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines) as disk_used, avg(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)) as disk_used_pcnt, sum((100-eth0_busy_pcnt)*j.NumOfMachines) as network_unused, sum(eth0_busy_pcnt*j.NumOfMachines) as network_used, avg(eth0_busy_pcnt) as network_used_pcnt, sum((100-mem_used_pcnt)*j.NumOfMachines) as memory_unused, sum(mem_used_pcnt*j.NumOfMachines) as memory_used, avg(mem_used_pcnt) as memory_used_pcnt from [hod_job_digest] d,[HodJob] j where (d.HodID = j.HodID) and Timestamp between '[past_hour]' and '[now]' group by j.UserID);
+#insert into [node_util] select starttime, avg(unused) as unused, avg(used) as used from (select DATE_FORMAT(m.LAUNCH_TIME,'%Y-%m-%d %H:%i:%s') as starttime,sum(AvgCPUBusy*j.NumOfMachines/(60*100)) as unused,sum((100-AvgCPUBusy)*j.NumOfMachines/(60*100)) as used from HodJobDigest d join HodJob j on (d.HodID = j.HodID) join MRJob m on (m.HodID = j.HodID) where m.LAUNCH_TIME >= '2008-09-12 21:11' and m.LAUNCH_TIME <= '2008-09-12 22:11' and d.Timestamp >= m.LAUNCH_TIME and d.Timestamp <= m.FINISH_TIME group by m.MRJobID order by m.LAUNCH_TIME) as t group by t.starttime 
+#insert into [jobtype_util] select CASE WHEN MRJobName like 'PigLatin%' THEN 'Pig' WHEN MRJobName like 'streamjob%' THEN 'Streaming' WHEN MRJobName like '%abacus%' THEN 'Abacus' ELSE 'Other' END as m, count(*)*j.NumOfMachines/60 as nodehours,count(distinct(MRJobID)) as jobs from HodJobDigest d join HodJob j on (d.HodID = j.HodID) join MRJob m on (m.HodID = j.HodID) where d.Timestamp >= '2008-09-12 21:11' and d.Timestamp <= '2008-09-12 22:11' and d.Timestamp >= m.LAUNCH_TIME and d.Timestamp <= m.FINISH_TIME group by CASE WHEN MRJobName like 'PigLatin%' THEN 'Pig' WHEN MRJobName like 'streamjob%' THEN 'Streaming' WHEN MRJobName like '%abacus%' THEN 'Abacus' ELSE 'Other' END order by CASE WHEN MRJobName like 'PigLatin%' THEN 'Pig' WHEN MRJobName like 'streamjob%' THEN 'Streaming' WHEN MRJobName like '%abacus%' THEN 'Abacus' ELSE 'Other' END
+#insert into [a] select d.Timestamp as starttime,((AvgCPUBusy * j.NumOfMachines) / (sum(j.NumOfMachines) * 1)) as used from Digest d join HodJob j on (d.HodID = j.HodID) where d.Timestamp >= '[past_hour]' and d.Timestamp <= '[now]' group by d.Timestamp order by d.Timestamp 
+#insert into [b] select m, sum(foo.nodehours) as nodehours from (select m.MRJobID, round(avg(if(AvgCPUBusy is null,0,AvgCPUBusy)),0) as m, count(*)*j.NumOfMachines/60 as nodehours from HodJobDigest d join HodJob j on (d.HodID = j.HodID) join MRJob m on (m.HodID = j.HodID) where d.Timestamp >= '[past_hour]' and d.Timestamp <= '[now]' and d.Timestamp >= m.LAUNCH_TIME and d.Timestamp <= m.FINISH_TIME group by m.MRJobID) as foo group by m; 
+#insert into [c] select if(AvgCPUBusy is null,0,AvgCPUBusy) as m, CASE WHEN MRJobName like 'PigLatin%' THEN 'Pig' WHEN MRJobName like 'streamjob%' THEN 'Streaming' WHEN MRJobName like '%abacus%' THEN 'Abacus' ELSE 'Other' END as interface, count(*)*j.NumOfMachines/60 as nodehours,count(distinct(MRJobID)) as jobs from HodJobDigest d join HodJob j on (d.HodID = j.HodID) join MRJob m on (m.HodID = j.HodID) where d.Timestamp >= '[past_hour]' and d.Timestamp <= '[now]' and d.Timestamp >= m.LAUNCH_TIME and d.Timestamp <= m.FINISH_TIME group by AvgCPUBusy,CASE WHEN MRJobName like 'PigLatin%' THEN 'Pig' WHEN MRJobName like 'streamjob%' THEN 'Streaming' WHEN MRJobName like '%abacus%' THEN 'Abacus' ELSE 'Other' END order by if(AvgCPUBusy is null,0,AvgCPUBusy)

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template Fri Dec  5 12:30:14 2008
@@ -5,6 +5,13 @@
 
 <configuration>
   
+  
+  <property>
+    <name>chukwaAgent.tags</name>
+    <value>cluster="demo"</value>
+    <description>The cluster's name for this agent</description>
+  </property>
+  
   <property>
     <name>chukwaAgent.control.port</name>
     <value>9093</value>
@@ -18,12 +25,6 @@
   </property>
 
   <property>
-    <name>chukwaAgent.tags</name>
-    <value>cluster="demo"</value>
-    <description>The cluster's name for this agent</description>
-  </property>
-
-  <property>
     <name>chukwaAgent.checkpoint.name</name>
     <value>chukwa_agent_checkpoint</value>
     <description>the prefix to to prepend to the agent's checkpoint file(s)</description>
@@ -31,7 +32,7 @@
   
   <property>
     <name>chukwaAgent.checkpoint.dir</name>
-    <value>/tmp/</value>
+    <value>${CHUKWA_HOME}/var/tmp/</value>
     <description>the location to put the agent's checkpoint file(s)</description>
   </property>
 

Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agents.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agents.template?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agents.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agents.template Fri Dec  5 12:30:14 2008
@@ -0,0 +1 @@
+localhost

Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-demux-conf.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-demux-conf.xml?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-demux-conf.xml (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-demux-conf.xml Fri Dec  5 12:30:14 2008
@@ -0,0 +1,107 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+
+  <property>
+    <name>chukwaArchiveBuilder.reduceCount</name>
+    <value>5</value>
+    <description>Reduce count </description>
+  </property>
+
+  
+  <property>
+    <name>SysLog</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.SysLog</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>Df</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Df</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>HadoopLog</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopLogProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>HadoopMetricsProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HadoopMetricsProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+  <property>
+    <name>Iostat</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Iostat</value>
+    <description>Parser class for </description>
+  </property>
+ 
+   <property>
+    <name>Log4jJobHistoryProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4jJobHistoryProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+   <property>
+    <name>JobLogHistoryProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.JobLogHistoryProcessor</value>
+    <description>Parser class for </description>
+  </property>
+
+ 
+   <property>
+    <name>PbsNodes</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.PbsNodes</value>
+    <description>Parser class for </description>
+  </property>
+ 
+   <property>
+    <name>Sar</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Sar</value>
+    <description>Parser class for </description>
+  </property>
+
+   <property>
+    <name>TsProcessor</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.TsProcessor</value>
+    <description>Parser class for </description>
+   </property>
+  
+   <property>
+    <name>Top</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Top</value>
+    <description>Parser class for </description>
+   </property>
+
+   <property>
+    <name>Torque</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Torque</value>
+    <description>Parser class for Parsing qstat and tracejob</description>
+   </property>
+  
+   <property>
+    <name>YWatch</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.YWatch</value>
+    <description>Parser class for </description>
+   </property>
+  
+   <property>
+    <name>DbLoader</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.TsProcessor</value>
+    <description>Parser class for </description>
+   </property>
+
+   <property>
+    <name>JobConf</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.JobConfProcessor</value>
+    <description>Parser class for Map reduce Job Configuration</description>
+   </property>
+    
+</configuration>

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template Fri Dec  5 12:30:14 2008
@@ -6,35 +6,61 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-export JAVA_HOME=/usr/lib/j2sdk1.5-sun
+#export JAVA_HOME=/usr/lib/j2sdk1.5-sun
 
-# The location of the Hadoop the collector should use. Default 
-# assumes that this chukwa is living in hadoop's src/contrib directory
-export HADOOP_HOME="/usr/lib/hadoop/current"
+# Optional (.ie. will try sensible defaults)
+# The location of the Hadoop the collector should use. Chukwa uses this to 
+# find the hadoop classes or jars as well as the hadoop executables
+# for running the mapreduce demux job. Chukwa will look first in
+# HADOOP_HOME/build for a hadoop-*-core.jar file, 
+# then in HADOOP_HOME/build/classes for hadoop classes.
+# If you want to use a hadoop jar straight up, without
+# a HADOOP_HOME/bin directory, you should set HADOOP_JAR (below), but
+# then chukwa mapreduce jobs wont work. If HADOOP_HOME is not set, default
+# assumes that this chukwa is living in hadoop's src/contrib directory, but
+# if jars or classes can not be found using that default assumption, 
+# Chukwa will default to the hadoop jars that come with it
+# in the hadoopjars directory. 
+#export HADOOP_HOME="/home/user/Development/hadoop-trunk"
+
+# Optional (i.e. will try sensible defaults)
+# The location of a hadoop jars. Setting this will overrided the search for 
+# hadoop jars described above (using HADOOP_HOME). If HADOOP_JAR is not set, 
+# the default is to check HADOOP_HOME/build (above) for 
+# jars or classes, if those are not found, uses hadoop jars which
+# come with chukwa in $CHUKWA_HOME/hadoopjars. YOU SHOULD ONLY NEED TO 
+# USE THIS IF YOU ARE RUNNING A COLLECTOR WITHOUT A RUNNING 
+# HDFS! (i.e. writing datasink sequence files to local disk). Be careful 
+# if you use HDFS for chukwa storage, but haven't built the 
+# hadoop classes or jar because chukwa will use the default hadoop jars 
+# that come with ckukwa, you can easily suffer
+# errors due to protocol mismatch between hadoop versions.
+#export HADOOP_JAR=${HADOOP_HOME}/build/hadoop-*-core.jar
+
+# The location of chukwa data repository (in either HDFS or your local
+# file system, whichever you are using)
+export chukwaRecordsRepository="/chukwa/repos/"
 
 # The directory where pid files are stored. CHUKWA_HOME/var/run by default.
-#export CHUKWA_PID_DIR=
+#export CHUKWA_PID_DIR="/tmp/chukwa-pid-dir"
 
 # The location of chukwa logs, defaults to CHUKWA_HOME/logs
-export CHUKWA_LOG_DIR=${CHUKWA_HOME}/logs
-
-# The location of a hadoop jars. use this if you are running a collector
-# without a running HDFS (i.e. which writes sequence files to local disk)
-# if this is not set, the default is to check HADOOP_HOME for jars or
-# classes, if those are not found, uses hadoop jars which come with chukwa
-export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
-
-# The location of chukwa data repository
-export chuwaRecordsRepository="/chukwa/repos/"
+#export CHUKWA_LOG_DIR="/tmp/chukwa-log-dir"
 
 # The location of torque pbsnodes command
-export nodeActivityCmde="/usr/lib/torque/current/bin/pbsnodes "
+#export nodeActivityCmde="
 
 # The server which contain pbsnodes, qstat and tracejob.
-export TORQUE_SERVER=localhost
+#export TORQUE_SERVER=localhost
 
 # The location contain torque binaries.
-export TORQUE_HOME=/usr/lib/torque
+#export TORQUE_HOME=/usr/local/torque
+
+# Instance name for chukwa deployment
+export CHUKWA_IDENT_STRING=demo
+
+# Datatbase driver name for storing Chukwa Data.
+# export JDBC_DRIVER=
 
-# The domain of the cluster
-#export DOMAIN=
+# Database URL prefix for Database Loader.
+# export JDBC_URL_PREFIX=jdbc://

Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-hadoop-metrics-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-hadoop-metrics-log4j.properties?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-hadoop-metrics-log4j.properties (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-hadoop-metrics-log4j.properties Fri Dec  5 12:30:14 2008
@@ -0,0 +1,31 @@
+log4j.appender.chukwa.rpc.recordType=HadoopMetricsProcessor
+log4j.appender.chukwa.rpc.chukwaClientHostname=localhost
+log4j.appender.chukwa.rpc.chukwaClientPortNum=9093
+log4j.appender.chukwa.rpc.DatePattern=.yyyy-MM-dd
+log4j.appender.chukwa.rpc.layout=org.apache.log4j.PatternLayout
+log4j.appender.chukwa.rpc.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.chukwa.rpc.Dir=/tmp
+
+log4j.appender.chukwa.jvm.recordType=HadoopMetricsProcessor
+log4j.appender.chukwa.jvm.chukwaClientHostname=localhost
+log4j.appender.chukwa.jvm.chukwaClientPortNum=9093
+log4j.appender.chukwa.jvm.DatePattern=.yyyy-MM-dd
+log4j.appender.chukwa.jvm.layout=org.apache.log4j.PatternLayout
+log4j.appender.chukwa.jvm.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.chukwa.jvm.Dir=/tmp
+
+log4j.appender.chukwa.dfs.recordType=HadoopMetricsProcessor
+log4j.appender.chukwa.dfs.chukwaClientHostname=localhost
+log4j.appender.chukwa.dfs.chukwaClientPortNum=9093
+log4j.appender.chukwa.dfs.DatePattern=.yyyy-MM-dd
+log4j.appender.chukwa.dfs.layout=org.apache.log4j.PatternLayout
+log4j.appender.chukwa.dfs.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.chukwa.dfs.Dir=/tmp
+
+log4j.appender.chukwa.mapred.recordType=HadoopMetricsProcessor
+log4j.appender.chukwa.mapred.chukwaClientHostname=localhost
+log4j.appender.chukwa.mapred.chukwaClientPortNum=9093
+log4j.appender.chukwa.mapred.DatePattern=.yyyy-MM-dd
+log4j.appender.chukwa.mapred.layout=org.apache.log4j.PatternLayout
+log4j.appender.chukwa.mapred.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.chukwa.mapred.Dir=/tmp
\ No newline at end of file

Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-log4j.properties?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-log4j.properties (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-log4j.properties Fri Dec  5 12:30:14 2008
@@ -0,0 +1,19 @@
+log4j.rootLogger=INFO, R 
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.File=${CHUKWA_LOG_DIR}/${APP}.log
+log4j.appender.R.MaxFileSize=10MB
+log4j.appender.R.MaxBackupIndex=10
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/collectors.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/collectors.template?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/collectors.template (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/collectors.template Fri Dec  5 12:30:14 2008
@@ -1 +1 @@
-http://localhost:8080/
+localhost

Added: hadoop/core/trunk/src/contrib/chukwa/conf/database_create_tables
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/database_create_tables?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/database_create_tables (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/database_create_tables Fri Dec  5 12:30:14 2008
@@ -0,0 +1,602 @@
+create table if not exists node_activity_template 
+(
+    timestamp  timestamp default CURRENT_TIMESTAMP,
+    used int(11) default NULL,
+    usedMachines text,
+    free int(11) default NULL,
+    freeMachines text,
+    down int(11) default NULL,
+    downMachines text,
+    primary key(timestamp),
+    index (Timestamp)
+);
+
+create table if not exists switch_data_template (
+    timestamp timestamp default CURRENT_TIMESTAMP,
+    host varchar(40),
+    port varchar(10),
+    poller varchar(40),
+    metricName varchar(20),
+    value double,
+    primary key(timestamp, host, port),
+    index (Timestamp)
+);
+
+create table if not exists system_metrics_template (
+    timestamp  timestamp default CURRENT_TIMESTAMP,
+    host varchar(40),
+    load_15 double, 
+    load_5 double,
+    load_1 double,
+    task_total double,
+    task_running double,
+    task_sleep double,
+    task_stopped double,
+    task_zombie double,
+    mem_total double,
+    mem_buffers double,
+    mem_cached double,
+    mem_used double,
+    mem_free double,
+    eth0_rxerrs double,
+    eth0_rxbyts double,
+    eth0_rxpcks double,
+    eth0_rxdrops double,
+    eth0_txerrs double,
+    eth0_txbyts double,
+    eth0_txpcks double,
+    eth0_txdrops double,
+    eth1_rxerrs double,
+    eth1_rxbyts double,
+    eth1_rxpcks double,
+    eth1_rxdrops double,
+    eth1_txerrs double,
+    eth1_txbyts double,
+    eth1_txpcks double,
+    eth1_txdrops double,
+    sda_rkbs double,
+    sda_wkbs double,
+    sdb_rkbs double,
+    sdb_wkbs double,
+    sdc_rkbs double,
+    sdc_wkbs double,
+    sdd_rkbs double,
+    sdd_wkbs double,
+    cpu_idle_pcnt float,
+    cpu_nice_pcnt float,
+    cpu_system_pcnt float,
+    cpu_user_pcnt float,
+    cpu_hirq_pcnt float,
+    cpu_sirq_pcnt float,
+    iowait_pcnt float,
+    mem_buffers_pcnt float,
+    mem_used_pcnt float,
+    eth0_busy_pcnt float,
+    eth1_busy_pcnt float,
+    sda_busy_pcnt float,
+    sdb_busy_pcnt float,
+    sdc_busy_pcnt float,
+    sdd_busy_pcnt float,
+    swap_used_pcnt float,
+    primary key(host, timestamp),
+    index (timestamp)
+);
+
+create table if not exists disk_template (
+    timestamp  timestamp default CURRENT_TIMESTAMP,
+    host varchar(40),
+    mount varchar(40),
+    used double,
+    available double,
+    used_percent double,
+    fs varchar(40),
+    primary key(timestamp,host,mount),
+    index (timestamp)
+);
+
+create table if not exists cluster_disk_template (
+    timestamp  timestamp default CURRENT_TIMESTAMP,
+    mount varchar(40),
+    used double,
+    available double,
+    used_percent double,
+    primary key(timestamp,mount),
+    index (timestamp)
+);
+
+create table if not exists cluster_system_metrics_template (
+    timestamp  timestamp default CURRENT_TIMESTAMP,
+    host int,
+    load_15 double, 
+    load_5 double,
+    load_1 double,
+    task_total double,
+    task_running double,
+    task_sleep double,
+    task_stopped double,
+    task_zombie double,
+    mem_total double,
+    mem_buffers double,
+    mem_cached double,
+    mem_used double,
+    mem_free double,
+    eth0_rxerrs double,
+    eth0_rxbyts double,
+    eth0_rxpcks double,
+    eth0_rxdrops double,
+    eth0_txerrs double,
+    eth0_txbyts double,
+    eth0_txpcks double,
+    eth0_txdrops double,
+    eth1_rxerrs double,
+    eth1_rxbyts double,
+    eth1_rxpcks double,
+    eth1_rxdrops double,
+    eth1_txerrs double,
+    eth1_txbyts double,
+    eth1_txpcks double,
+    eth1_txdrops double,
+    sda_rkbs double,
+    sda_wkbs double,
+    sdb_rkbs double,
+    sdb_wkbs double,
+    sdc_rkbs double,
+    sdc_wkbs double,
+    sdd_rkbs double,
+    sdd_wkbs double,
+    cpu_idle_pcnt float,
+    cpu_nice_pcnt float,
+    cpu_system_pcnt float,
+    cpu_user_pcnt float,
+    cpu_hirq_pcnt float,
+    cpu_sirq_pcnt float,
+    iowait_pcnt float,
+    mem_buffers_pcnt float,
+    mem_used_pcnt float,
+    eth0_busy_pcnt float,
+    eth1_busy_pcnt float,
+    sda_busy_pcnt float,
+    sdb_busy_pcnt float,
+    sdc_busy_pcnt float,
+    sdd_busy_pcnt float,
+    swap_used_pcnt float,
+    primary key(host, timestamp),
+    index (timestamp)
+);
+
+create table if not exists dfs_namenode_template (
+    timestamp timestamp default 0,
+    host varchar(80),
+    add_block_ops double,
+    blocks_corrupted double,
+    create_file_ops double,
+    delete_file_ops double,
+    files_created double,
+    files_renamed double,
+    files_deleted double,
+    get_block_locations double,
+    get_listing_ops double,
+    safe_mode_time double,
+    syncs_avg_time double,
+    syncs_num_ops double,
+    transactions_avg_time double,
+    transactions_num_ops double,
+    block_report_avg_time double,
+    block_report_num_ops double,
+    fs_image_load_time double,
+    primary key(timestamp, host),
+    index(timeStamp)
+);
+
+create table if not exists dfs_datanode_template (
+    timestamp timestamp default 0,
+    host varchar(80),
+    block_reports_avg_time double,
+    block_reports_num_ops double,
+    block_verification_failures double,
+    blocks_read double,
+    blocks_removed double,
+    blocks_replicated double,
+    blocks_verified double,
+    blocks_written double,
+    bytes_read double,
+    bytes_written double,
+    copy_block_op_avg_time double,
+    copy_block_op_num_ops double,
+    heart_beats_avg_time double,
+    heart_beats_num_ops double,
+    read_block_op_avg_time double,
+    read_block_op_num_ops double,
+    read_metadata_op_avg_time double,
+    read_metadata_op_num_ops double,
+    reads_from_local_client double,
+    reads_from_remote_client double,
+    replace_block_op_avg_time double,
+    replace_block_op_num_ops double,
+    session_id double,
+    write_block_op_avg_time double,
+    write_block_op_num_ops double,
+    writes_from_local_client double,
+    writes_from_remote_client double,
+    primary key(timestamp, host),
+    index(timestamp)
+);
+
+create table if not exists dfs_fsnamesystem_template (
+    timestamp timestamp default 0,
+    host VARCHAR(80),
+    blocks_total double,
+    capacity_remaining_gb double,
+    capacity_total_gb double,
+    capacity_used_gb double,
+    files_total double,
+    pending_replication_blocks double,
+    scheduled_replication_blocks double,
+    total_load double,
+    under_replicated_blocks double,
+    primary key(timestamp, host),
+    index(timestamp)
+);
+
+create table if not exists dfs_throughput_template (
+    timestamp timestamp default 0,
+    host int,
+    block_reports_avg_time double,
+    block_reports_num_ops double,
+    block_verification_failures double,
+    blocks_read double,
+    blocks_removed double,
+    blocks_replicated double,
+    blocks_verified double,
+    blocks_written double,
+    bytes_read double,
+    bytes_written double,
+    copy_block_op_avg_time double,
+    copy_block_op_num_ops double,
+    heart_beats_avg_time double,
+    heart_beats_num_ops double,
+    read_block_op_avg_time double,
+    read_block_op_num_ops double,
+    read_metadata_op_avg_time double,
+    read_metadata_op_num_ops double,
+    reads_from_local_client double,
+    reads_from_remote_client double,
+    replace_block_op_avg_time double,
+    replace_block_op_num_ops double,
+    session_id double,
+    write_block_op_avg_time double,
+    write_block_op_num_ops double,
+    writes_from_local_client double,
+    writes_from_remote_client double,
+    primary key(timestamp),
+    index(timestamp)
+);
+
+create table if not exists hadoop_jvm_template (
+    timestamp timestamp default 0,
+    host VARCHAR(80),
+    process_name VARCHAR(80),
+    gc_timemillis double,
+    gc_count double,
+    log_error double,
+    log_fatal double,
+    log_info double,
+    log_warn double,
+    mem_heap_committed_m double,
+    mem_heap_used_m double,
+    mem_non_heap_committed_m double,
+    mem_non_heap_used_m double,
+    threads_blocked double,
+    threads_new double,
+    threads_runnable double,
+    threads_terminated double,
+    threads_timed_waiting double,
+    threads_waiting double,
+    primary key (timestamp, host, process_name),
+    index(timestamp)
+);
+
+create table if not exists hadoop_mapred_template (
+    timestamp timestamp default 0,
+    host VARCHAR(80),
+    jobs_completed double,
+    jobs_submitted double,
+    maps_completed double,
+    maps_launched double,
+    reduces_completed double,
+    reduces_launched double,
+    primary key (timestamp, host),
+    index(timestamp)
+);
+
+create table if not exists hadoop_rpc_template (
+    timestamp timestamp default 0,
+    host VARCHAR(80),
+    rpc_processing_time_avg_time double,
+    rpc_processing_time_num_ops double,
+    rpc_queue_time_avg_time double,
+    rpc_queue_time_num_ops double,
+    get_build_version_avg_time double,
+    get_build_version_num_ops double,
+    get_job_counters_avg_time double,
+    get_job_counters_num_ops double,
+    get_job_profile_avg_time double,
+    get_job_profile_num_ops double,
+    get_job_status_avg_time double,
+    get_job_status_num_ops double,
+    get_new_job_id_avg_time double,
+    get_new_job_id_num_ops double,
+    get_protocol_version_avg_time double,
+    get_protocol_version_num_ops double,
+    get_system_dir_avg_time double,
+    get_system_dir_num_ops double,
+    get_task_completion_events_avg_time double,
+    get_task_completion_events_num_ops double,
+    get_task_diagnostics_avg_time double,
+    get_task_diagnostics_num_ops double,
+    heartbeat_avg_time double,
+    heartbeat_num_ops double,
+    killJob_avg_time double,
+    killJob_num_ops double,
+    submit_job_avg_time double,
+    submit_job_num_ops double,
+    primary key (timestamp, host),
+    index(timestamp)
+);
+
+create table if not exists cluster_hadoop_rpc_template (
+    timestamp timestamp default 0,
+    host int,
+    rpc_processing_time_avg_time double,
+    rpc_processing_time_num_ops double,
+    rpc_queue_time_avg_time double,
+    rpc_queue_time_num_ops double,
+    get_build_version_avg_time double,
+    get_build_version_num_ops double,
+    get_job_counters_avg_time double,
+    get_job_counters_num_ops double,
+    get_job_profile_avg_time double,
+    get_job_profile_num_ops double,
+    get_job_status_avg_time double,
+    get_job_status_num_ops double,
+    get_new_job_id_avg_time double,
+    get_new_job_id_num_ops double,
+    get_protocol_version_avg_time double,
+    get_protocol_version_num_ops double,
+    get_system_dir_avg_time double,
+    get_system_dir_num_ops double,
+    get_task_completion_events_avg_time double,
+    get_task_completion_events_num_ops double,
+    get_task_diagnostics_avg_time double,
+    get_task_diagnostics_num_ops double,
+    heartbeat_avg_time double,
+    heartbeat_num_ops double,
+    killJob_avg_time double,
+    killJob_num_ops double,
+    submit_job_avg_time double,
+    submit_job_num_ops double,
+    primary key (timestamp),
+    index(timestamp)
+);
+
+create table if not exists hadoop_rpc_calls_template (
+    timestamp timestamp default 0,
+    method varchar(80),
+    calls double,
+    primary key(timestamp, method),
+    index(timestamp)
+);
+
+create table if not exists mssrgraph_template (
+    timestamp timestamp default 0,
+    job_id VARCHAR(80),
+    type VARCHAR(10),
+    count double,
+    primary key (timestamp, job_id),
+    index(timestamp)
+);
+
+create table if not exists mr_job(
+    HodID varchar(20),
+    MRJobID varchar(80),
+    MRJobName varchar(100),
+    STATUS varchar(10),
+    SUBMIT_TIME timestamp default 0,
+    LAUNCH_TIME timestamp default 0,
+    FINISH_TIME timestamp default 0,
+    MAPPER_PHASE_END_TIME timestamp default 0,
+    TOTAL_MAPS int unsigned,
+    TOTAL_REDUCES int unsigned,
+    FINISHED_MAPS  int unsigned,
+    FINISHED_REDUCES  int unsigned,      
+    NumOfLocalSplits int unsigned,
+    NumOfRackLocalSplits int unsigned,
+    NUM_OF_MAPPER_ATTEMPTS int unsigned,
+    NUM_OF_REDUCER_ATTEMPTS int unsigned,
+    MAPPER_PHASE_EXECUTION_TIME int,
+    AvgMapperExecutionTime int unsigned,
+    AvgLocalMapperExecutionTime int unsigned,
+    AvgRackLocalMapperExecutionTime int unsigned,
+    AvgRemoteMapperExecutionTime int unsigned,
+    AvgReducerExecutionTime int unsigned,
+    AvgShuffleExecutionTime int unsigned,
+    AvgSortExecutionTime int unsigned,
+    MapperClass varchar(80),
+    ReducerClass varchar(80),
+    PartitionerClass varchar(80),
+    CombinerClass varchar(80),
+    InputFormatClass varchar(80),
+    InputKeyClass varchar(80),
+    InputValueClass varchar(80),
+    OutputFormatClass varchar(80),
+    OutputCompressed tinyint,
+    OutputCompressionType  varchar(20),
+    OutputCompressionCodec varchar(20),
+    OutputKeyClass varchar(80),
+    OutputValueClass varchar(80),
+    MapOutputKeyClass varchar(80),
+    MapOutputValueClass varchar(80),
+    MapOutputCompressed tinyint,
+    MapOutputCompressionType  varchar(20),
+    MapOutputCompressionCodec varchar(20),
+    InputDataSizeInMB int unsigned,
+    MapOutputSizeInMB int unsigned,
+    OutputDataSizeInMB int unsigned,
+    MR_JOBCONF  text, 
+    InputDir  text, 
+    primary key(SUBMIT_TIME, HodID, MRJobID),
+    index(SUBMIT_TIME)
+);
+
+create table if not exists mr_job_counters_template (
+    timestamp timestamp default 0,
+    job_id BIGINT,
+    hdfs_bytes_read double,
+    hdfs_bytes_written double,
+    local_bytes_read double,
+    local_bytes_written double,
+    data_local_map_tasks double,
+    launched_map_tasks double,
+    launched_reduce_tasks double,
+    combine_input_records double,
+    combine_output_records double,
+    map_input_bytes double,
+    map_output_bytes double,
+    map_input_records double,
+    map_output_records double,
+    reduce_input_groups double,
+    reduce_input_records double,
+    reduce_output_records double,
+    primary key (timestamp, job_id),
+    index(timestamp)
+);
+
+create table if not exists mr_job_timeline_template (
+    timestamp timestamp default CURRENT_TIMESTAMP,
+    mr_job_id varchar(80),
+    task_type varchar(20),
+    tasks double,
+    time double,
+    primary key(timestamp, mr_job_id),
+    index(timestamp)
+);
+
+create table if not exists mr_finish_time_vs_size_template (
+    timestamp timestamp default CURRENT_TIMESTAMP,
+    mr_job_id varchar(80),
+    task_type varchar(20),
+    size double,
+    time double,
+    primary key(timestamp, mr_job_id),
+    index(timestamp)
+);
+    
+create table if not exists hod_machine_template (
+    hodid varchar(20) not null, 
+    host varchar(40) not null,
+    index(HodId)
+);
+
+create table if not exists HodJob_template (
+    HodID varchar(20), 
+    UserID varchar(20), 
+    Status  smallint,
+    JobTracker varchar(40), 
+    TimeQueued mediumint unsigned,
+    StartTime timestamp default CURRENT_TIMESTAMP, 
+    EndTime timestamp default 0,  
+    NumOfMachines smallint unsigned,  
+    SlotLimitPerTracker smallint unsigned,
+    LogProcessStatus varchar(20),
+    primary key(HodId),
+    index(StartTime, EndTime)
+);
+
+create table if not exists hod_job_digest_template (
+    timestamp timestamp default 0,
+    HodID VARCHAR(20),
+    UserID VARCHAR(20),
+    host int,
+    load_15 double, 
+    load_5 double,
+    load_1 double,
+    task_total double,
+    task_running double,
+    task_sleep double,
+    task_stopped double,
+    task_zombie double,
+    mem_total double,
+    mem_buffers double,
+    mem_cached double,
+    mem_used double,
+    mem_free double,
+    eth0_rxerrs double,
+    eth0_rxbyts double,
+    eth0_rxpcks double,
+    eth0_rxdrops double,
+    eth0_txerrs double,
+    eth0_txbyts double,
+    eth0_txpcks double,
+    eth0_txdrops double,
+    eth1_rxerrs double,
+    eth1_rxbyts double,
+    eth1_rxpcks double,
+    eth1_rxdrops double,
+    eth1_txerrs double,
+    eth1_txbyts double,
+    eth1_txpcks double,
+    eth1_txdrops double,
+    sda_rkbs double,
+    sda_wkbs double,
+    sdb_rkbs double,
+    sdb_wkbs double,
+    sdc_rkbs double,
+    sdc_wkbs double,
+    sdd_rkbs double,
+    sdd_wkbs double,
+    cpu_idle_pcnt float,
+    cpu_nice_pcnt float,
+    cpu_system_pcnt float,
+    cpu_user_pcnt float,
+    cpu_hirq_pcnt float,
+    cpu_sirq_pcnt float,
+    iowait_pcnt float,
+    mem_buffers_pcnt float,
+    mem_used_pcnt float,
+    eth0_busy_pcnt float,
+    eth1_busy_pcnt float,
+    sda_busy_pcnt float,
+    sdb_busy_pcnt float,
+    sdc_busy_pcnt float,
+    sdd_busy_pcnt float,
+    swap_used_pcnt float,
+    primary key(HodId, timestamp),
+    index(timeStamp)
+); 
+
+create table if not exists user_util_template (
+    timestamp timestamp default CURRENT_TIMESTAMP,
+    user VARCHAR(20),
+    node_total int,
+    cpu_unused double,
+    cpu_used double,
+    cpu_used_pcnt float,
+    disk_unused double,
+    disk_used double,
+    disk_used_pcnt float,
+    network_unused double,
+    network_used double,
+    network_used_pcnt float,
+    memory_unused double,
+    memory_used double,
+    memory_used_pcnt float
+);
+
+create table if not exists QueueInfo(
+    Timestamp timestamp default 0,
+    HodID VARCHAR(20),
+    Queue VARCHAR(20),
+    NumOfMachine smallint unsigned,
+    status varchar(1),
+    index(TimeStamp)
+);

Added: hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-log4j.properties?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-log4j.properties (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-log4j.properties Fri Dec  5 12:30:14 2008
@@ -0,0 +1,111 @@
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+#log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+#
+# CHUKWA
+#
+log4j.appender.DRFA=org.apache.hadoop.chukwa.inputtools.log4j.ChukwaDailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+log4j.appender.DRFA.recordType=HadoopLog
+log4j.appender.DRFA.chukwaClientHostname=localhost
+log4j.appender.DRFA.chukwaClientPortNum=9093
+
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# AUDIT LOGGING - All audit events are logged at INFO level
+#
+# CHUKWA AUDIT LOG
+
+log4j.appender.DRFAAUDIT=org.apache.hadoop.chukwa.inputtools.log4j.ChukwaDailyRollingFileAppender
+log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/audit.log
+log4j.appender.DRFAAUDIT.recordType=HadoopLog
+log4j.appender.DRFAAUDIT.chukwaClientHostname=localhost
+log4j.appender.DRFAAUDIT.chukwaClientPortNum=9093
+log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd
+log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.logger.org.apache.hadoop.fs.FSNamesystem.audit=INFO,DRFAAUDIT
+log4j.additivity.org.apache.hadoop.fs.FSNamesystem.audit=false
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# Custom Logging levels
+
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
+

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties Fri Dec  5 12:30:14 2008
@@ -4,4 +4,8 @@
 jvm.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
 jvm.period=60
 
+mapred.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+mapred.period=60
 
+rpc.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+rpc.period=60

Added: hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf Fri Dec  5 12:30:14 2008
@@ -0,0 +1 @@
+unknown=jdbc:://localhost:3306/demo?user=example

Added: hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf.template?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf.template Fri Dec  5 12:30:14 2008
@@ -0,0 +1 @@
+unknown=jdbc:://localhost:3306/demo?user=example

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/log4j.properties?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/log4j.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/log4j.properties Fri Dec  5 12:30:14 2008
@@ -1,6 +1,6 @@
-log4j.rootLogger=INFO, stdout, R
+log4j.rootLogger=INFO, R
 log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/chukwa.log
+log4j.appender.R.File=${CHUKWA_LOG_DIR}/chukwa.log
 log4j.appender.R.MaxFileSize=10MB
 log4j.appender.R.MaxBackupIndex=10
 log4j.appender.R.layout=org.apache.log4j.PatternLayout
@@ -10,14 +10,3 @@
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
 
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=DEBUG, R
-
-# In our hadoop log4j.properties, replace the following line:
-# log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-# with this line:
-#log4j.appender.DRFA=org.apache.hadoop.chukwa.inputtools.log4j.DailyRollingFileAppender
-
-# Then add these lines as well:
-#log4j.appender.DRFA.recordType=hadoop_log
-#log4j.appender.DRFA.chukwaClientHostname=localhost
-#log4j.appender.DRFA.chukwaClientPortNum=9094



Mime
View raw message