hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r709533 [1/2] - in /hadoop/core/trunk: ./ src/contrib/chukwa/ src/contrib/chukwa/bin/ src/contrib/chukwa/conf/ src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/...
Date Fri, 31 Oct 2008 18:57:06 GMT
Author: omalley
Date: Fri Oct 31 11:57:04 2008
New Revision: 709533

URL: http://svn.apache.org/viewvc?rev=709533&view=rev
Log:
HADOOP-4433. Improve data loader for collecting metrics and log files.
(Eric Yang via omalley)

Added:
    hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt
    hadoop/core/trunk/src/contrib/chukwa/bin/chukwa
    hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template
    hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4JMetricsContext.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/ErStreamHandler.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueDataLoader.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueInfoProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueTimerTask.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/ExecHelper.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ClusterConfig.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/PidFile.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
    hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh
    hadoop/core/trunk/src/contrib/chukwa/build.xml
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
    hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/util.properties
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Log4JMetricsContext.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Oct 31 11:57:04 2008
@@ -46,9 +46,6 @@
     HADOOP-4284. Support filters that apply to all requests, or global filters,
     to HttpServer. (Kan Zhang via cdouglas)
     
-    HADOOP-4431. Add versionning/tags to Chukwa Chunk. 
-    (Jerome Boulon via Johan)
-
     HADOOP-4276. Improve the hashing functions and deserialization of the 
     mapred ID classes. (omalley)
 

Added: hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt (added)
+++ hadoop/core/trunk/src/contrib/chukwa/CHANGES.txt Fri Oct 31 11:57:04 2008
@@ -0,0 +1,25 @@
+Trunk (unreleased changes)
+
+  INCOMPATIBLE CHANGES
+
+  NEW FEATURES
+
+  IMPROVEMENTS
+
+    HADOOP-4431. Add versionning/tags to Chukwa Chunk. 
+    (Jerome Boulon via Johan)
+
+    HADOOP-4433. Improve data loader for collecting metrics and log files.
+    (Eric Yang via omalley)
+
+  OPTIMIZATIONS
+
+  BUG FIXES
+
+Release 0.19.0 - Unreleased
+
+  NEW FEATURES
+
+    HADOOP-3719. Initial checkin of Chukwa, which is a data collection and 
+    analysis framework. (Jerome Boulon, Andy Konwinski, Ari Rabkin, 
+    and Eric Yang)

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/VERSION?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/VERSION (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/VERSION Fri Oct 31 11:57:04 2008
@@ -1 +1 @@
-0.0.1
+0.1.0

Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa Fri Oct 31 11:57:04 2008
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The Chukwa command script
+#
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# get arguments
+COMMAND=$1
+shift
+
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+  . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+
+exec "${CHUKWA_HOME}/bin/$@"

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh Fri Oct 31 11:57:04 2008
@@ -36,69 +36,70 @@
 bin=`cd "$bin"; pwd`
 this="$bin/$script"
 
-. ${bin}/../conf/chukwa-env.sh
-
-export HADOOP_HOME="${HADOOP_HOME:-${bin}/../../../..}"
 
 # the root of the Chukwa installation
-if [ -z $CHUKWA_HOME ] ; then
-CHUKWA_HOME=`dirname "$this"`/..
-export CHUKWA_HOME=`cd $CHUKWA_HOME; pwd`
+export CHUKWA_HOME=`dirname "$this"`/..
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -gt 1 ]
+then
+    if [ "--config" = "$1" ]
+          then
+              shift
+              confdir=$1
+              shift
+              CHUKWA_CONF_DIR=$confdir
+    fi
 fi
 
-chukwaVersion=`cat ${CHUKWA_HOME}/bin/VERSION`
-DEFAULT_CHUKWA_HOME=${CHUKWA_HOME}/logs/
-export CHUKWA_LOG_DIR="${CHUKWA_LOG_DIR:-$DEFAULT_CHUKWA_HOME}"
-if [ ! -d $CHUKWA_LOG_DIR ]; then
-  mkdir -p $CHUKWA_LOG_DIR
+#check to see it is specified whether to use the slaves or the
+# masters file
+if [ $# -gt 1 ]
+then
+    if [ "--hosts" = "$1" ]
+    then
+        shift
+        slavesfile=$1
+        shift
+        export CHUKWA_SLAVES="${CHUKWA_CONF_DIR}/$slavesfile"
+    fi
 fi
 
-export chuwaRecordsRepository="/chukwa/repos/demo"
+#check to see if the conf dir is given as an optional argument
+if [ $# -gt 1 ]
+then
+    if [ "--watchdog" = "$1" ]
+          then
+              shift
+              WATCHDOG="true"
+    fi
+fi
 
-export DATACONFIG=${CHUKWA_HOME}/conf/mdl.xml
-common=`ls ${CHUKWA_HOME}/lib/*.jar`
-export common=`echo ${common} | sed 'y/ /:/'`
+export CHUKWA_LOG_DIR="$CHUKWA_HOME/logs"
 
-#chukwaCore=${HADOOP_HOME}/build/contrib/chukwa/chukwa-core-${chukwaVersion}.jar
-chukwaCore=${HADOOP_HOME}/build/contrib/chukwa
-if [ -a $chukwaCore ] ; then
-  export chukwaCore
-else
-  echo ${chukwaCore} does not exist
-  export chukwaCore=${CHUKWA_HOME}/chukwa-core-${chukwaVersion}.jar
-fi
+CHUKWA_VERSION=`cat ${CHUKWA_HOME}/bin/VERSION`
 
-#chukwaAgent=${HADOOP_HOME}/build/contrib/chukwa/chukwa-agent-${chukwaVersion}.jar
-chukwaAgent=${HADOOP_HOME}/build/contrib/chukwa
-if [ -a $chukwaAgent ] ; then
-  export chukwaAgent
-else
-  echo ${chukwaAgent} does not exist
-  export chukwaAgent=${CHUKWA_HOME}/chukwa-agent-${chukwaVersion}.jar
+# Allow alternate conf dir location.
+if [ -z "$CHUKWA_CONF_DIR" ]; then
+    CHUKWA_CONF_DIR="${CHUKWA_CONF_DIR:-$CHUKWA_HOME/conf}"
+    export CHUKWA_CONF_DIR=${CHUKWA_HOME}/conf
 fi
 
-echo chukwaCore is ${chukwaCore} and chukwaAgent is ${chukwaAgent}
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+  . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
 
+export DATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml
+COMMON=`ls ${CHUKWA_HOME}/lib/*.jar`
+export COMMON=`echo ${COMMON} | sed 'y/ /:/'`
+export CHUKWA_CORE=${CHUKWA_HOME}/chukwa-core-${CHUKWA_VERSION}.jar
+export CHUKWA_AGENT=${CHUKWA_HOME}/chukwa-agent-${CHUKWA_VERSION}.jar
+export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
 export CURRENT_DATE=`date +%Y%m%d%H%M`
-export TS_CONFIG=${CHUKWA_HOME}/conf/ts
-export tomcat=${CHUKWA_HOME}/opt/apache-tomcat-6.0.16
-if [ -d ${HADOOP_HOME}/build/classes ]; then
-  DEFAULT_HADOOP_JAR=${HADOOP_HOME}/build/classes
-# this doesn't work, but needs to be replaced with something that does
-#elif [ls ${HADOOP_HOME}/build/hadoop-*-core.jar` ]; then
-#  echo setting DEFAULT_HADOOP_JAR to `ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
-#  DEFAULT_HADOOP_JAR=`ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
-else
-  DEFAULT_HADOOP_JAR=${CHUKWA_HOME}/hadoopjars/hadoop-0.18.0-core.jar
-fi
-export HADOOP_JAR=${HADOOP_JAR:-$DEFAULT_HADOOP_JAR}
-
-echo
-echo HADOOP_JAR is $HADOOP_JAR
-echo
-
-export CHUKWA_LOG_DIR="${CHUKWA_HOME}/logs/"
-DEFAULT_PID_DIR=${CHUKWA_HOME}/var/run
-export CHUKWA_PID_DIR="${CHUKWA_PID_DIR:-$DEFAULT_PID_DIR}"
-export chuwaRecordsRepository="/chukwa/repos/demo"
+
+if [ -z "$JAVA_HOME" ] ; then
+    export JAVA_HOME=/usr/lib/j2sdk1.5-sun
+fi
+
+export JPS=${JAVA_HOME}/bin/jps
 

Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Runs a Chukwa command as a daemon.
+#
+# Environment Variables
+#
+#   CHUKWA_CONF_DIR  Alternate conf dir. Default is ${CHUKWA_HOME}/conf.
+#   CHUKWA_LOG_DIR   Where log files are stored.  PWD by default.
+#   CHUKWA_MASTER    host:path where chukwa code should be rsync'd from
+#   CHUKWA_PID_DIR   The pid files are stored. ${CHUKWA_HOME}/var/tmp by default.
+#   CHUKWA_IDENT_STRING   A string representing this instance of chukwa. $USER by default
+#   CHUKWA_NICENESS The scheduling priority for daemons. Defaults to 0.
+##
+
+usage="Usage: chukwa-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <chukwa-command> <args...>"
+
+# if no args specified, show usage
+if [ $# -le 1 ]; then
+  echo $usage
+  exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# get arguments
+startStop=$1
+shift
+command=$1
+shift
+
+chukwa_rotate_log ()
+{
+    log=$1;
+    num=5;
+    if [ -n "$2" ]; then
+	num=$2
+    fi
+    if [ -f "$log" ]; then # rotate logs
+	while [ $num -gt 1 ]; do
+	    prev=`expr $num - 1`
+	    [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
+	    num=$prev
+	done
+	mv "$log" "$log.$num";
+    fi
+}
+
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+  . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+
+# get log directory
+if [ "$CHUKWA_LOG_DIR" = "" ]; then
+  export CHUKWA_LOG_DIR="$CHUKWA_HOME/logs"
+fi
+mkdir -p "$CHUKWA_LOG_DIR"
+
+if [ "$CHUKWA_PID_DIR" = "" ]; then
+  CHUKWA_PID_DIR=$CHUKWA_HOME/var/run
+fi
+
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+  export CHUKWA_IDENT_STRING="$USER"
+fi
+
+# some variables
+export CHUKWA_LOGFILE=chukwa-$CHUKWA_IDENT_STRING-$command-$HOSTNAME.log
+export CHUKWA_ROOT_LOGGER="INFO,DRFA"
+log=$CHUKWA_LOG_DIR/chukwa-$CHUKWA_IDENT_STRING-$command-$HOSTNAME.out
+pid=$CHUKWA_PID_DIR/chukwa-$CHUKWA_IDENT_STRING-$command.pid
+
+# Set default scheduling priority
+if [ "$CHUKWA_NICENESS" = "" ]; then
+    export CHUKWA_NICENESS=0
+fi
+
+case $startStop in
+
+  (start)
+    MAIL=`cat ${CHUKWA_HOME}/conf/alert.conf`
+
+    if [ "${WATCHDOG}" != "" ]; then
+        mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
+        crontab -l > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+        crontest=$?
+
+        if [ "X${crontest}" != "X0" ]; then
+          cat > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+MAILTO=${MAIL}
+1,30 * * * * ${CHUKWA_HOME}/bin/watchdog.sh
+CRON
+        else
+          grep -v "${CHUKWA_HOME}/bin/watchdog.sh" ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} | grep -v MAILTO > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+          echo "MAILTO=${MAIL}" > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+          cat ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2 >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+          rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+          cat >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+1,30 * * * * ${CHUKWA_HOME}/bin/watchdog.sh
+CRON
+        fi
+
+        # save crontab
+        echo -n "Registering watchdog.."
+        mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
+        crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
+        rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+        echo "done"
+    fi
+
+    mkdir -p "$CHUKWA_PID_DIR"
+
+    if [ -f $pid ]; then
+      if kill -0 `cat $pid` > /dev/null 2>&1; then
+        echo $command running as process `cat $pid`.  Stop it first.
+        exit 1
+      fi
+    fi
+
+    if [ "$CHUKWA_MASTER" != "" ]; then
+      echo rsync from $CHUKWA_MASTER
+      rsync -a -e ssh --delete --exclude=.svn $CHUKWA_MASTER/ "$CHUKWA_HOME"
+    fi
+
+    chukwa_rotate_log $log
+    echo starting $command, logging to $log
+    cd "$CHUKWA_HOME"
+    nohup nice -n $CHUKWA_NICENESS "$CHUKWA_HOME"/bin/chukwa -config $command "$@" > "$log" 2>&1 < /dev/null &
+    echo $! > $pid
+    sleep 1; head "$log"
+    ;;
+          
+  (stop)
+
+    if [ "${WATCHDOG}" != "" ]; then
+        # remove watchdog
+        crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+        crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+        rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+    fi
+
+    if [ -f $CHUKWA_HOME/bin/$command ]; then
+      $CHUKWA_HOME/bin/$command stop
+      rm -f $pid
+    else
+      if [ -f $pid ]; then
+        if kill -0 `cat $pid` > /dev/null 2>&1; then
+          echo stopping $command
+          kill `cat $pid`
+          rm -f $pid
+        else
+          echo no $command to stop
+        fi
+      else
+        echo no $command to stop
+      fi
+    fi
+    ;;
+
+  (*)
+    echo $usage
+    exit 1
+    ;;
+
+esac
+
+

Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemons.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Run a Hadoop command on all slave hosts.
+
+usage="Usage: chukwa-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..."
+
+# if no args specified, show usage
+if [ $# -le 1 ]; then
+  echo $usage
+  exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. $bin/chukwa-config.sh
+
+if [ ${WATCHDOG}!="" ]; then
+  exec "$bin/slaves.sh" --config $CHUKWA_CONF_DIR cd "$CHUKWA_HOME" \; "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR --watchdog "$@"
+else
+  exec "$bin/slaves.sh" --config $CHUKWA_CONF_DIR cd "$CHUKWA_HOME" \; "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR "$@"
+fi

Added: hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+if [ "X$1" = "Xstop" ]; then
+  echo -n "Shutting down Node Activity Data Loader..."
+  if [ -f ${CHUKWA_HOME}/var/run/PbsNodes-data-loader.pid ]; then
+    kill -TERM `cat ${CHUKWA_HOME}/var/run/PbsNodes-data-loader.pid`
+  fi
+  echo "done"
+  exit 0
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/PbsNodes-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+    ${JAVA_HOME}/bin/java -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=PbsNodes -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec "${nodeActivityCmde}" &
+fi
+
+

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh Fri Oct 31 11:57:04 2008
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-java=$JAVA_HOME/bin/java
-
-. "$bin"/chukwa-config.sh
-
-# remove watchdog
-crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh > /tmp/cron.${CURRENT_DATE}
-crontab /tmp/cron.${CURRENT_DATE}
-rm -f /tmp/cron.${CURRENT_DATE}
-
-# stop torque data loader
-pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
-if [ -f $pidFile ]; then  
-   echo -n "Shutting down Torque Data Loader.."
-   torquepid=`head ${pidFile}`
-   kill -HUP ${torquepid}
-   # kill -HUP `ps eww |grep TorqueDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
-   rm ${pidFile}
-   echo "done"
-else
-  echo " no $pidFile"
-fi
-
-# stop util data loader
-pidFile=$CHUKWA_HOME/var/run/UtilDataLoader.pid
-if [ -f $pidFile ]; then  
-    echo -n "Shutting down Util Data Loader.."
-    utilpid=`head ${pidFile}`
-    #kill -HUP `ps eww |grep UtilDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
-    kill -HUP ${utilpid}
-    rm ${pidFile}
-    echo "done"
-else
-  echo " no $pidFile"
-fi
-
-# stop queue info data loader
-pidFile=$CHUKWA_HOME/var/run/QueueInfoDataLoader.pid
-if [ -f $pidFile ]; then  
-    echo -n "Shutting down Queue Info Data Loader.."
-    queuepid=`head ${pidFile}`
-    #kill -HUP `ps eww |grep QueueInfoDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
-    kill -HUP ${queuepid}
-    rm ${pidFile}
-    echo "done"
-else 
-  echo " no $pidFile"
-fi
-
-
-# stop queue info data loader
-pidFile=$CHUKWA_HOME/var/run/MapReduceLogLoader.pid
-if [ -f $pidFile ]; then  
-    echo -n "Shutting down Map Reduce Log Loader.."
-    logpid=`head ${pidFile}`
-    #kill -HUP `ps eww |grep MapReduceLogLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
-    kill -HUP ${logpid}
-    rm ${pidFile}
-    echo "done"
-else
-  echo " no $pidFile"
-fi
- 

Added: hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Run a shell command on all slave hosts.
+#
+# Environment Variables
+#
+#   CHUKWA_SLAVES    File naming remote hosts.
+#     Default is ${CHUKWA_CONF_DIR}/chukwa-slaves.
+#   CHUKWA_CONF_DIR  Alternate conf dir. Default is ${CHUKWA_HOME}/conf.
+#   CHUKWA_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   CHUKWA_SSH_OPTS Options passed to ssh when running remote commands.
+##
+
+usage="Usage: slaves.sh [--config confdir] command..."
+
+# if no args specified, show usage
+if [ $# -le 0 ]; then
+  echo $usage
+  exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# If the slaves file is specified in the command line,
+# then it takes precedence over the definition in 
+# hadoop-env.sh. Save it here.
+HOSTLIST=$CHUKWA_SLAVES
+
+if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
+  . "${CHUKWA_CONF_DIR}/chukwa-env.sh"
+fi
+
+if [ "$HOSTLIST" = "" ]; then
+  if [ "$CHUKWA_SLAVES" = "" ]; then
+    export HOSTLIST="${CHUKWA_CONF_DIR}/chukwa-slaves"
+  else
+    export HOSTLIST="${CHUKWA_SLAVES}"
+  fi
+fi
+
+for slave in `cat "$HOSTLIST"`; do
+ ssh $CHUKWA_SSH_OPTS $slave $"${@// /\\ }" \
+   2>&1 | sed "s/^/$slave: /" &
+ if [ "$CHUKWA_SLAVE_SLEEP" != "" ]; then
+   sleep $CHUKWA_SLAVE_SLEEP
+ fi
+done
+
+wait

Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start all chukwa daemons.  Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start probes
+"$bin"/start-probes.sh --config $CHUKWA_CONF_DIR
+
+# start data processors
+"$bin"/start-data-processors.sh --config $CHUKWA_CONF_DIR
+
+# start collectors
+"$bin"/start-collectors.sh --config $CHUKWA_CONF_DIR

Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-collectors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start hadoop dfs daemons.
+# Optinally upgrade or rollback dfs state.
+# Run this on master node.
+
+usage="Usage: start-collectors.sh"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start jetty collectors
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts collectors --watchdog start jettyCollector.sh

Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-data-processors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+java=$JAVA_HOME/bin/java
+
+. "$bin"/chukwa-config.sh
+if [ ! -d ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16 ]; then
+  if [ -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz ]; then
+    tar fxz ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz -C ${CHUKWA_HOME}/opt
+  fi
+fi
+
+if [ ! -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps/hicc-${CHUKWA_VERSION}.war ]; then
+  if [ -f ${CHUKWA_HOME}/hicc-${CHUKWA_VERSION}.war ]; then
+    cp ${CHUKWA_HOME}/hicc-${CHUKWA_VERSION}.war ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps
+  fi
+fi 
+
+# start data processors
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR --watchdog start processSinkFiles.sh watchdog
+
+# start database admin script
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start dbAdmin.sh

Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start hadoop dfs daemons.
+# Optinally upgrade or rollback dfs state.
+# Run this on master node.
+
+usage="Usage: start-probes.sh"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start agents
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --watchdog start agent.sh
+# start system data loader daemons
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR start systemDataLoader.sh
+
+# start torque data loader daemons
+if [ ${TORQUE_HOME} != "" ]; then
+  "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start torqueDataLoader.sh
+fi
+if [ ${nodeActivityCmde} != "" ]; then
+  "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start nodeActivityDataLoader.sh
+fi

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh Fri Oct 31 11:57:04 2008
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-java=$JAVA_HOME/bin/java
-
-. "$bin"/chukwa-config.sh
-crontab -l > /tmp/cron.${CURRENT_DATE}
-crontest=$?
-
-if [ ! -d ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16 ]; then
-  tar fxz ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz -C ${CHUKWA_HOME}/opt
-fi
-
-if [ ! -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps/ts.war ]; then
-  cp ${CHUKWA_HOME}/ts.war ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps
-fi 
-
-if [ "X$?" != "X0" ]; then
-  cat > /tmp/cron.${CURRENT_DATE} << CRON
-* * * * * ${CHUKWA_HOME}/bin/watchdog.sh
-10 * * * * ${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl
-CRON
-else
-  grep -v "${CHUKWA_HOME}/bin/watchdog.sh" /tmp/cron.${CURRENT_DATE} | \
-  grep -v "${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl" > /tmp/cron.${CURRENT_DATE}.2
-  mv /tmp/cron.${CURRENT_DATE}.2 /tmp/cron.${CURRENT_DATE}
-  cat >> /tmp/cron.${CURRENT_DATE} << CRON
-* * * * * ${CHUKWA_HOME}/bin/watchdog.sh
-10 * * * * ${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl
-CRON
-fi
-
-# save crontab
-echo -n "Registering data loader cron jobs.."
-crontab /tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
-rm -f /tmp/cron.${CURRENT_DATE}
-echo "done"
-

Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop all chukwa daemons.  Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/stop-probes.sh --config $CHUKWA_CONF_DIR
+"$bin"/stop-data-processors.sh --config $CHUKWA_CONF_DIR
+"$bin"/stop-collectors.sh --config $CHUKWA_CONF_DIR

Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-collectors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop collectors.  Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts collectors --watchdog stop jettyCollector.sh

Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+java=$JAVA_HOME/bin/java
+
+. "$bin"/chukwa-config.sh
+
+# stop processSinkFiles.sh
+pidFile=$CHUKWA_HOME/var/run/ProcessSinkFiles.pid
+if [ -f $pidFile ]; then  
+   echo -n "Shutting down Data Processors.."
+   DP_PID=`head ${pidFile}`
+   kill -TERM ${DP_PID}
+   rm ${pidFile}
+   echo "done"
+else
+  echo " no $pidFile"
+fi
+
+# stop dbAdmin.sh
+pidFile=$CHUKWA_HOME/var/run/dbAdmin.pid
+if [ -f $pidFile ]; then  
+   echo -n "Shutting down Database Admin.."
+   DBADMIN_PID=`head ${pidFile}`
+   kill -TERM ${DBADMIN_PID}
+   rm ${pidFile}
+   echo "done"
+else
+  echo " no $pidFile"
+fi
+

Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop probes.  Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR stop systemDataLoader.sh
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR stop torqueDataLoader.sh
+"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR stop nodeActivityDataLoader.sh
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --watchdog stop agent.sh

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh Fri Oct 31 11:57:04 2008
@@ -1,3 +1,6 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
@@ -12,17 +15,92 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-pid=$$
-
 bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 
 . "$bin"/chukwa-config.sh
 
-echo "${pid}" > "$CHUKWA_HOME/var/run/systemDataLoader.pid"
+JVM_OPTS="-Xms4M -Xmx4M"
 
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
-#${JAVA_HOME}/bin/java -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaAgent}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -x nfs -x none &
+if [ "X$1" = "Xstop" ]; then
+  echo -n "Shutting down System Data Loader..."
+  if [ -f ${CHUKWA_HOME}/var/run/Sar-data-loader.pid ]; then
+    kill -TERM `cat ${CHUKWA_HOME}/var/run/Sar-data-loader.pid`
+  fi
+  if [ -f ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid ]; then
+    kill -TERM `cat ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid`
+  fi
+  if [ -f ${CHUKWA_HOME}/var/run/Top-data-loader.pid ]; then
+    kill -TERM `cat ${CHUKWA_HOME}/var/run/Top-data-loader.pid`
+  fi
+  if [ -f ${CHUKWA_HOME}/var/run/Df-data-loader.pid ]; then
+    kill -TERM `cat ${CHUKWA_HOME}/var/run/Df-data-loader.pid`
+  fi
+  echo "done"
+  exit 0
+fi
+
+echo -n "Starting System Data Loader..."
+
+#test=`grep -q SysLog ${CHUKWA_HOME}/var/chukwa_checkpoint*`
+#if [ "X${test}"="X1" ]; then
+#  echo "add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped SysLog 0 /var/log/messages 0" | nc localhost 9093 >&/dev/null & disown -h 
+#fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Sar-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+    ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Iostat-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Top-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Df-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -l &
+fi
 
+echo "done"

Added: hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh Fri Oct 31 11:57:04 2008
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+java=$JAVA_HOME/bin/java
+
+if [ "X$1" = "Xstop" ]; then
+  echo -n "Shutting down Torque Data Loader..."
+  if [ -f ${CHUKWA_HOME}/var/run/TorqueDataLoader.pid ]; then
+    kill -TERM `cat ${CHUKWA_HOME}/var/run/TorqueDataLoader.pid`
+    rm -f ${CHUKWA_HOME}/var/run/TorqueDataLoader.pid
+  fi
+  echo "done"
+  exit 0
+fi
+
+min=`date +%M`
+
+
+# start torque data loader
+pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep TorqueDataLoader | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
+  fi 
+else
+      ${java} -DDOMAIN=${DOMAIN} -DTORQUE_SERVER=${TORQUE_SERVER} -DTORQUE_HOME=${TORQUE_HOME} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Torque -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.mdl.TorqueDataLoader&
+fi

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh Fri Oct 31 11:57:04 2008
@@ -21,92 +21,127 @@
 . "$bin"/chukwa-config.sh
 
 java=$JAVA_HOME/bin/java
-jps=$JAVA_HOME/bin/jps
 
 
 min=`date +%M`
 
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+  export CHUKWA_IDENT_STRING="$USER"
+fi
 
-# start torque data loader
-pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+# monitor agent
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-agent.sh.pid
 if [ -f $pidFile ]; then
   pid=`head ${pidFile}`
-  ChildPIDRunningStatus=`${jps} | grep ${pid} | grep TorqueDataLoader | grep -v grep | wc -l`
-  #ChildPIDRunningStatus=`ps -ef | grep TorqueDataLoader | grep -v grep | wc -l`
+  ChildPIDRunningStatus=`ps ax | grep ${pid} | grep agent.sh | grep -v grep | wc -l`
   if [ $ChildPIDRunningStatus -lt 1 ]; then
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=torque.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.TorqueDataLoader&
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: agent pid file exists, but process missing.  Restarting agent.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start agent.sh &
   fi 
-else
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=torque.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.TorqueDataLoader&
 fi
-# start util data loader
-pidFile=$CHUKWA_HOME/var/run/UtilDataLoader.pid
+
+# monitor collector
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-jettyCollector.sh.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`ps ax | grep ${pid} | grep jettyCollector.sh | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: collector pid file exists, but process missing.  Restarting jettyCollector.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start jettyCollector.sh &
+  fi
+fi
+
+# monitor node activity data loader
+pidFile=$CHUKWA_HOME/var/run/PbsNodes-data-loader.pid
 if [ -f $pidFile ]; then
   pid=`head ${pidFile}`
-  ChildPIDRunningStatus=`${jps} | grep ${pid} | grep UtilDataLoader | grep -v grep | wc -l`
-  #ChildPIDRunningStatus=`ps -ef | grep UtilDataLoader | grep -v grep | wc -l`
-  if [ $ChildPIDRunningStatus -lt 1 ]; then
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=util.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.UtilDataLoader&
-  fi
-else 
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=util.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.UtilDataLoader&
-fi
-
-# start queue info data loader
-pidFile=$CHUKWA_HOME/var/run/QueueInfoDataLoader.pid
-if [ -f $pidFile ]; then
-  pid=`head ${pidFile}`
-  ChildPIDRunningStatus=`${jps} | grep ${pid} | grep QueueInfoDataLoader | grep -v grep | wc -l`
-  #ChildPIDRunningStatus=`ps -ef | grep QueueInfoDataLoader | grep -v grep | wc -l`
-  if [ $ChildPIDRunningStatus -lt 1 ]; then
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=queueinfo.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.QueueInfoDataLoader&
-  fi
-else
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=queueinfo.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.QueueInfoDataLoader&
-fi
-
-# start map reduce log data loader
-tenmin=`echo ${min} | cut -b 2-`
-if [ "X${tenmin}" == "X0" ]; then
-    pidFile=$CHUKWA_HOME/var/run/JobLogDataLoader.pid
-    if [ -f $pidFile ]; then
-        pid=`head ${pidFile}`
-        ChildPIDRunningStatus=`${jps} | grep ${pid} | grep JobLogDataLoader |  wc -l`
-        if [ $ChildPIDRunningStatus -lt 1 ]; then
-            ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=joblog.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.JobLogDataLoader &
-        fi
-    else
-        ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=joblog.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.JobLogDataLoader &
-    fi
-fi
-
-# start node activity plugin
-tenmin=`echo ${min} | cut -b 2-`
-if [ "X${tenmin}" == "X0" ]; then
-  pidFile=$CHUKWA_HOME/var/run/NodeActivityPlugin.pid
-  if [ -f $pidFile ]; then
-    pid=`head ${pidFile}`
-    ChildPIDRunningStatus=`${jps} | grep ${pid} | grep NodeActivityMDL | wc -l`
-    if [ $ChildPIDRunningStatus -lt 1 ]; then
-       ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=nodeActivity.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.plugin.nodeactivity.NodeActivityMDL&
-    fi
-  else
-      ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=nodeActivity.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.plugin.nodeactivity.NodeActivityMDL&
-  fi
-fi
-
-# start database summary loader
-tenmin=`echo ${min} | cut -b 2-`
-if [ "X${tenmin}" == "X0" ]; then
-    pidFile=$CHUKWA_HOME/var/run/DBSummaryLoader.pid
-    if [ -f $pidFile ]; then
-        pid=`head ${pidFile}`
-        ChildPIDRunningStatus=`${jps} | grep ${pid} | grep DBSummaryLoader | wc -l`
-        if [ $ChildPIDRunningStatus -lt 1 ]; then
-            ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${ckit}:${common} org.apache.hadoop.chukwa.extraction.DBSummaryLoader &
-        fi
-    else
-        ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${ckit}:${common} org.apache.hadoop.chukwa.extraction.DBSummaryLoader &
-    fi
+  ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: PbsNodes-data-loader pid file exists, but process missing.  Restarting nodeActivityDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start nodeActivityDataLoader.sh &
+  fi
 fi
 
+# monitor system data loader
+pidFile=$CHUKWA_HOME/var/run/Df-data-loader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: Df-data-loader pid file exists, but process missing.  Restarting systemDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+  fi
+fi
+
+pidFile=$CHUKWA_HOME/var/run/Iostat-data-loader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: Iostat-data-loader pid file exists, but process missing.  Restarting systemDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+  fi
+fi
+
+pidFile=$CHUKWA_HOME/var/run/Sar-data-loader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: Sar-data-loader pid file exists, but process missing.  Restarting systemDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+  fi
+fi
+
+pidFile=$CHUKWA_HOME/var/run/Top-data-loader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: Top-data-loader pid file exists, but process missing.  Restarting systemDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+  fi
+fi
+
+# monitor torque data loader
+pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ^${pid} | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: pid file exists, but process missing.  Restarting torqueDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start torqueDataLoader.sh &
+  fi
+fi
+
+# monitor dataSinkFiles.sh
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-processSinkFiles.sh.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`ps ax | grep ${pid} | grep processSinkFiles.sh | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: pid file exists, but process missing.  Restarting processSinkFiles.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start processSinkFiles.sh &
+  fi
+fi
+
+# monitor dbAdmin.sh
+pidFile=$CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-dbAdmin.sh.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`ps ax | grep ${pid} | grep dbAdmin.sh | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: pid file exists, but process missing.  Restarting dbAdmin.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start dbAdmin.sh &
+  fi
+fi

Modified: hadoop/core/trunk/src/contrib/chukwa/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/build.xml?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/build.xml (original)
+++ hadoop/core/trunk/src/contrib/chukwa/build.xml Fri Oct 31 11:57:04 2008
@@ -20,9 +20,7 @@
 <project name="chukwa" default="main">
 
   <import file="../build-contrib.xml"/>
-
-
-	<property name="chukwaVersion" value="0.0.1"/>
+	<property name="chukwaVersion" value="0.1.0"/>
 
 	<property name="lib.dir" value="${basedir}/lib"/>
 	<property name="src.dir" value="${basedir}/src"/>
@@ -60,15 +58,6 @@
 		<mkdir dir="${build.dir}"/>
 		<mkdir dir="${build.classes}"/>
 		<mkdir dir="${build.dir}/test"/>
-
-		<!-- 
-    	<copy todir="${build.dir}">
-            <fileset dir="${basedir}">
-                <exclude name="**/VERSION"/>
-            </fileset>
-        </copy>
-      
--->
 		<exec executable="echo" output="${basedir}/bin/VERSION">
 			<arg line="${chukwaVersion}" />
 		</exec>
@@ -106,9 +95,6 @@
 
 
 
-	<target name="test" depends="compile,compile-test,test-chunk" description="Chukwa Tests">
-	</target>
-
 	<target name="test-chunk" depends="compile,compile-test" description="Test chunk">
                
           <junit showoutput="yes"
@@ -136,16 +122,16 @@
 		<mkdir dir="${build.dir}/collector/WEB-INF/lib"/>
 		<mkdir dir="${build.dir}/collector/META-INF"/>
 		<copy todir="${build.dir}/collector/WEB-INF/classes">
-			<fileset dir="${build.dir}">
+			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/chukwa/conf/**/*.class" />
 			</fileset>
-			<fileset dir="${build.dir}">
+			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/chukwa/datacollection/**/*.class" />
 			</fileset>
-			<fileset dir="${build.dir}">
+			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/chukwa/extraction/**/*.class" />
 			</fileset>
-			<fileset dir="${build.dir}">
+			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/chukwa/util/**/*.class" />
 			</fileset>
 			<fileset dir="${basedir}/src/java">
@@ -182,7 +168,7 @@
 		</jar>
 	</target>
 	<target name="tools_jar" depends="compile, collector" description="Create tools jar">
-		<jar jarfile="${build.dir}/tools-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/inputtools/**/*.class">
+		<jar jarfile="${build.dir}/tools-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/inputtools/**/*.class">
 			<fileset dir="${basedir}/src/java">
 				<include name="org/apache/hadoop/chukwa/inputtools/**/*.java"/>
 			</fileset>
@@ -190,7 +176,7 @@
 	</target>
 
 	<target name="agent_jar" depends="compile, collector" description="Create agent jar">
-		<jar jarfile="${build.dir}/chukwa-agent-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/client/**/*.class" >
+		<jar jarfile="${build.dir}/chukwa-agent-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/client/**/*.class" >
 			<fileset dir="${build.dir}">
 				<include name="org/apache/hadoop/chukwa/conf/**/*.class"/>
 			</fileset>
@@ -226,7 +212,7 @@
 	</target>
 
 	<target name="chukwa_jar" depends="compile, collector" description="Create chukwa-core jar">
-		<jar jarfile="${build.dir}/chukwa-core-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/datacollection/**/*.class" >
+		<jar jarfile="${build.dir}/chukwa-core-${chukwaVersion}.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/datacollection/**/*.class" >
                         <manifest>
                             <section name="org/apache/hadoop/chukwa">
                                 <attribute name="Implementation-Title" value="Chukwa"/>
@@ -246,7 +232,7 @@
 
 	<target name="chukwa-hadoop_jar" depends="compile" description="Create chukwa_hadoop jar for use with getting hadoop to use chukwa">
 
-		<jar jarfile="${build.dir}/chukwa-hadoop-${chukwaVersion}-client.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class">
+		<jar jarfile="${build.dir}/chukwa-hadoop-${chukwaVersion}-client.jar" basedir="${build.classes}" includes="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class">
 			<fileset dir="${basedir}/src/java">
 				<include name="org/apache/hadoop/chukwa/inputtools/log4j/**/*.java"/>
 		        <include name="org/apache/hadoop/chukwa/datacollection/client/**/*.java"/>
@@ -261,7 +247,6 @@
 	</target>
 
 	<target name="compress" depends="compile,collector,collector_jar,tools_jar,agent_jar,chukwa_jar,chukwa-hadoop_jar" description="Compression target">
-		<!-- 	<jar jarfile="${build.dir}/chukwa.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/**/*.class" /> -->
 
 		<copy todir="." includeEmptyDirs="false">
 			<fileset dir="${build.dir}">
@@ -271,6 +256,32 @@
 		</copy>
 	</target>
 
+	<target name="test" depends="compile,compile-test,test-chunk,test-input-tools" description="Automated Test Framework">
+	</target>
+
+	<target name="test-input-tools" depends="compile,compile-test" description="Test Input Tools">
+            <junit showoutput="${test.output}"
+             printsummary="${test.junit.printsummary}"
+             haltonfailure="${test.junit.haltonfailure}"
+             fork="yes"
+             forkmode="${test.junit.fork.mode}"
+             maxmemory="${test.junit.maxmemory}"
+             dir="${basedir}" timeout="${test.timeout}"
+             errorProperty="tests.failed" failureProperty="tests.failed">
+               <sysproperty key="CHUKWA_HOME" value="${basedir}"/>
+               <sysproperty key="CHUKWA_CONF_DIR" value="${basedir}/conf"/>
+               <classpath refid="chukwaTestClasspath"/>
+               <formatter type="${test.junit.output.format}" />
+               <batchtest todir="${build.dir}/test" >
+               <fileset dir="${test.src.dir}" includes="org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/*.java"/>
+               </batchtest>
+            </junit>
+            <delete>
+               <fileset dir="${basedir}/var" includes="*"/>
+            </delete>
+            <fail if="tests.failed">Tests failed!</fail>
+	</target>
+
 	<!-- ====================================================== -->
 	<!-- Macro definitions                                      -->
 	<!-- ====================================================== -->
@@ -323,7 +334,7 @@
 		</copy>
 
 		<copy todir="${build.dir}/${final.name}/conf">
-			<fileset dir="${conf.dir}" excludes="**/*.template"/>
+			<fileset dir="${conf.dir}"/>
 		</copy>
 
 		<copy todir="${build.dir}/${final.name}/docs">
@@ -372,6 +383,8 @@
 			<param.listofitems>
 				<tarfileset dir="${build.dir}" mode="664">
 					<exclude name="${final.name}/org/*" />
+					<exclude name="${final.name}/collector/**" />
+					<exclude name="${final.name}/${final.name}/**" />
 					<include name="${final.name}/**" />
 				</tarfileset>
 				<tarfileset dir="${build.dir}" mode="755">
@@ -420,8 +433,8 @@
 	</path>
         <path id="chukwaTestClasspath">
                 <pathelement location="${hadoop.root}/build/classes"/>
-                <pathelement location="${hadoop.root}/build/contrib/chukwa/classes/"/>
-                <pathelement location="${hadoop.root}/build/contrib/chukwa/test/classes/"/>
+                <pathelement location="${build.classes}"/>
+                <pathelement location="${test.build.classes}"/>
                 <fileset dir="${lib.dir}">
                         <include name="**/*.jar" />
                         <exclude name="**/excluded/" />

Added: hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/alert.conf.template Fri Oct 31 11:57:04 2008
@@ -0,0 +1 @@
+user@example.com

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh Fri Oct 31 11:57:04 2008
@@ -6,20 +6,35 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-export JAVA_HOME=/Library/Java/Home/
+export JAVA_HOME=/usr/lib/j2sdk1.5-sun
 
 # The location of the Hadoop the collector should use. Default 
 # assumes that this chukwa is living in hadoop's src/contrib directory
-#export HADOOP_HOME=
+export HADOOP_HOME="/usr/lib/hadoop/current"
 
 # The directory where pid files are stored. CHUKWA_HOME/var/run by default.
 #export CHUKWA_PID_DIR=
 
 # The location of chukwa logs, defaults to CHUKWA_HOME/logs
-#export CHUKWA_LOG_DIR=
+export CHUKWA_LOG_DIR=${CHUKWA_HOME}/logs
 
 # The location of a hadoop jars. use this if you are running a collector
 # without a running HDFS (i.e. which writes sequence files to local disk)
 # if this is not set, the default is to check HADOOP_HOME for jars or
 # classes, if those are not found, uses hadoop jars which come with chukwa
-#export HADOOP_JAR=
+export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
+
+# The location of chukwa data repository
+export chuwaRecordsRepository="/chukwa/repos/"
+
+# The location of torque pbsnodes command
+export nodeActivityCmde="/usr/lib/torque/current/bin/pbsnodes "
+
+# The server which contain pbsnodes, qstat and tracejob.
+export TORQUE_SERVER=localhost
+
+# The location contain torque binaries.
+export TORQUE_HOME=/usr/lib/torque
+
+# The domain of the cluster
+#export DOMAIN=

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template Fri Oct 31 11:57:04 2008
@@ -6,20 +6,35 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-export JAVA_HOME=/Library/Java/Home/
+export JAVA_HOME=/usr/lib/j2sdk1.5-sun
 
 # The location of the Hadoop the collector should use. Default 
 # assumes that this chukwa is living in hadoop's src/contrib directory
-#export HADOOP_HOME=
+export HADOOP_HOME="/usr/lib/hadoop/current"
 
 # The directory where pid files are stored. CHUKWA_HOME/var/run by default.
 #export CHUKWA_PID_DIR=
 
 # The location of chukwa logs, defaults to CHUKWA_HOME/logs
-#export CHUKWA_LOG_DIR=
+export CHUKWA_LOG_DIR=${CHUKWA_HOME}/logs
 
 # The location of a hadoop jars. use this if you are running a collector
 # without a running HDFS (i.e. which writes sequence files to local disk)
 # if this is not set, the default is to check HADOOP_HOME for jars or
 # classes, if those are not found, uses hadoop jars which come with chukwa
-#export HADOOP_JAR=
+export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
+
+# The location of chukwa data repository
+export chuwaRecordsRepository="/chukwa/repos/"
+
+# The location of torque pbsnodes command
+export nodeActivityCmde="/usr/lib/torque/current/bin/pbsnodes "
+
+# The server which contain pbsnodes, qstat and tracejob.
+export TORQUE_SERVER=localhost
+
+# The location contain torque binaries.
+export TORQUE_HOME=/usr/lib/torque
+
+# The domain of the cluster
+#export DOMAIN=

Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template Fri Oct 31 11:57:04 2008
@@ -0,0 +1 @@
+localhost

Added: hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties Fri Oct 31 11:57:04 2008
@@ -0,0 +1,7 @@
+dfs.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+dfs.period=60
+
+jvm.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4JMetricsContext
+jvm.period=60
+
+

Added: hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template?rev=709533&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/initial_adaptors.template Fri Oct 31 11:57:04 2008
@@ -0,0 +1 @@
+add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped SysLog 0 /var/log/messages 0

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R 
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/NodeActivityPlugin.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R 
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/QueueInfoDataLoader.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R 
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/TorqueDataLoader.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/util.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/util.properties?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/util.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/util.properties Fri Oct 31 11:57:04 2008
@@ -1,8 +0,0 @@
-log4j.rootLogger=INFO, R 
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${CHUKWA_HOME}/logs/UtilDataLoader.log
-log4j.appender.R.MaxFileSize=10MB
-log4j.appender.R.MaxBackupIndex=10
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
-log4j.logger.org.apache.hadoop.chukwa.ikit.DataConfig=INFO, R

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java?rev=709533&r1=709532&r2=709533&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java (original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java Fri Oct 31 11:57:04 2008
@@ -27,27 +27,28 @@
 /**
  * An instrumentation plugin for Hadoop, to trigger Chukwa-based task logfile collection.
  * 
- * WARNING:  This code depends on hadoop features that have not yet been committed.
- *   To allow it to compile, the key lines have been commented out, and marked with
- *   'PENDING'.
+ * WARNING:  This code depends on hadoop features only available in 0.19.
+ * It won't do any good if you try to use it with an earlier Hadoop.
  *
  */
-public class ChukwaTTInstru 
-extends TaskTrackerMetricsInst  //PENDING on getting new metrics code into Hadoop
+public class ChukwaTTInstru extends TaskTrackerMetricsInst 
 {
 
   private Map<TaskAttemptID, Long> stdOutAdaptors;
   private Map<TaskAttemptID, Long> stdErrAdaptors;
   private ChukwaAgentController chukwa;
+//  private TaskTrackerMetricsInst parent; //for chaining together multiple
+      //instrumentation subsystems
   
   public ChukwaTTInstru(TaskTracker t) {
-    super(t);  //PENDING
+    super(t);
     stdOutAdaptors = new HashMap<TaskAttemptID, Long>();
     stdErrAdaptors = new HashMap<TaskAttemptID, Long>();
     chukwa = new ChukwaAgentController();
   }
   
   public void reportTaskLaunch(TaskAttemptID taskid, File stdout, File stderr)  {
+//    parent.reportTaskLaunch(taskid, stdout, stderr);
     long stdoutID = chukwa.addFile("unknown-userdata", stdout.getAbsolutePath());
     long stderrID = chukwa.addFile("unknown-userdata", stderr.getAbsolutePath());
     stdOutAdaptors.put(taskid, stdoutID);



Mime
View raw message