chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r761148 - in /hadoop/chukwa/trunk: ./ bin/ src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ tools/service/chukwa-ps/
Date Thu, 02 Apr 2009 02:22:14 GMT
Author: eyang
Date: Thu Apr  2 02:22:13 2009
New Revision: 761148

URL: http://svn.apache.org/viewvc?rev=761148&view=rev
Log:
CHUKWA-17. Collect PS output every 10 minutes and parse into key/value pairs sequence file.

Added:
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Ps.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PsOutputTest.java
    hadoop/chukwa/trunk/tools/service/chukwa-ps/
    hadoop/chukwa/trunk/tools/service/chukwa-ps/run
Modified:
    hadoop/chukwa/trunk/bin/exec-data-loader.sh
    hadoop/chukwa/trunk/bin/systemDataLoader.sh
    hadoop/chukwa/trunk/bin/watchdog.sh
    hadoop/chukwa/trunk/build.xml

Modified: hadoop/chukwa/trunk/bin/exec-data-loader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/exec-data-loader.sh?rev=761148&r1=761147&r2=761148&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/exec-data-loader.sh (original)
+++ hadoop/chukwa/trunk/bin/exec-data-loader.sh Thu Apr  2 02:22:13 2009
@@ -35,6 +35,9 @@
   if [ "X$PARM" == "Xtop" ]; then
     kill -9 `cat ${CHUKWA_PID_DIR}/Top-data-loader.pid`
   fi
+  if [ "X$PARM" == "Xps" ]; then
+    kill -9 `cat ${CHUKWA_PID_DIR}/Ps-data-loader.pid`
+  fi
   if [ "X$PARM" == "Xdf" ]; then
     kill -9 `cat ${CHUKWA_PID_DIR}/Df-data-loader.pid`
   fi
@@ -74,6 +77,10 @@
   ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
 fi
 
+if [ "X$PARM" == "Xps" ]; then
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Ps -Dlog4j.configuration=system-data-loader.properties
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec ps axo pid,user,vsize,size,pcpu,pmem,time,start,cmd
&
+fi
+
 if [ "X$PARM" == "Xdf" ]; then
   ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -l &
 fi

Modified: hadoop/chukwa/trunk/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/systemDataLoader.sh?rev=761148&r1=761147&r2=761148&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/systemDataLoader.sh (original)
+++ hadoop/chukwa/trunk/bin/systemDataLoader.sh Thu Apr  2 02:22:13 2009
@@ -78,6 +78,23 @@
   fi
 
   EXISTS=0
+  pidFile="${CHUKWA_PID_DIR}/Ps-data-loader.pid"
+  if [ -f $pidFile ]; then
+    pid=`head ${pidFile}`
+    ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+    if [ $ChildPIDRunningStatus -ge 1 ]; then
+      EXISTS=1
+    fi
+  fi
+
+  if [ ${EXISTS} -lt 1 ]; then
+    echo "ps data loader is stopped."
+    RESULT=1
+  else
+    echo "ps data loader is running."
+  fi
+
+  EXISTS=0
   pidFile="${CHUKWA_PID_DIR}/Df-data-loader.pid"
   if [ -f $pidFile ]; then
     pid=`head ${pidFile}`
@@ -135,6 +152,9 @@
   if [ -f ${CHUKWA_PID_DIR}/Top-data-loader.pid ]; then
     kill -9 `cat ${CHUKWA_PID_DIR}/Top-data-loader.pid`
   fi
+  if [ -f ${CHUKWA_PID_DIR}/Ps-data-loader.pid ]; then
+    kill -9 `cat ${CHUKWA_PID_DIR}/Ps-data-loader.pid`
+  fi
   if [ -f ${CHUKWA_PID_DIR}/Df-data-loader.pid ]; then
     kill -9 `cat ${CHUKWA_PID_DIR}/Df-data-loader.pid`
   fi
@@ -213,6 +233,20 @@
 fi
 
 EXISTS=0
+pidFile="${CHUKWA_PID_DIR}/Ps-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Ps -Dlog4j.configuration=system-data-loader.properties
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec ps axo pid,user,vsize,size,pcpu,pmem,time,start,cmd
&
+fi
+
+EXISTS=0
 pidFile="${CHUKWA_PID_DIR}/Df-data-loader.pid"
 if [ -f $pidFile ]; then
   pid=`head ${pidFile}`

Modified: hadoop/chukwa/trunk/bin/watchdog.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/watchdog.sh?rev=761148&r1=761147&r2=761148&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/watchdog.sh (original)
+++ hadoop/chukwa/trunk/bin/watchdog.sh Thu Apr  2 02:22:13 2009
@@ -110,6 +110,17 @@
 #  fi
 #fi
 
+pidFile=$CHUKWA_PID_DIR/Ps-data-loader.pid
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep Exec | grep -v grep | grep -o "[^ ].*" | grep ${pid}
| wc -l`
+  if [ $ChildPIDRunningStatus -lt 1 ]; then
+      HOSTNAME=`hostname`
+      echo "${HOSTNAME}: Ps-data-loader pid file exists, but process missing.  Restarting
systemDataLoader.sh."
+      "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR start systemDataLoader.sh &
+  fi
+fi
+
 # monitor torque data loader
 pidFile=$CHUKWA_PID_DIR/TorqueDataLoader.pid
 if [ -f $pidFile ]; then
@@ -154,3 +165,5 @@
 fi
 
 
+
+

Modified: hadoop/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/build.xml?rev=761148&r1=761147&r2=761148&view=diff
==============================================================================
--- hadoop/chukwa/trunk/build.xml (original)
+++ hadoop/chukwa/trunk/build.xml Thu Apr  2 02:22:13 2009
@@ -874,6 +874,7 @@
 mkdir -p %{buildroot}/service/chukwa-pbsnodes
 mkdir -p %{buildroot}/service/chukwa-sar
 mkdir -p %{buildroot}/service/chukwa-top
+mkdir -p %{buildroot}/service/chukwa-ps
 mkdir -p %{buildroot}/service/chukwa-torque
 cat %{buildroot}%{_prefix}/tools/init.d/chukwa-data-processors | \
 sed 's:- chukwa -c:- %{uid} -c:' | \
@@ -895,6 +896,8 @@
 sed 's:CHUKWA_CONF_DIR=/usr/local/chukwa/conf:CHUKWA_CONF_DIR=%{_conf_dir}:' > %{buildroot}/service/chukwa-sar/run
 cat %{buildroot}%{_prefix}/tools/service/chukwa-top/run | sed 's:/usr/local/chukwa:%{_prefix}:'
| \
 sed 's:CHUKWA_CONF_DIR=/usr/local/chukwa/conf:CHUKWA_CONF_DIR=%{_conf_dir}:' > %{buildroot}/service/chukwa-top/run
+cat %{buildroot}%{_prefix}/tools/service/chukwa-ps/run | sed 's:/usr/local/chukwa:%{_prefix}:'
| \
+sed 's:CHUKWA_CONF_DIR=/usr/local/chukwa/conf:CHUKWA_CONF_DIR=%{_conf_dir}:' > %{buildroot}/service/chukwa-ps/run
 #cat %{buildroot}%{_prefix}/tools/service/chukwa-collector/run | sed 's:/usr/local/chukwa:%{_prefix}:'
| \
 #sed 's:CHUKWA_CONF_DIR=/usr/local/chukwa/conf:CHUKWA_CONF_DIR=%{_conf_dir}:' > %{buildroot}/service/chukwa-collector/run
 #cat %{buildroot}%{_prefix}/tools/service/chukwa-pbsnodes/run | sed 's:/usr/local/chukwa:%{_prefix}:'
| \
@@ -929,6 +932,7 @@
 /service/chukwa-netstat/run
 /service/chukwa-sar/run
 /service/chukwa-top/run
+/service/chukwa-ps/run
                 </echo>
 		<rpm specFile="chukwa.spec" topDir="${build.dir}" cleanBuildDir="true" failOnError="true"/>
         </target>

Added: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Ps.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Ps.java?rev=761148&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Ps.java
(added)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Ps.java
Thu Apr  2 02:22:13 2009
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.extraction.demux.processor.mapper;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
+import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.log4j.Logger;
+
+public class Ps extends AbstractProcessor {
+  static Logger log = Logger.getLogger(Ps.class);
+
+  @Override
+  protected void parse(String recordEntry,
+      OutputCollector<ChukwaRecordKey, ChukwaRecord> output, Reporter reporter)
+      throws Throwable {
+    LogEntry log = new LogEntry(recordEntry);
+    PsOutput ps = new PsOutput(log.getBody());
+    for (HashMap<String, String> processInfo : ps.getProcessList()) {
+      key = new ChukwaRecordKey();
+      ChukwaRecord record = new ChukwaRecord();
+      this.buildGenericRecord(record, null, log.getDate().getTime(), "Ps");
+      for (Entry<String, String> entry : processInfo.entrySet()) {
+        record.add(entry.getKey(), entry.getValue());
+      }
+      output.collect(key, record);
+    }
+  }
+
+  public static class PsOutput {
+
+    // processes info
+    private ArrayList<HashMap<String, String>> recordList = new ArrayList<HashMap<String,
String>>();
+
+    public PsOutput(String psCmdOutput) throws InvalidPsRecord {
+      if (psCmdOutput == null || psCmdOutput.length() == 0)
+        return;
+
+      String[] lines = psCmdOutput.split("[\n\r]+");
+
+      // at least two lines
+      if (lines.length < 2)
+        return;
+
+      // header
+      ArrayList<String> header = new ArrayList<String>();
+      Matcher matcher = Pattern.compile("[^ ^\t]+").matcher(lines[0]);
+      while (matcher.find()) {
+        header.add(matcher.group(0));
+      }
+      if (!header.get(header.size() - 1).equals("CMD")) {
+        throw new InvalidPsRecord("CMD must be the last column");
+      }
+
+      // records
+      boolean foundInitCmd = false;
+      for (int line = 1; line < lines.length; line++) {
+        HashMap<String, String> record = new HashMap<String, String>();
+        recordList.add(record);
+
+        matcher = Pattern.compile("[^ ^\t]+").matcher(lines[line]);
+        for (int index = 0; index < header.size(); index++) {
+          String key = header.get(index);
+          matcher.find();
+          if (!key.equals("CMD")) {
+            String value = matcher.group(0);
+            /**
+             * For STARTED column, it could be in two formats: "MMM dd" or
+             * "hh:mm:ss". If we use ' ' as the delimiter, we must read twice to
+             * the date if it's with "MMM dd" format.
+             */
+            if (key.equals("STARTED")) {
+              char c = value.charAt(0);
+              if (c < '0' || c > '9') {
+                matcher.find();
+                value += matcher.group(0);
+              }
+            }
+            record.put(key, value);
+          } else {
+            // reached the cmd part. all remains should be put
+            // together as the command
+            String value = lines[line].substring(matcher.start());
+            record.put(key, value);
+            if (!foundInitCmd)
+              foundInitCmd = value.startsWith("init");
+            break;
+          }
+        }
+      }
+      if (!foundInitCmd)
+        throw new InvalidPsRecord("Did not find 'init' cmd");
+    }
+
+    public ArrayList<HashMap<String, String>> getProcessList() {
+      return recordList;
+    }
+  }
+
+  public static class InvalidPsRecord extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public InvalidPsRecord() {
+    }
+
+    public InvalidPsRecord(String arg0) {
+      super(arg0);
+    }
+
+    public InvalidPsRecord(Throwable arg0) {
+      super(arg0);
+    }
+
+    public InvalidPsRecord(String arg0, Throwable arg1) {
+      super(arg0, arg1);
+    }
+  }
+}

Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PsOutputTest.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PsOutputTest.java?rev=761148&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PsOutputTest.java
(added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PsOutputTest.java
Thu Apr  2 02:22:13 2009
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.extraction.demux.processor.mapper.ps;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Ps.InvalidPsRecord;
+import org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Ps.PsOutput;
+
+import junit.framework.TestCase;
+
+public class PsOutputTest extends TestCase {
+
+  public void testGetRecordListFromPsCmd() throws IOException, InvalidPsRecord {
+    Runtime runtime = Runtime.getRuntime();
+    Process process = runtime
+        .exec("ps axo pid,user,vsize,size,pcpu,pmem,time,start_time,start,cmd");
+    StringBuffer sb = new StringBuffer();
+    InputStream is = process.getInputStream();
+    byte[] buffer = new byte[1024];
+    while (true) {
+      int len = is.read(buffer);
+      if (len == -1)
+        break;
+      sb.append(new String(buffer, 0, len));
+    }
+    String output = sb.toString();
+
+    PsOutput pso = new PsOutput(output);
+
+    // search init process
+    for (HashMap<String, String> processInfo : pso.getProcessList()) {
+      for (Entry<String, String> entry : processInfo.entrySet()) {
+        if (entry.getKey().equals("CMD") && entry.getValue().startsWith("init"))
{
+          return;
+        }
+      }
+    }
+
+    throw new InvalidPsRecord(output);
+  }
+
+  public void testGetRecordList() throws IOException, InvalidPsRecord {
+    // below is from command
+    // "ps axo pid,user,vsize,size,pcpu,pmem,time,start_time,start,cmd"
+    String output = "  PID USER        VSZ    SZ %CPU %MEM     TIME START  STARTED CMD\n"
+        + "    1 root       2064   284  0.0  0.0 00:00:02  2008   Dec 29 init [5]\n"
+        + "    2 root          0     0  0.0  0.0 00:00:01  2008   Dec 29 [migration/0]\n"
+        + "20270 chzhang    4248   588  0.0  0.0 00:00:00 15:32 15:32:36 ps axo pid,user,vsize,size,pcpu,pmem,time,start_time,start,cmd\n"
+        + "28371 angelac2   7100  1716  0.0  0.0 00:00:00 Feb27   Feb 27 /usr/libexec/gconfd-2
5\n";
+
+    PsOutput pso = new PsOutput(output);
+    ArrayList<HashMap<String, String>> processes = pso.getProcessList();
+    assertEquals(4, processes.size());
+    assertEquals("Dec29", processes.get(0).get("STARTED"));
+    assertEquals("15:32:36", processes.get(2).get("STARTED"));
+    assertEquals(
+        "ps axo pid,user,vsize,size,pcpu,pmem,time,start_time,start,cmd",
+        processes.get(2).get("CMD"));
+  }
+
+}

Added: hadoop/chukwa/trunk/tools/service/chukwa-ps/run
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/tools/service/chukwa-ps/run?rev=761148&view=auto
==============================================================================
--- hadoop/chukwa/trunk/tools/service/chukwa-ps/run (added)
+++ hadoop/chukwa/trunk/tools/service/chukwa-ps/run Thu Apr  2 02:22:13 2009
@@ -0,0 +1,5 @@
+#!/bin/sh
+CHUKWA_CONF_DIR=/usr/local/chukwa/conf
+
+exec setuidgid gmon /usr/local/chukwa/bin/exec-data-loader.sh --config $CHUKWA_CONF_DIR ps
+



Mime
View raw message