chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r887765 - in /hadoop/chukwa/trunk: CHANGES.txt bin/fsm-clienttrace-dn.sh bin/fsm-clienttrace-mr.sh bin/fsm-jobhist.sh src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestMemBuffer.java
Date Sun, 06 Dec 2009 20:48:48 GMT
Author: asrabkin
Date: Sun Dec  6 20:48:47 2009
New Revision: 887765

URL: http://svn.apache.org/viewvc?rev=887765&view=rev
Log:
CHUKWA-424. FSMBuilder driver script. Contributed by Jiaqi Tan.

Added:
    hadoop/chukwa/trunk/bin/fsm-clienttrace-dn.sh
    hadoop/chukwa/trunk/bin/fsm-clienttrace-mr.sh
    hadoop/chukwa/trunk/bin/fsm-jobhist.sh
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestMemBuffer.java
Modified:
    hadoop/chukwa/trunk/CHANGES.txt

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=887765&r1=887764&r2=887765&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Sun Dec  6 20:48:47 2009
@@ -4,6 +4,8 @@
 
   NEW FEATURES
 
+    CHUKWA-424. FSMBuilder driver script. (Jiaqi Tan via asrabkin)
+
     CHUKWA-395. Support for generalized buffering of adaptor data. (asrabkin)
 
     CHUKWA-405  Add a "stop all" command. (asrabkin)

Added: hadoop/chukwa/trunk/bin/fsm-clienttrace-dn.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/fsm-clienttrace-dn.sh?rev=887765&view=auto
==============================================================================
--- hadoop/chukwa/trunk/bin/fsm-clienttrace-dn.sh (added)
+++ hadoop/chukwa/trunk/bin/fsm-clienttrace-dn.sh Sun Dec  6 20:48:47 2009
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+${JAVA_HOME}/bin/java \
+-Djava.library.path=${JAVA_LIBRARY_PATH} \
+-DCHUKWA_HOME=${CHUKWA_HOME} \
+-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} \
+-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} \
+-DAPP=fsm \
+-Dlog4j.configuration=chukwa-log4j.properties \
+-classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_CONF_DIR}:${HADOOP_JAR}:${COMMON}:${tools}
\
+org.apache.hadoop.chukwa.analysis.salsa.fsm.FSMBuilder \
+-Dchukwa.salsa.fsm.mapclass=org.apache.hadoop.chukwa.analysis.salsa.fsm.DataNodeClientTraceMapper
\
+"$@"
+

Added: hadoop/chukwa/trunk/bin/fsm-clienttrace-mr.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/fsm-clienttrace-mr.sh?rev=887765&view=auto
==============================================================================
--- hadoop/chukwa/trunk/bin/fsm-clienttrace-mr.sh (added)
+++ hadoop/chukwa/trunk/bin/fsm-clienttrace-mr.sh Sun Dec  6 20:48:47 2009
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+${JAVA_HOME}/bin/java \
+-Djava.library.path=${JAVA_LIBRARY_PATH} \
+-DCHUKWA_HOME=${CHUKWA_HOME} \
+-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} \
+-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} \
+-DAPP=fsm \
+-Dlog4j.configuration=chukwa-log4j.properties \
+-classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_CONF_DIR}:${HADOOP_JAR}:${COMMON}:${tools}
\
+org.apache.hadoop.chukwa.analysis.salsa.fsm.FSMBuilder \
+-Dchukwa.salsa.fsm.mapclass=org.apache.hadoop.chukwa.analysis.salsa.fsm.TaskTrackerClientTraceMapper
\
+"$@"
+

Added: hadoop/chukwa/trunk/bin/fsm-jobhist.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/fsm-jobhist.sh?rev=887765&view=auto
==============================================================================
--- hadoop/chukwa/trunk/bin/fsm-jobhist.sh (added)
+++ hadoop/chukwa/trunk/bin/fsm-jobhist.sh Sun Dec  6 20:48:47 2009
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+${JAVA_HOME}/bin/java \
+-Djava.library.path=${JAVA_LIBRARY_PATH} \
+-DCHUKWA_HOME=${CHUKWA_HOME} \
+-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} \
+-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} \
+-DAPP=fsm \
+-Dlog4j.configuration=chukwa-log4j.properties \
+-classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_CONF_DIR}:${HADOOP_JAR}:${COMMON}:${tools}
\
+org.apache.hadoop.chukwa.analysis.salsa.fsm.FSMBuilder \
+-Dchukwa.salsa.fsm.mapclass=org.apache.hadoop.chukwa.analysis.salsa.fsm.JobHistoryTaskDataMapper
\
+"$@"
+

Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestMemBuffer.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestMemBuffer.java?rev=887765&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestMemBuffer.java
(added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestMemBuffer.java
Sun Dec  6 20:48:47 2009
@@ -0,0 +1,61 @@
+package org.apache.hadoop.chukwa.datacollection.adaptor;
+
+import static org.apache.hadoop.chukwa.util.TempFileUtil.makeTestFile;
+import java.io.File;
+import java.io.IOException;
+import junit.framework.TestCase;
+import org.apache.hadoop.chukwa.Chunk;
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.hadoop.chukwa.datacollection.connector.ChunkCatcherConnector;
+import org.apache.hadoop.conf.Configuration;
+
+public class TestMemBuffer extends TestCase {
+
+  Configuration conf = new Configuration();
+  static File baseDir;
+  File testFile;
+  ChunkCatcherConnector chunks;
+  
+  public TestMemBuffer() throws IOException {
+    baseDir = new File(System.getProperty("test.build.data", "/tmp"));
+    conf.setInt("chukwaAgent.control.port", 0);
+    conf.set("chukwaAgent.checkpoint.dir", baseDir.getCanonicalPath());
+    conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
+    conf.setInt("chukwaAgent.adaptor.fileadaptor.timeoutperiod", 100);
+    conf.setInt("chukwaAgent.adaptor.context.switch.time", 100);
+    testFile = makeTestFile(baseDir);
+
+    chunks = new ChunkCatcherConnector();
+    chunks.start();
+  }
+  
+  
+  //start a wrapped FileAdaptor. Pushes a chunk. Stop it and restart.
+  //chunk hasn't been acked, so should get pushed again.
+  //we delete the file and also change the data type each time through the loop
+  //to make sure we get the cached chunk.
+  public void testResendAfterStop()  throws IOException,
+  ChukwaAgent.AlreadyRunningException, InterruptedException {
+    
+    ChukwaAgent agent = new ChukwaAgent(conf);
+    
+    assertEquals(0, agent.adaptorCount());
+
+    for(int i=0; i< 5; ++i) {
+      String name =agent.processAddCommand("add adaptor_test = MemBuffered FileAdaptor raw"+i+
" "+testFile.getCanonicalPath() + " 0");
+      assertEquals(name, "adaptor_test");
+      Chunk c = chunks.waitForAChunk(5000);
+      assertNotNull(c);
+      String dat = new String(c.getData());
+      assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
+      assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
+      assertTrue(c.getDataType().equals("raw0"));
+      agent.stopAdaptor(name, true);
+      testFile.delete(); //file won't be there after the first time around.
+    }    
+    assertEquals(0, agent.adaptorCount());
+
+    agent.shutdown();
+  }
+
+}



Mime
View raw message