chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r781590 - in /hadoop/chukwa/trunk: ./ conf/ src/java/org/apache/hadoop/chukwa/datacollection/controller/ src/java/org/apache/hadoop/chukwa/inputtools/log4j/ src/test/org/apache/hadoop/chukwa/inputtools/log4j/
Date Wed, 03 Jun 2009 21:34:56 GMT
Author: eyang
Date: Wed Jun  3 21:34:54 2009
New Revision: 781590

URL: http://svn.apache.org/viewvc?rev=781590&view=rev
Log:
CHUKWA-267. Added TaskLogAppender for collecting Task Tracker Log file. (Eric Yang)

Added:
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties
Modified:
    hadoop/chukwa/trunk/build.xml
    hadoop/chukwa/trunk/conf/hadoop-log4j.properties
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java

Modified: hadoop/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/build.xml?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/build.xml (original)
+++ hadoop/chukwa/trunk/build.xml Wed Jun  3 21:34:54 2009
@@ -449,6 +449,7 @@
 		<copy todir="${test.build.dir}/classes/org/apache/hadoop/chukwa/inputtools/log4j/">
 			<fileset dir="${basedir}/src/test/org/apache/hadoop/chukwa/inputtools/log4j/">
 				<include name="late-log4j.properties" />
+				<include name="tasklog-log4j.properties" />
 			</fileset>
 		</copy>
 		<copy todir="${test.build.dir}/classes">
@@ -477,6 +478,7 @@
 			<env key="CHUKWA_CONF_DIR" value="${test.build.dir}/conf" />
 			<env key="CHUKWA_DATA_DIR" value="${test.build.dir}/var" />
 			<sysproperty key="test.src.dir" value="${test.src.dir}" />
+			<sysproperty key="test.build.classes" value="${test.build.classes}" />
 			<formatter type="${test.junit.output.format}" />
 			<batchtest todir="${test.build.dir}" unless="testcase">
 				<fileset dir="${test.src.dir}">

Modified: hadoop/chukwa/trunk/conf/hadoop-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/hadoop-log4j.properties?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/hadoop-log4j.properties (original)
+++ hadoop/chukwa/trunk/conf/hadoop-log4j.properties Wed Jun  3 21:34:54 2009
@@ -99,7 +99,8 @@
 hadoop.tasklog.purgeLogSplits=true
 hadoop.tasklog.logsRetainHours=12
 
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA=org.apache.hadoop.chukwa.inputtools.log4j.TaskLogAppender
+log4j.appender.TLA.recordType=TaskLog
 log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
 log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
Wed Jun  3 21:34:54 2009
@@ -186,6 +186,7 @@
   }
 
   Map<Long, ChukwaAgentController.Adaptor> runningAdaptors = new HashMap<Long, Adaptor>();
+  Map<Long, ChukwaAgentController.Adaptor> runningInstanceAdaptors = new HashMap<Long,
Adaptor>();
   Map<Long, ChukwaAgentController.Adaptor> pausedAdaptors;
   String hostname;
   int portno;
@@ -259,6 +260,7 @@
 
         if (adaptorID > 0) {
           runningAdaptors.put(adaptorID, adaptor);
+          runningInstanceAdaptors.put(adaptorID, adaptor);
         } else {
           System.err
               .println("Failed to successfully add the adaptor in AgentClient, adaptorID
returned by add() was negative.");
@@ -323,6 +325,21 @@
     }
   }
 
+  public void removeInstanceAdaptors() {
+    // Remove adaptors created by this instance of chukwa agent controller.
+    // Instead of removing using id, this is removed by using the stream name
+    // and record type.  This prevents the system to shutdown the wrong
+    // adaptor after agent crashes.
+    for (Adaptor a : runningInstanceAdaptors.values()) {
+      try {
+        remove(a.name, a.appType, a.params);
+      } catch (IOException ioe) {
+        System.out.println("Error removing an adaptor in removeInstanceAdaptors()");
+        ioe.printStackTrace();
+      }
+    }
+  }
+
   Map<Long, ChukwaAgentController.Adaptor> list() throws IOException {
     Socket s = new Socket(hostname, portno);
     try {

Added: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java
(added)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/controller/ClientFinalizer.java
Wed Jun  3 21:34:54 2009
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.datacollection.controller;
+
+import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.LogLog;
+
+
+public class ClientFinalizer extends Thread {
+  private ChukwaAgentController chukwaClient = null;
+  private String recordType = null;
+  private String fileName = null;
+
+  public ClientFinalizer(ChukwaAgentController chukwaClient) {
+    this.chukwaClient = chukwaClient;
+  }
+
+  public synchronized void run() {
+    try {
+      if (chukwaClient != null) {
+        chukwaClient.removeInstanceAdaptors();
+      } else {
+        LogLog.warn("chukwaClient is null cannot do any cleanup");
+      }
+    } catch (Throwable e) {
+      LogLog.warn("closing the controller threw an exception:\n" + e);
+    }
+  }
+}
+

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java?rev=781590&r1=781589&r2=781590&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Wed Jun  3 21:34:54 2009
@@ -23,6 +23,7 @@
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
+import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
 import org.apache.hadoop.chukwa.util.RecordConstants;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Layout;
@@ -460,33 +461,6 @@
     }
   }
 
-  private class ClientFinalizer extends Thread {
-    private ChukwaAgentController chukwaClient = null;
-    private String recordType = null;
-    private String fileName = null;
-
-    public ClientFinalizer(ChukwaAgentController chukwaClient,
-                           String recordType, String fileName) {
-      this.chukwaClient = chukwaClient;
-      this.recordType = recordType;
-      this.fileName = fileName;
-    }
-
-    public synchronized void run() {
-      try {
-        if (chukwaClient != null) {
-          log.debug("ShutdownHook: removing:" + fileName);
-          chukwaClient.removeFile(recordType, fileName);
-        } else {
-          LogLog.warn("chukwaClient is null cannot do any cleanup");
-        }
-      } catch (Throwable e) {
-        LogLog.warn("closing the controller threw an exception:\n" + e);
-      }
-    }
-  }
-
-  
   /**
    * Fix for late-initialisation
    */
@@ -573,8 +547,7 @@
                 numRetries, retryInterval);
 
             // Setup a shutdownHook for the controller
-            clientFinalizer = new ClientFinalizer(chukwaClient, recordType,
-                log4jFileName);
+            clientFinalizer = new ClientFinalizer(chukwaClient);
             Runtime.getRuntime().addShutdownHook(clientFinalizer);
 
             if (adaptorID > 0) {

Added: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java
(added)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/TaskLogAppender.java
Wed Jun  3 21:34:54 2009
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.inputtools.log4j;
+
+import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
+import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
+import org.apache.hadoop.chukwa.util.RecordConstants;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+
+public class TaskLogAppender extends 
+    org.apache.hadoop.mapred.TaskLogAppender {
+  static Logger log = Logger.getLogger(TaskLogAppender.class);
+  static final String adaptorType = ChukwaAgentController.CharFileTailUTF8NewLineEscaped;
+  ChukwaAgentController chukwaClient;
+  String recordType = null;
+  static boolean chukwaClientIsNull = true;
+  static final Object chukwaLock = new Object();
+  private ClientFinalizer clientFinalizer = null;
+
+  public String getRecordType() {
+    if (recordType != null)
+      return recordType;
+    else
+      return "unknown";
+  }
+
+  public void setRecordType(String recordType) {
+    this.recordType = recordType;
+  }
+
+  public void subAppend(LoggingEvent event) {
+      this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",this.layout.format(event)));
+      // Make sure only one thread can do this
+      // and use the boolean to avoid the first level locking
+      if (chukwaClientIsNull) {
+        synchronized (chukwaLock) {
+          if (chukwaClient == null) {
+            String log4jFileName = getFile();
+            String recordType = getRecordType();
+            long currentLength = 0L;
+            chukwaClient = new ChukwaAgentController();
+            chukwaClientIsNull = false;
+            long adaptorID = chukwaClient.add(ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
+              recordType,currentLength + " " + log4jFileName, currentLength);
+
+            // Setup a shutdownHook for the controller
+            clientFinalizer = new ClientFinalizer(chukwaClient);
+            Runtime.getRuntime().addShutdownHook(clientFinalizer);
+            if (adaptorID > 0) {
+              log.debug("Added file tailing adaptor to chukwa agent for file "
+                  + log4jFileName + ", adaptorId:" + adaptorID
+                  + " using this recordType :" + recordType
+                  + ", starting at offset:" + currentLength);
+            } else {
+              log.debug("Chukwa adaptor not added, addFile(" + log4jFileName
+                  + ") returned " + adaptorID
+                  + ", current offset:" + currentLength);
+            }
+          }
+        }
+      }
+  }
+}

Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java
(added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestTaskLogAppender.java
Wed Jun  3 21:34:54 2009
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.inputtools.log4j;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Date;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.LogLog;
+import org.apache.log4j.helpers.OptionConverter;
+
+public class TestTaskLogAppender extends TestCase {
+
+  @SuppressWarnings("deprecation")
+  public void testTaskLogAppender() {
+
+    String folder = System.getProperty("test.build.classes");
+    File logFile = new File(folder + "/userlogs/attempt_200905220200_13470_r_000000_0/syslog");
+    if (logFile.exists()) {
+      logFile.delete();
+    }
+    Assert.assertTrue("Log file should not be there", logFile.exists() == false);
+    File tempDir = new File(folder);
+    if (!tempDir.exists()) {
+      tempDir.mkdirs();
+    }
+    // load new log4j
+    String configuratorClassName = OptionConverter.getSystemProperty(
+        LogManager.CONFIGURATOR_CLASS_KEY, null);
+
+    URL url = TestTaskLogAppender.class
+        .getResource("tasklog-log4j.properties");
+
+    System.getProperties().setProperty("CHUKWA_LOG_DIR", folder);
+
+    if (url != null) {
+      LogLog
+          .debug("Using URL [" + url + "] for automatic log4j configuration.");
+      try {
+        OptionConverter.selectAndConfigure(url, configuratorClassName,
+            LogManager.getLoggerRepository());
+      } catch (NoClassDefFoundError e) {
+        LogLog.warn("Error during default initialization", e);
+      }
+    } else {
+      Assert.fail("URL should not be null");
+    }
+    Logger log = Logger.getLogger(TestTaskLogAppender.class);
+    try {
+      Thread.sleep(2000);
+    }catch (Exception e) {
+      // do nothing
+    }
+    log.warn("test 123 " + new Date());
+    Assert.assertTrue("Log file should exist", logFile.exists() == true);
+    logFile.delete();
+  }
+
+}

Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties?rev=781590&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties
(added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/inputtools/log4j/tasklog-log4j.properties
Wed Jun  3 21:34:54 2009
@@ -0,0 +1,11 @@
+log4j.rootLogger=INFO, R
+log4j.appender.R=org.apache.hadoop.chukwa.inputtools.log4j.TaskLogAppender
+log4j.appender.R.recordType=TaskLog
+log4j.appender.R.taskId=attempt_200905220200_13470_r_000000_0
+log4j.appender.R.totalLogFileSize=5000
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p %t %c{1} - %m%n



Mime
View raw message