tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-535. Redirect AM logs to different files when running multiple DAGs in the same AM. Contributed by Mohammad Kamrul Islam.
Date Fri, 28 Feb 2014 02:49:50 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master 74078d309 -> 6d29107ce


TEZ-535. Redirect AM logs to different files when running multiple DAGs
in the same AM. Contributed by  Mohammad Kamrul Islam.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6d29107c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6d29107c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6d29107c

Branch: refs/heads/master
Commit: 6d29107ceef4f7972a4f134caaf8e93307b99fc7
Parents: 74078d3
Author: Siddharth Seth <sseth@apache.org>
Authored: Thu Feb 27 18:49:07 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu Feb 27 18:49:07 2014 -0800

----------------------------------------------------------------------
 .../tez/common/TezContainerLogAppender.java     | 79 ++++++++++++++++++++
 .../java/org/apache/tez/common/TezUtils.java    | 45 +++++++++++
 .../hadoop/mapred/TezContainerLogAppender.java  | 79 --------------------
 .../apache/hadoop/mapred/YarnTezDagChild.java   | 45 +----------
 .../org/apache/tez/dag/app/DAGAppMaster.java    | 20 ++++-
 .../resources/tez-container-log4j.properties    |  2 +-
 6 files changed, 145 insertions(+), 125 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java
new file mode 100644
index 0000000..a6455c3
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.io.File;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.log4j.FileAppender;
+import org.apache.tez.dag.api.TezConfiguration;
+
+/**
+ * A simple log4j-appender for a tez container's logs.
+ * 
+ */
+@Public
+@Unstable
+public class TezContainerLogAppender extends FileAppender {
+  private String containerLogDir;
+  private String logFileName = TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME;
+  //so that log4j can configure it from the configuration(log4j.properties). 
+
+  @Override
+  public void activateOptions() {
+    synchronized (this) {
+      setFile(new File(this.containerLogDir, logFileName).toString());
+      setAppend(true);
+      super.activateOptions();
+    }
+  }
+
+  /**
+   * Set the name of the file for logging. This should NOT be an absolute path.
+   * The file will be created within the container's log directory.
+   * 
+   * @param fileName
+   */
+  public void setLogFileName(String fileName) {
+    if (fileName == null || fileName.contains(File.pathSeparator)) {
+      throw new RuntimeException(
+          "Invalid filename specified: "
+              + fileName
+              + " . FileName should not have a path component and should not be empty.");
+    }
+    this.logFileName = fileName;
+  }
+
+  public String getLogFileName() {
+    return logFileName;
+  }
+
+  /**
+   * Getter/Setter methods for log4j.
+   */
+
+  public String getContainerLogDir() {
+    return this.containerLogDir;
+  }
+
+  public void setContainerLogDir(String containerLogDir) {
+    this.containerLogDir = containerLogDir;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtils.java b/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
index 3df9adb..c0f5528 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezUtils.java
@@ -18,9 +18,12 @@
 package org.apache.tez.common;
 
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map.Entry;
@@ -33,6 +36,8 @@ import java.util.zip.InflaterInputStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
 import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
@@ -216,4 +221,44 @@ public class TezUtils {
     byte[] output = bos.toByteArray();
     return output;
   }
+
+  public static void updateLoggers(String addend) throws FileNotFoundException {
+    String containerLogDir = null;
+
+    LOG.info("Redirecting log files based on addend: " + addend);
+
+    Appender appender = Logger.getRootLogger().getAppender(
+        TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
+    if (appender != null) {
+      if (appender instanceof TezContainerLogAppender) {
+        TezContainerLogAppender claAppender = (TezContainerLogAppender) appender;
+        containerLogDir = claAppender.getContainerLogDir();
+        claAppender.setLogFileName(constructLogFileName(
+            TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME, addend));
+        claAppender.activateOptions();
+      } else {
+        LOG.warn("Appender is a " + appender.getClass() + "; require an instance of "
+            + TezContainerLogAppender.class.getName() + " to reconfigure the logger output");
+      }
+    } else {
+      LOG.warn("Not configured with appender named: " + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME
+          + ". Cannot reconfigure logger output");
+    }
+
+    if (containerLogDir != null) {
+      System.setOut(new PrintStream(new File(containerLogDir, constructLogFileName(
+          TezConfiguration.TEZ_CONTAINER_OUT_FILE_NAME, addend))));
+      System.setErr(new PrintStream(new File(containerLogDir, constructLogFileName(
+          TezConfiguration.TEZ_CONTAINER_ERR_FILE_NAME, addend))));
+    }
+  }
+
+  private static String constructLogFileName(String base, String addend) {
+    if (addend == null || addend.isEmpty()) {
+      return base;
+    } else {
+      return base + "_" + addend;
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java
deleted file mode 100644
index cd5f4c9..0000000
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezContainerLogAppender.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.log4j.FileAppender;
-import org.apache.tez.dag.api.TezConfiguration;
-
-/**
- * A simple log4j-appender for a tez container's logs.
- * 
- */
-@Public
-@Unstable
-public class TezContainerLogAppender extends FileAppender {
-  private String containerLogDir;
-  private String logFileName = TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME;
-  //so that log4j can configure it from the configuration(log4j.properties). 
-
-  @Override
-  public void activateOptions() {
-    synchronized (this) {
-      setFile(new File(this.containerLogDir, logFileName).toString());
-      setAppend(true);
-      super.activateOptions();
-    }
-  }
-
-  /**
-   * Set the name of the file for logging. This should NOT be an absolute path.
-   * The file will be created within the container's log directory.
-   * 
-   * @param fileName
-   */
-  public void setLogFileName(String fileName) {
-    if (fileName == null || fileName.contains(File.pathSeparator)) {
-      throw new RuntimeException(
-          "Invalid filename specified: "
-              + fileName
-              + " . FileName should not have a path component and should not be empty.");
-    }
-    this.logFileName = fileName;
-  }
-
-  public String getLogFileName() {
-    return logFileName;
-  }
-
-  /**
-   * Getter/Setter methods for log4j.
-   */
-
-  public String getContainerLogDir() {
-    return this.containerLogDir;
-  }
-
-  public void setContainerLogDir(String containerLogDir) {
-    this.containerLogDir = containerLogDir;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 1288369..81d211a 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -679,50 +679,13 @@ public class YarnTezDagChild {
 
   private static void updateLoggers(TezTaskAttemptID tezTaskAttemptID)
       throws FileNotFoundException {
-    String containerLogDir = null;
-
-    LOG.info("Redirecting log files based on TaskAttemptId: " + tezTaskAttemptID);
-
-    Appender appender = Logger.getRootLogger().getAppender(
-        TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
-    if (appender != null) {
-      if (appender instanceof TezContainerLogAppender) {
-        TezContainerLogAppender claAppender = (TezContainerLogAppender) appender;
-        containerLogDir = claAppender.getContainerLogDir();
-        claAppender.setLogFileName(constructLogFileName(
-            TezConfiguration.TEZ_CONTAINER_LOG_FILE_NAME, tezTaskAttemptID));
-        claAppender.activateOptions();
-      } else {
-        LOG.warn("Appender is a " + appender.getClass()
-            + "; require an instance of "
-            + TezContainerLogAppender.class.getName()
-            + " to reconfigure the logger output");
-      }
-    } else {
-      LOG.warn("Not configured with appender named: "
-          + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME
-          + ". Cannot reconfigure logger output");
-    }
-
-    if (containerLogDir != null) {
-      System.setOut(new PrintStream(new File(containerLogDir,
-          constructLogFileName(TezConfiguration.TEZ_CONTAINER_OUT_FILE_NAME,
-              tezTaskAttemptID))));
-      System.setErr(new PrintStream(new File(containerLogDir,
-          constructLogFileName(TezConfiguration.TEZ_CONTAINER_ERR_FILE_NAME,
-              tezTaskAttemptID))));
+    String addend = "";
+    if (tezTaskAttemptID != null) {
+      addend = tezTaskAttemptID.toString();
     }
+    TezUtils.updateLoggers(addend);
   }
 
-  private static String constructLogFileName(String base,
-      TezTaskAttemptID tezTaskAttemptID) {
-    if (tezTaskAttemptID == null) {
-      return base;
-    } else {
-      return base + "_" + tezTaskAttemptID.toString();
-    }
-  }
-  
   private static void processAdditionalResources(Map<String, TezLocalResource> additionalResources,
       Configuration conf) throws IOException, TezException {
     if (additionalResources == null || additionalResources.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 6098289..a481aaa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -20,6 +20,7 @@ package org.apache.tez.dag.app;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.security.PrivilegedExceptionAction;
@@ -400,6 +401,7 @@ public class DAGAppMaster extends AbstractService {
     case INTERNAL_ERROR:
       state = DAGAppMasterState.ERROR;
       if(currentDAG != null) {
+        _updateLoggers(currentDAG, "_post");
         // notify dag to finish which will send the DAG_FINISHED event
         LOG.info("Internal Error. Notifying dags to finish.");
         sendEvent(new DAGEvent(currentDAG.getID(), DAGEventType.INTERNAL_ERROR));
@@ -412,6 +414,7 @@ public class DAGAppMaster extends AbstractService {
       DAGAppMasterEventDAGFinished finishEvt =
           (DAGAppMasterEventDAGFinished) event;
       if (!isSession) {
+        _updateLoggers(currentDAG, "_post");
         setStateOnDAGCompletion();
         LOG.info("Shutting down on completion of dag:" +
               finishEvt.getDAGId().toString());
@@ -421,6 +424,7 @@ public class DAGAppMaster extends AbstractService {
             + finishEvt.getDAGId().toString()
             + ", dagState=" + finishEvt.getDAGState());
         lastDAGCompletionTime = clock.getTime();
+        _updateLoggers(currentDAG, "_post");
         switch(finishEvt.getDAGState()) {
         case SUCCEEDED:
           if (!currentDAG.getName().startsWith(
@@ -468,6 +472,14 @@ public class DAGAppMaster extends AbstractService {
     }
   }
 
+  private void _updateLoggers(DAG dag, String appender) {
+    try {
+      TezUtils.updateLoggers(dag.getID().toString() + appender);
+    } catch (FileNotFoundException e) {
+      LOG.warn("Unable to update the logger. Continue with the old logger", e );
+    }
+  }
+
   private class DAGAppMasterEventHandler implements
       EventHandler<DAGAppMasterEvent> {
     @Override
@@ -1581,6 +1593,10 @@ public class DAGAppMaster extends AbstractService {
   private void startDAG(DAGPlan dagPlan) {
     long submitTime = this.clock.getTime();
     this.state = DAGAppMasterState.RUNNING;
+    this.appName = dagPlan.getName();
+    // /////////////////// Create the job itself.
+    DAG newDAG = createDAG(dagPlan);
+    _updateLoggers(newDAG, "");
     if (LOG.isDebugEnabled()) {
       LOG.debug("Running a DAG with " + dagPlan.getVertexCount()
           + " vertices ");
@@ -1592,10 +1608,6 @@ public class DAGAppMaster extends AbstractService {
     LOG.info("Running DAG: " + dagPlan.getName());
     // Job name is the same as the app name until we support multiple dags
     // for an app later
-    appName = dagPlan.getName();
-
-    // /////////////////// Create the job itself.
-    DAG newDAG = createDAG(dagPlan);
     DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(newDAG.getID(),
         submitTime, dagPlan, this.appAttemptID);
     historyEventHandler.handle(new DAGHistoryEvent(newDAG.getID(), submittedEvent));

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d29107c/tez-dag/src/main/resources/tez-container-log4j.properties
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/resources/tez-container-log4j.properties b/tez-dag/src/main/resources/tez-container-log4j.properties
index 2bbfad8..628d3b5 100644
--- a/tez-dag/src/main/resources/tez-container-log4j.properties
+++ b/tez-dag/src/main/resources/tez-container-log4j.properties
@@ -24,7 +24,7 @@ log4j.threshold=ALL
 # ContainerLog Appender
 #
 
-log4j.appender.CLA=org.apache.hadoop.mapred.TezContainerLogAppender
+log4j.appender.CLA=org.apache.tez.common.TezContainerLogAppender
 log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir}
 
 log4j.appender.CLA.layout=org.apache.log4j.PatternLayout


Mime
View raw message