tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject git commit: TEZ-1424. Fixes to DAG text representation in debug mode (Rajesh Balamohan)
Date Thu, 23 Oct 2014 22:33:46 GMT
Repository: tez
Updated Branches:
  refs/heads/master d523db658 -> 2266cccf9


TEZ-1424. Fixes to DAG text representation in debug mode  (Rajesh Balamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2266cccf
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2266cccf
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2266cccf

Branch: refs/heads/master
Commit: 2266cccf9290601662944bdfa17ef5a2e96e9371
Parents: d523db6
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Fri Oct 24 04:02:46 2014 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Fri Oct 24 04:02:46 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/common/TezCommonUtils.java   | 30 +++++++++-----
 .../apache/tez/dag/api/TezConfiguration.java    |  4 +-
 .../org/apache/tez/common/TezUtilsInternal.java | 28 +++++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    | 41 ++++++++++----------
 5 files changed, 73 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/2266cccf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1c8ed15..9a4e19a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -61,6 +61,7 @@ ALL CHANGES:
   TEZ-1267. Exception handling for VertexManager.
   TEZ-1688. Add applicationId as a primary filter for all Timeline data for easier export.
   TEZ-1141. DAGStatus.Progress should include number of failed and killed attempts.
+  TEZ-1424. Fixes to DAG text representation in debug mode.
 
 Release 0.5.1: 2014-10-02
 

http://git-wip-us.apache.org/repos/asf/tez/blob/2266cccf/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
index 04d2bfc..345177a 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
@@ -21,6 +21,7 @@ package org.apache.tez.common;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.StringTokenizer;
@@ -336,17 +337,28 @@ public class TezCommonUtils {
     return bytes;
   }
 
+  public static String getCredentialsInfo(Credentials credentials, String identifier) {
+    StringBuilder sb = new StringBuilder();
+    sb.append("Credentials: #" + identifier + "Tokens=").append(credentials.numberOfTokens());
+    if (credentials.numberOfTokens() > 0) {
+      sb.append(", Services=");
+      Iterator<Token<?>> tokenItr = credentials.getAllTokens().iterator();
+      if (tokenItr.hasNext()) {
+        Token token = tokenItr.next();
+        sb.append(token.getService()).append("(").append(token.getKind()).append(")");
+
+      }
+      while(tokenItr.hasNext()) {
+        Token token = tokenItr.next();
+        sb.append(",").append(token.getService()).append("(").append(token.getKind()).append(")");
+      }
+    }
+    return sb.toString();
+  }
+
   public static void logCredentials(Log log, Credentials credentials, String identifier)
{
     if (log.isDebugEnabled()) {
-      StringBuilder sb = new StringBuilder();
-      sb.append("#" + identifier + "Tokens=").append(credentials.numberOfTokens());
-      if (credentials.numberOfTokens() > 0) {
-        sb.append(", Services: ");
-        for (Token<?> t : credentials.getAllTokens()) {
-          sb.append(t.getService()).append(",");
-        }
-      }
-      log.debug(sb.toString());
+      log.debug(getCredentialsInfo(credentials, identifier));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/2266cccf/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 3dfe32b..5bb565b 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -649,11 +649,11 @@ public class TezConfiguration extends Configuration {
 
   @Unstable
   /**
-   * Boolean value. Generate debug artifacts like DAG visualization.
+   * Boolean value. Generate debug artifacts like DAG plan in text format.
    */
   public static final String TEZ_GENERATE_DEBUG_ARTIFACTS =
       TEZ_PREFIX + "generate.debug.artifacts";
-  public static final boolean TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT = true;
+  public static final boolean TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT = false;
 
   /**
    * Set of tasks for which specific launch command options need to be added.

http://git-wip-us.apache.org/repos/asf/tez/blob/2266cccf/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
index e2010b9..2ca153c 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
@@ -24,6 +24,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.BitSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -31,15 +32,21 @@ import java.util.zip.DataFormatException;
 import java.util.zip.Deflater;
 import java.util.zip.Inflater;
 
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.TextFormat;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.log4j.Appender;
 import org.apache.log4j.Logger;
+import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.records.DAGProtos;
 import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
 import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
 
@@ -221,4 +228,25 @@ public class TezUtilsInternal {
     return logDirs[logIndex];
   }
 
+  /**
+   * Convert DAGPlan to text. Skip sensitive informations like credentials.
+   *
+   * @param dagPlan
+   * @return
+   */
+  public static String convertDagPlanToString(DAGProtos.DAGPlan dagPlan) throws IOException
{
+    StringBuilder sb = new StringBuilder();
+    for (Map.Entry<Descriptors.FieldDescriptor, Object> entry : dagPlan.getAllFields().entrySet())
{
+      if (entry.getKey().getNumber() != DAGProtos.DAGPlan.CREDENTIALS_BINARY_FIELD_NUMBER)
{
+        TextFormat.printField(entry.getKey(), entry.getValue(), sb);
+      } else {
+        Credentials credentials =
+            DagTypeConverters.convertByteStringToCredentials(dagPlan.getCredentialsBinary());
+        TextFormat.printField(entry.getKey(),
+            ByteString.copyFrom(TezCommonUtils.getCredentialsInfo(credentials,"dag").getBytes()),
sb);
+      }
+    }
+    return sb.toString();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/2266cccf/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 321fb01..407ede1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -714,11 +714,8 @@ public class DAGAppMaster extends AbstractService {
       LOG.warn("Failed to generate json for DAG", e);
     }
 
-    if (dagConf.getBoolean(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS,
-        TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT)) {
-      generateDAGVizFile(dagId, dagPB);
-    }
-
+    generateDAGVizFile(dagId, dagPB);
+    writePBTextFile(newDag);
     return newDag;
   } // end createDag()
 
@@ -800,6 +797,25 @@ public class DAGAppMaster extends AbstractService {
     }
   }
 
+  private void writePBTextFile(DAG dag) {
+    if (dag.getConf().getBoolean(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS,
+        TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT)) {
+
+      String logFile = TezUtilsInternal.getContainerLogDir() + File.separatorChar + dag.getID().toString()
+          + "-" + TezConstants.TEZ_PB_PLAN_TEXT_NAME;
+
+      LOG.info("Writing DAG plan to: " + logFile);
+      File outFile = new File(logFile);
+      try {
+        PrintWriter printWriter = new PrintWriter(outFile);
+        printWriter.println(TezUtilsInternal.convertDagPlanToString(dag.getJobPlan()));
+        printWriter.close();
+      } catch (IOException e) {
+        LOG.warn("Failed to write TEZ_PLAN to " + outFile.toString(), e);
+      }
+    }
+  }
+
   protected void addIfService(Object object, boolean addDispatcher) {
     if (object instanceof Service) {
       Service service = (Service) object;
@@ -1006,22 +1022,7 @@ public class DAGAppMaster extends AbstractService {
     
     if (LOG.isDebugEnabled()) {
       LOG.debug("Invoked with additional local resources: " + additionalResources);
-      
-      LOG.debug("Writing DAG plan to: "
-          + TezConstants.TEZ_PB_PLAN_TEXT_NAME);
-
-      File outFile = new File(TezConstants.TEZ_PB_PLAN_TEXT_NAME);
-      try {
-        PrintWriter printWriter = new PrintWriter(outFile);
-        String dagPbString = dagPlan.toString();
-        printWriter.println(dagPbString);
-        printWriter.close();
-      } catch (IOException e) {
-        throw new TezException("Failed to write TEZ_PLAN to "
-            + outFile.toString(), e);
-      }
     }
-
     submittedDAGs.incrementAndGet();
     startDAG(dagPlan, additionalResources);
     return currentDAG.getID().toString();


Mime
View raw message