hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject hive git commit: HIVE-13488: Restore dag summary when tez exec print summary enabled and in-place updates disabled (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Date Thu, 21 Apr 2016 20:58:23 GMT
Repository: hive
Updated Branches:
  refs/heads/master 1ec276fdb -> aac9263b4


HIVE-13488: Restore dag summary when tez exec print summary enabled and in-place updates disabled
(Prasanth Jayachandran reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aac9263b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aac9263b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aac9263b

Branch: refs/heads/master
Commit: aac9263b4c14ca1986c297e6601a1742259fc65e
Parents: 1ec276f
Author: Prasanth Jayachandran <prasanthj@apache.org>
Authored: Thu Apr 21 15:58:13 2016 -0500
Committer: Prasanth Jayachandran <prasanthj@apache.org>
Committed: Thu Apr 21 15:58:13 2016 -0500

----------------------------------------------------------------------
 .../hadoop/hive/ql/exec/tez/TezJobMonitor.java  | 62 ++++++++++++++++----
 1 file changed, 49 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/aac9263b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index 0c635b2..b22991c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -72,7 +72,10 @@ public class TezJobMonitor {
 
   private static final int COLUMN_1_WIDTH = 16;
   private static final int SEPARATOR_WIDTH = InPlaceUpdates.MIN_TERMINAL_WIDTH;
+  private static final int FILE_HEADER_SEPARATOR_WIDTH = InPlaceUpdates.MIN_TERMINAL_WIDTH
+ 34;
   private static final String SEPARATOR = new String(new char[SEPARATOR_WIDTH]).replace("\0",
"-");
+  private static final String FILE_HEADER_SEPARATOR =
+      new String(new char[FILE_HEADER_SEPARATOR_WIDTH]).replace("\0", "-");
   private static final String QUERY_EXEC_SUMMARY_HEADER = "Query Execution Summary";
   private static final String TASK_SUMMARY_HEADER = "Task Execution Summary";
   private static final String LLAP_IO_SUMMARY_HEADER = "LLAP IO Summary";
@@ -90,6 +93,12 @@ public class TezJobMonitor {
   private static final String SUMMARY_HEADER = String.format(SUMMARY_HEADER_FORMAT,
       "VERTICES", "DURATION(ms)", "CPU_TIME(ms)", "GC_TIME(ms)", "INPUT_RECORDS", "OUTPUT_RECORDS");
 
+  // used when I/O redirection is used
+  private static final String FILE_HEADER_FORMAT = "%10s %12s %16s %13s %14s %13s %12s %14s
%15s";
+  private static final String FILE_HEADER = String.format(FILE_HEADER_FORMAT,
+      "VERTICES", "TOTAL_TASKS", "FAILED_ATTEMPTS", "KILLED_TASKS", "DURATION(ms)",
+      "CPU_TIME(ms)", "GC_TIME(ms)", "INPUT_RECORDS", "OUTPUT_RECORDS");
+
   // LLAP counters
   private static final String LLAP_SUMMARY_HEADER_FORMAT = "%10s %9s %9s %10s %9s %10s %11s
%8s %9s";
   private static final String LLAP_SUMMARY_HEADER = String.format(LLAP_SUMMARY_HEADER_FORMAT,
@@ -365,8 +374,12 @@ public class TezJobMonitor {
       console.printInfo("");
 
       console.printInfo(TASK_SUMMARY_HEADER);
-      printDagSummary(progressMap, console, dagClient, conf, dag);
-      console.printInfo(SEPARATOR);
+      printDagSummary(progressMap, console, dagClient, conf, dag, inPlaceEligible);
+      if (inPlaceEligible) {
+        console.printInfo(SEPARATOR);
+      } else {
+        console.printInfo(FILE_HEADER_SEPARATOR);
+      }
 
       if (llapIoEnabled) {
         console.printInfo("");
@@ -462,7 +475,7 @@ public class TezJobMonitor {
   }
 
   private void printDagSummary(Map<String, Progress> progressMap, LogHelper console,
-      DAGClient dagClient, HiveConf conf, DAG dag) {
+      DAGClient dagClient, HiveConf conf, DAG dag, final boolean inPlaceEligible) {
 
     /* Strings for headers and counters */
     String hiveCountersGroup = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
@@ -482,15 +495,24 @@ public class TezJobMonitor {
     }
 
     /* Print the per Vertex summary */
-    console.printInfo(SEPARATOR);
-    reprintLineWithColorAsBold(SUMMARY_HEADER, Ansi.Color.CYAN);
-    console.printInfo(SEPARATOR);
+    if (inPlaceEligible) {
+      console.printInfo(SEPARATOR);
+      reprintLineWithColorAsBold(SUMMARY_HEADER, Ansi.Color.CYAN);
+      console.printInfo(SEPARATOR);
+    } else {
+      console.printInfo(FILE_HEADER_SEPARATOR);
+      reprintLineWithColorAsBold(FILE_HEADER, Ansi.Color.CYAN);
+      console.printInfo(FILE_HEADER_SEPARATOR);
+    }
     SortedSet<String> keys = new TreeSet<String>(progressMap.keySet());
     Set<StatusGetOpts> statusOptions = new HashSet<StatusGetOpts>(1);
     statusOptions.add(StatusGetOpts.GET_COUNTERS);
     for (String vertexName : keys) {
       Progress progress = progressMap.get(vertexName);
       if (progress != null) {
+        final int totalTasks = progress.getTotalTaskCount();
+        final int failedTaskAttempts = progress.getFailedTaskAttemptCount();
+        final int killedTaskAttempts = progress.getKilledTaskAttemptCount();
         final double duration = perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName);
         VertexStatus vertexStatus = null;
         try {
@@ -572,13 +594,27 @@ public class TezJobMonitor {
                     + vertexName.replace(" ", "_"))
                 + hiveOutputIntermediateRecords;
 
-        String vertexExecutionStats = String.format(SUMMARY_HEADER_FORMAT,
-            vertexName,
-            secondsFormat.format((duration)),
-            commaFormat.format(cpuTimeMillis),
-            commaFormat.format(gcTimeMillis),
-            commaFormat.format(hiveInputRecords),
-            commaFormat.format(hiveOutputRecords));
+        final String vertexExecutionStats;
+        if (inPlaceEligible) {
+          vertexExecutionStats = String.format(SUMMARY_HEADER_FORMAT,
+              vertexName,
+              secondsFormat.format((duration)),
+              commaFormat.format(cpuTimeMillis),
+              commaFormat.format(gcTimeMillis),
+              commaFormat.format(hiveInputRecords),
+              commaFormat.format(hiveOutputRecords));
+        } else {
+          vertexExecutionStats = String.format(FILE_HEADER_FORMAT,
+              vertexName,
+              totalTasks,
+              failedTaskAttempts,
+              killedTaskAttempts,
+              secondsFormat.format((duration)),
+              commaFormat.format(cpuTimeMillis),
+              commaFormat.format(gcTimeMillis),
+              commaFormat.format(hiveInputRecords),
+              commaFormat.format(hiveOutputRecords));
+        }
         console.printInfo(vertexExecutionStats);
       }
     }


Mime
View raw message