hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-15840: Webhcat test TestPig_5 failing with Pig on Tez at check for percent complete of job (Daniel Dai, reviewed by Thejas Nair)
Date Wed, 08 Feb 2017 03:40:47 GMT
Repository: hive
Updated Branches:
  refs/heads/master 5a0a15f3d -> aa62dadb1


HIVE-15840: Webhcat test TestPig_5 failing with Pig on Tez at check for percent complete of
job (Daniel Dai, reviewed by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aa62dadb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aa62dadb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aa62dadb

Branch: refs/heads/master
Commit: aa62dadb1c56f254ada7cc64316450b38ae9eb54
Parents: 5a0a15f
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Tue Feb 7 19:39:15 2017 -0800
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Tue Feb 7 19:39:39 2017 -0800

----------------------------------------------------------------------
 .../hcatalog/templeton/tool/TempletonUtils.java | 30 ++++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/aa62dadb/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
index 0b6301c..030dec2 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
@@ -119,7 +118,17 @@ public class TempletonUtils {
    * "(Map|Reducer) (\\d+:) ((-/-)|(\\d+(\\(\\+\\d+\\))?/\\d+))" is the complete pattern
but we'll drop "-/-" to exclude
    * groups that don't add information such as "Map 1: -/-"
    */
-  public static final Pattern TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:) (\\d+(\\(\\+\\d+\\))?/\\d+)");
+  public static final Pattern HIVE_TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:)
(\\d+(\\(\\+\\d+\\))?/\\d+)");
+  /**
+   * Pig on Tez produces progress report that looks like this
+   * DAG Status: status=RUNNING, progress=TotalTasks: 3 Succeeded: 0 Running: 0 Failed: 0
Killed: 0
+   *
+   * Use Succeeded/TotalTasks to report progress
+   * There is a hole as Pig might launch more than one DAGs. If this happens, user might
+   * see progress rewind since the percentage is for the new DAG. To fix this, We need to
fix
+   * Pig print total number of DAGs on console, and track complete DAGs in WebHCat.
+   */
+  public static final Pattern PIG_TEZ_COMPLETE = Pattern.compile("progress=TotalTasks: (\\d+)
Succeeded: (\\d+)");
   public static final Pattern TEZ_COUNTERS = Pattern.compile("\\d+");
 
   /**
@@ -138,14 +147,14 @@ public class TempletonUtils {
     if(hive.find()) {
       return "map " + hive.group(1) + " reduce " + hive.group(2);
     }
-    Matcher tez = TEZ_COMPLETE.matcher(line);
-    if(tez.find()) {
+    Matcher hiveTez = HIVE_TEZ_COMPLETE.matcher(line);
+    if(hiveTez.find()) {
       int totalTasks = 0;
       int completedTasks = 0;
       do {
         //here each group looks something like "Map 2: 2/4" "Reducer 3: 1(+2)/4"
         //just parse the numbers and ignore one from "Map 2" and from "(+2)" if it's there
-        Matcher counts = TEZ_COUNTERS.matcher(tez.group());
+        Matcher counts = TEZ_COUNTERS.matcher(hiveTez.group());
         List<String> items = new ArrayList<String>(4);
         while(counts.find()) {
           items.add(counts.group());
@@ -157,12 +166,21 @@ public class TempletonUtils {
         else {
           totalTasks += Integer.parseInt(items.get(3));
         }
-      } while(tez.find());
+      } while(hiveTez.find());
       if(totalTasks == 0) {
         return "0% complete (0 total tasks)";
       }
       return completedTasks * 100 / totalTasks + "% complete";
     }
+    Matcher pigTez = PIG_TEZ_COMPLETE.matcher(line);
+    if(pigTez.find()) {
+      int totalTasks = Integer.parseInt(pigTez.group(1));
+      int completedTasks = Integer.parseInt(pigTez.group(2));
+      if(totalTasks == 0) {
+          return "0% complete (0 total tasks)";
+        }
+        return completedTasks * 100 / totalTasks + "% complete";
+    }
     return null;
   }
 


Mime
View raw message