hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-15840: Webhcat test TestPig_5 failing with Pig on Tez at check for percent complete of job (Daniel Dai, reviewed by Thejas Nair)
Date Wed, 08 Feb 2017 03:44:01 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 f0b8f1786 -> 92730ca5b


HIVE-15840: Webhcat test TestPig_5 failing with Pig on Tez at check for percent complete of
job (Daniel Dai, reviewed by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92730ca5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92730ca5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92730ca5

Branch: refs/heads/branch-1
Commit: 92730ca5baeb56b809e25227ccfd6e670bd8bbea
Parents: f0b8f17
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Tue Feb 7 19:43:34 2017 -0800
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Tue Feb 7 19:43:34 2017 -0800

----------------------------------------------------------------------
 .../hcatalog/templeton/tool/TempletonUtils.java | 29 ++++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/92730ca5/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
index a7c6137..78a549e 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
@@ -118,7 +118,17 @@ public class TempletonUtils {
    * "(Map|Reducer) (\\d+:) ((-/-)|(\\d+(\\(\\+\\d+\\))?/\\d+))" is the complete pattern
but we'll drop "-/-" to exclude
    * groups that don't add information such as "Map 1: -/-"
    */
-  public static final Pattern TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:) (\\d+(\\(\\+\\d+\\))?/\\d+)");
+  public static final Pattern HIVE_TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:)
(\\d+(\\(\\+\\d+\\))?/\\d+)");
+  /**
+   * Pig on Tez produces progress report that looks like this
+   * DAG Status: status=RUNNING, progress=TotalTasks: 3 Succeeded: 0 Running: 0 Failed: 0
Killed: 0
+   *
+   * Use Succeeded/TotalTasks to report progress
+   * There is a hole as Pig might launch more than one DAGs. If this happens, user might
+   * see progress rewind since the percentage is for the new DAG. To fix this, We need to
fix
+   * Pig print total number of DAGs on console, and track complete DAGs in WebHCat.
+   */
+  public static final Pattern PIG_TEZ_COMPLETE = Pattern.compile("progress=TotalTasks: (\\d+)
Succeeded: (\\d+)");
   public static final Pattern TEZ_COUNTERS = Pattern.compile("\\d+");
 
   /**
@@ -137,14 +147,14 @@ public class TempletonUtils {
     if(hive.find()) {
       return "map " + hive.group(1) + " reduce " + hive.group(2);
     }
-    Matcher tez = TEZ_COMPLETE.matcher(line);
-    if(tez.find()) {
+    Matcher hiveTez = HIVE_TEZ_COMPLETE.matcher(line);
+    if(hiveTez.find()) {
       int totalTasks = 0;
       int completedTasks = 0;
       do {
         //here each group looks something like "Map 2: 2/4" "Reducer 3: 1(+2)/4"
         //just parse the numbers and ignore one from "Map 2" and from "(+2)" if it's there
-        Matcher counts = TEZ_COUNTERS.matcher(tez.group());
+        Matcher counts = TEZ_COUNTERS.matcher(hiveTez.group());
         List<String> items = new ArrayList<String>(4);
         while(counts.find()) {
           items.add(counts.group());
@@ -156,12 +166,21 @@ public class TempletonUtils {
         else {
           totalTasks += Integer.parseInt(items.get(3));
         }
-      } while(tez.find());
+      } while(hiveTez.find());
       if(totalTasks == 0) {
         return "0% complete (0 total tasks)";
       }
       return completedTasks * 100 / totalTasks + "% complete";
     }
+    Matcher pigTez = PIG_TEZ_COMPLETE.matcher(line);
+    if(pigTez.find()) {
+      int totalTasks = Integer.parseInt(pigTez.group(1));
+      int completedTasks = Integer.parseInt(pigTez.group(2));
+      if(totalTasks == 0) {
+          return "0% complete (0 total tasks)";
+        }
+        return completedTasks * 100 / totalTasks + "% complete";
+    }
     return null;
   }
 


Mime
View raw message