hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [2/2] hive git commit: HIVE-12357 : Allow user to set tez job name (Gunther Hagleitner, reviewed by Vikram Dixit K, Sergey Shelukhin)
Date Wed, 27 Jan 2016 21:51:18 GMT
HIVE-12357 : Allow user to set tez job name (Gunther Hagleitner, reviewed by Vikram Dixit K,
Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/36f8b759
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/36f8b759
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/36f8b759

Branch: refs/heads/branch-2.0
Commit: 36f8b7590998d5ad95f8fed229469e447883feb7
Parents: e83dd41
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Jan 27 13:45:38 2016 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Jan 27 13:51:04 2016 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/conf/HiveConf.java  |  3 +++
 .../apache/hadoop/hive/ql/exec/tez/DagUtils.java    | 16 ++++++++++++++++
 .../org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 ++++++----
 .../org/apache/hadoop/hive/ql/hooks/ATSHook.java    |  4 +++-
 4 files changed, 28 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 14b86e3..7a25ece 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1198,6 +1198,9 @@ public class HiveConf extends Configuration {
     HIVETEZLOGLEVEL("hive.tez.log.level", "INFO",
         "The log level to use for tasks executing as part of the DAG.\n" +
         "Used only if hive.tez.java.opts is used to configure Java options."),
+    HIVEQUERYNAME ("hive.query.name", null,
+        "This named is used by Tez to set the dag name. This name in turn will appear on
\n" +
+        "the Tez UI representing the work that was done."),
 
     HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", true,
         "Don't create a reducer for enforcing \n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index e8864ae..63fb37a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
@@ -1226,6 +1227,21 @@ public class DagUtils {
     }
   }
 
+  public String createDagName(Configuration conf, QueryPlan plan) {
+    String name = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYNAME);
+
+    if (name == null) {
+      name = conf.get("mapred.job.name");
+    }
+
+    if (name == null) {
+      name = plan.getQueryId();
+    }
+
+    assert name != null;
+    return name;
+  }
+
   private DagUtils() {
     // don't instantiate
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 3cb7439..83defea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -176,9 +176,8 @@ public class TezTask extends Task<TezWork> {
       if (driverContext.getCtx() == null) {
         boolean a = false;
       }
-      CallerContext callerContext = CallerContext.create("HIVE",
-          conf.getLogIdVar(SessionState.get().getSessionId()) + " "
-              + conf.getVar(HiveConf.ConfVars.HIVEQUERYID),
+      CallerContext callerContext = CallerContext.create(
+          "HIVE", queryPlan.getQueryId(),
           "HIVE_QUERY_ID", queryPlan.getQueryStr());
       dag.setCallerContext(callerContext);
 
@@ -320,7 +319,10 @@ public class TezTask extends Task<TezWork> {
     FileSystem fs = scratchDir.getFileSystem(conf);
 
     // the name of the dag is what is displayed in the AM/Job UI
-    DAG dag = DAG.create(work.getName());
+    String dagName = utils.createDagName(conf, queryPlan);
+
+    LOG.info("Dag name: " + dagName);
+    DAG dag = DAG.create(dagName);
 
     // set some info for the query
     JSONObject json = new JSONObject(new LinkedHashMap()).put("context", "Hive")

http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 0be8b3c..f490161 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -50,13 +50,14 @@ public class ATSHook implements ExecuteWithHookContext {
 
   private static final Logger LOG = LoggerFactory.getLogger(ATSHook.class.getName());
   private static final Object LOCK = new Object();
+  private static final int VERSION = 2;
   private static ExecutorService executor;
   private static TimelineClient timelineClient;
   private enum EntityTypes { HIVE_QUERY_ID };
   private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
 
   private enum OtherInfoTypes {
-    QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME
+    QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME, VERSION
   };
   private enum PrimaryFilterTypes { user, requestuser, operationid };
   private static final int WAIT_TIME = 3;
@@ -193,6 +194,7 @@ public class ATSHook implements ExecuteWithHookContext {
     atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0);
     atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID);
     atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), Thread.currentThread().getName());
+    atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION);
     return atsEntity;
   }
 


Mime
View raw message