hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [1/2] hive git commit: HIVE-12357 : Allow user to set tez job name (Gunther Hagleitner, reviewed by Vikram Dixit K, Sergey Shelukhin)
Date Wed, 27 Jan 2016 21:51:17 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 e83dd4163 -> 36f8b7590
  refs/heads/master 01d06aa83 -> c0b04f7b0


HIVE-12357 : Allow user to set tez job name (Gunther Hagleitner, reviewed by Vikram Dixit
K, Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0b04f7b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0b04f7b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0b04f7b

Branch: refs/heads/master
Commit: c0b04f7b033cc4f2e7b2c105f0a5d0b2d0f9fb75
Parents: 01d06aa
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Jan 27 13:45:38 2016 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Jan 27 13:45:38 2016 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/conf/HiveConf.java  |  3 +++
 .../apache/hadoop/hive/ql/exec/tez/DagUtils.java    | 16 ++++++++++++++++
 .../org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 ++++++----
 .../org/apache/hadoop/hive/ql/hooks/ATSHook.java    |  4 +++-
 4 files changed, 28 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 74a8749..125d40c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1207,6 +1207,9 @@ public class HiveConf extends Configuration {
     HIVETEZLOGLEVEL("hive.tez.log.level", "INFO",
         "The log level to use for tasks executing as part of the DAG.\n" +
         "Used only if hive.tez.java.opts is used to configure Java options."),
+    HIVEQUERYNAME ("hive.query.name", null,
+        "This named is used by Tez to set the dag name. This name in turn will appear on
\n" +
+        "the Tez UI representing the work that was done."),
 
     HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", true,
         "Don't create a reducer for enforcing \n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index aa44d5f..319447b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
@@ -1261,6 +1262,21 @@ public class DagUtils {
     }
   }
 
+  public String createDagName(Configuration conf, QueryPlan plan) {
+    String name = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYNAME);
+
+    if (name == null) {
+      name = conf.get("mapred.job.name");
+    }
+
+    if (name == null) {
+      name = plan.getQueryId();
+    }
+
+    assert name != null;
+    return name;
+  }
+
   private DagUtils() {
     // don't instantiate
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 3cb7439..83defea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -176,9 +176,8 @@ public class TezTask extends Task<TezWork> {
       if (driverContext.getCtx() == null) {
         boolean a = false;
       }
-      CallerContext callerContext = CallerContext.create("HIVE",
-          conf.getLogIdVar(SessionState.get().getSessionId()) + " "
-              + conf.getVar(HiveConf.ConfVars.HIVEQUERYID),
+      CallerContext callerContext = CallerContext.create(
+          "HIVE", queryPlan.getQueryId(),
           "HIVE_QUERY_ID", queryPlan.getQueryStr());
       dag.setCallerContext(callerContext);
 
@@ -320,7 +319,10 @@ public class TezTask extends Task<TezWork> {
     FileSystem fs = scratchDir.getFileSystem(conf);
 
     // the name of the dag is what is displayed in the AM/Job UI
-    DAG dag = DAG.create(work.getName());
+    String dagName = utils.createDagName(conf, queryPlan);
+
+    LOG.info("Dag name: " + dagName);
+    DAG dag = DAG.create(dagName);
 
     // set some info for the query
     JSONObject json = new JSONObject(new LinkedHashMap()).put("context", "Hive")

http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 0be8b3c..f490161 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -50,13 +50,14 @@ public class ATSHook implements ExecuteWithHookContext {
 
   private static final Logger LOG = LoggerFactory.getLogger(ATSHook.class.getName());
   private static final Object LOCK = new Object();
+  private static final int VERSION = 2;
   private static ExecutorService executor;
   private static TimelineClient timelineClient;
   private enum EntityTypes { HIVE_QUERY_ID };
   private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
 
   private enum OtherInfoTypes {
-    QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME
+    QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME, VERSION
   };
   private enum PrimaryFilterTypes { user, requestuser, operationid };
   private static final int WAIT_TIME = 3;
@@ -193,6 +194,7 @@ public class ATSHook implements ExecuteWithHookContext {
     atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0);
     atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID);
     atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), Thread.currentThread().getName());
+    atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION);
     return atsEntity;
   }
 


Mime
View raw message