hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject [29/44] hive git commit: HIVE-9871: Print spark job id in history file [spark branch] (Chinna via Xuefu)
Date Thu, 23 Apr 2015 02:34:38 GMT
HIVE-9871: Print spark job id in history file [spark branch] (Chinna via Xuefu)

git-svn-id: https://svn.apache.org/repos/asf/hive/branches/spark@1665876 13f79535-47bb-0310-9956-ffa450edef68


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb50d7e6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb50d7e6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb50d7e6

Branch: refs/heads/master
Commit: cb50d7e6406dd6a5fd973342fda9b58d20cc3abb
Parents: fabd940
Author: Xuefu Zhang <xuefu@apache.org>
Authored: Wed Mar 11 14:03:41 2015 +0000
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Wed Apr 22 19:33:51 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/spark/SparkTask.java  | 11 +++++++++++
 .../org/apache/hadoop/hive/ql/history/HiveHistory.java   |  3 ++-
 2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/cb50d7e6/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index 7b013bc..1c8d2a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManager;
 import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
 import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
 import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobStatus;
+import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -65,6 +66,7 @@ import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.SparkWork;
 import org.apache.hadoop.hive.ql.plan.StatsWork;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hive.spark.counter.SparkCounters;
@@ -100,6 +102,7 @@ public class SparkTask extends Task<SparkWork> {
       SparkJobRef jobRef = sparkSession.submit(driverContext, sparkWork);
       perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB);
 
+      addToHistory(jobRef);
       rc = jobRef.monitorJob();
       SparkJobStatus sparkJobStatus = jobRef.getSparkJobStatus();
       if (rc == 0) {
@@ -136,6 +139,14 @@ public class SparkTask extends Task<SparkWork> {
     return rc;
   }
 
+  private void addToHistory(SparkJobRef jobRef) {
+    console.printInfo("Starting Spark Job = " + jobRef.getJobId());
+    if (SessionState.get() != null) {
+      SessionState.get().getHiveHistory()
+	  .setQueryProperty(SessionState.get().getQueryId(), Keys.SPARK_JOB_ID, jobRef.getJobId());
+    }
+  }
+
   private void logSparkStatistic(SparkStatistics sparkStatistic) {
     Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups();
     while (groupIterator.hasNext()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/cb50d7e6/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java b/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
index 7b0d978..45cd533 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
@@ -68,7 +68,8 @@ public interface HiveHistory {
     TASK_COUNTERS,
     TASK_NUM_MAPPERS,
     TASK_NUM_REDUCERS,
-    ROWS_INSERTED
+    ROWS_INSERTED,
+    SPARK_JOB_ID
   };
 
   /**


Mime
View raw message