hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject hive git commit: HIVE-11250: Change in spark.executor.instances (and others) doesn't take effect after RSC is launched for HS2 (Jimmy, reviewed by Xuefu)
Date Thu, 06 Aug 2015 16:00:56 GMT
Repository: hive
Updated Branches:
  refs/heads/master a0fdf00fe -> 8398fbf3d


HIVE-11250: Change in spark.executor.instances (and others) doesn't take effect after RSC
is launched for HS2 (Jimmy, reviewed by Xuefu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8398fbf3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8398fbf3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8398fbf3

Branch: refs/heads/master
Commit: 8398fbf3dd0937a0a4a3d540977a95f97425f566
Parents: a0fdf00
Author: Jimmy Xiang <jxiang@cloudera.com>
Authored: Tue Aug 4 11:23:34 2015 -0700
Committer: Jimmy Xiang <jxiang@cloudera.com>
Committed: Thu Aug 6 09:00:09 2015 -0700

----------------------------------------------------------------------
 common/src/java/org/apache/hadoop/hive/conf/HiveConf.java |  1 +
 .../ql/exec/spark/status/impl/LocalSparkJobStatus.java    |  2 +-
 .../ql/exec/spark/status/impl/RemoteSparkJobStatus.java   |  2 +-
 .../apache/hadoop/hive/ql/processors/SetProcessor.java    |  4 ++++
 .../org/apache/hadoop/hive/ql/session/SessionState.java   | 10 +++++++---
 5 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index fe4eb9f..36bb394 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2723,6 +2723,7 @@ public class HiveConf extends Configuration {
     super(other);
     hiveJar = other.hiveJar;
     auxJars = other.auxJars;
+    isSparkConfigUpdated = other.isSparkConfigUpdated;
     origProp = (Properties)other.origProp.clone();
     restrictList.addAll(other.restrictList);
     modWhiteListPattern = other.modWhiteListPattern;

http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java
index 5d62596..c6f1b8d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java
@@ -203,7 +203,7 @@ public class LocalSparkJobStatus implements SparkJobStatus {
       }
     }
 
-    results.put("EexcutorDeserializeTime", executorDeserializeTime);
+    results.put("ExecutorDeserializeTime", executorDeserializeTime);
     results.put("ExecutorRunTime", executorRunTime);
     results.put("ResultSize", resultSize);
     results.put("JvmGCTime", jvmGCTime);

http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java
index 9b1ef2c..072bac9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java
@@ -220,7 +220,7 @@ public class RemoteSparkJobStatus implements SparkJobStatus {
     Map<String, Long> results = new LinkedHashMap<String, Long>();
     Metrics allMetrics = metricsCollection.getAllMetrics();
 
-    results.put("EexcutorDeserializeTime", allMetrics.executorDeserializeTime);
+    results.put("ExecutorDeserializeTime", allMetrics.executorDeserializeTime);
     results.put("ExecutorRunTime", allMetrics.executorRunTime);
     results.put("ResultSize", allMetrics.resultSize);
     results.put("JvmGCTime", allMetrics.jvmGCTime);

http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index d271d6d..2414e12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -168,6 +168,10 @@ public class SetProcessor implements CommandProcessor {
       }
     }
     conf.verifyAndSet(key, value);
+    if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)
+        && !"spark".equals(value)) {
+      SessionState.get().closeSparkSession();
+    }
     if (register) {
       SessionState.get().getOverriddenConfigurations().put(key, value);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 510d8a7..7ed8e5f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -762,7 +762,7 @@ public class SessionState {
         getAuthorizer() : getAuthorizerV2();
   }
 
-  public Class getAuthorizerInterface() {
+  public Class<?> getAuthorizerInterface() {
     return getAuthorizationMode() == AuthorizationMode.V1 ?
         HiveAuthorizationProvider.class : HiveAuthorizer.class;
   }
@@ -1490,6 +1490,12 @@ public class SessionState {
       tezSessionState = null;
     }
 
+    closeSparkSession();
+    registry.closeCUDFLoaders();
+    dropSessionPaths(conf);
+  }
+
+  public void closeSparkSession() {
     if (sparkSession != null) {
       try {
         SparkSessionManagerImpl.getInstance().closeSession(sparkSession);
@@ -1499,8 +1505,6 @@ public class SessionState {
         sparkSession = null;
       }
     }
-    registry.closeCUDFLoaders();
-    dropSessionPaths(conf);
   }
 
   public AuthorizationMode getAuthorizationMode(){


Mime
View raw message