hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-11923 : allow qtests to run via a single client session for tez and llap (Siddharth Seth/Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Date Tue, 13 Oct 2015 21:54:49 GMT
Repository: hive
Updated Branches:
  refs/heads/master bb791bacb -> 07eaab396


HIVE-11923 : allow qtests to run via a single client session for tez and llap (Siddharth Seth/Sergey
Shelukhin, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/07eaab39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/07eaab39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/07eaab39

Branch: refs/heads/master
Commit: 07eaab39607e66a397f4efe6824b2eaf98653327
Parents: bb791ba
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Tue Oct 13 14:47:17 2015 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Tue Oct 13 14:54:34 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hive/hbase/HBaseQTestUtil.java       |  8 +--
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 61 ++++++++++++++------
 .../test/queries/clientpositive/scriptfile1.q   |  3 +
 .../results/clientpositive/scriptfile1.q.out    |  6 ++
 .../clientpositive/spark/scriptfile1.q.out      |  6 ++
 .../clientpositive/tez/scriptfile1.q.out        |  6 ++
 6 files changed, 69 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 9c20f90..59fab2c 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -69,8 +69,8 @@ public class HBaseQTestUtil extends QTestUtil {
   }
 
   @Override
-  public void createSources() throws Exception {
-    super.createSources();
+  public void createSources(String tname) throws Exception {
+    super.createSources(tname);
 
     conf.setBoolean("hive.test.init.phase", true);
 
@@ -96,8 +96,8 @@ public class HBaseQTestUtil extends QTestUtil {
   }
 
   @Override
-  public void cleanUp() throws Exception {
-    super.cleanUp();
+  public void cleanUp(String tname) throws Exception {
+    super.cleanUp(tname);
 
     // drop in case leftover from unsuccessful run
     db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, HBASE_SRC_NAME);

http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 174a988..309cdff 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -142,6 +142,7 @@ public class QTestUtil {
   private final Set<String> qSortQuerySet;
   private final Set<String> qHashQuerySet;
   private final Set<String> qSortNHashQuerySet;
+  private final Set<String> qNoSessionReuseQuerySet;
   private final Set<String> qJavaVersionSpecificOutput;
   private static final String SORT_SUFFIX = ".sorted";
   public static final HashSet<String> srcTables = new HashSet<String>();
@@ -401,6 +402,7 @@ public class QTestUtil {
     qSortQuerySet = new HashSet<String>();
     qHashQuerySet = new HashSet<String>();
     qSortNHashQuerySet = new HashSet<String>();
+    qNoSessionReuseQuerySet = new HashSet<String>();
     qJavaVersionSpecificOutput = new HashSet<String>();
     QTestUtil.clusterType = clusterType;
 
@@ -556,12 +558,16 @@ public class QTestUtil {
     } else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) {
       qSortNHashQuerySet.add(qf.getName());
     }
+    if (matches(NO_SESSION_REUSE, query)) {
+      qNoSessionReuseQuerySet.add(qf.getName());
+    }
   }
 
   private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
   private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS");
   private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
   private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
+  private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
 
   private boolean matches(Pattern pattern, String query) {
     Matcher matcher = pattern.matcher(query);
@@ -803,8 +809,13 @@ public class QTestUtil {
   }
 
   public void cleanUp() throws Exception {
+    cleanUp(null);
+  }
+
+  public void cleanUp(String tname) throws Exception {
+    boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
     if(!isSessionStateStarted) {
-      startSessionState();
+      startSessionState(canReuseSession);
     }
     if (System.getenv(QTEST_LEAVE_FILES) != null) {
       return;
@@ -867,8 +878,13 @@ public class QTestUtil {
   }
 
   public void createSources() throws Exception {
+    createSources(null);
+  }
+
+  public void createSources(String tname) throws Exception {
+    boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
     if(!isSessionStateStarted) {
-      startSessionState();
+      startSessionState(canReuseSession);
     }
 
     if(cliDriver == null) {
@@ -908,8 +924,8 @@ public class QTestUtil {
   }
 
   public void init(String tname) throws Exception {
-    cleanUp();
-    createSources();
+    cleanUp(tname);
+    createSources(tname);
     cliDriver.processCmd("set hive.cli.print.header=true;");
   }
 
@@ -919,8 +935,8 @@ public class QTestUtil {
 
   public String cliInit(String tname, boolean recreate) throws Exception {
     if (recreate) {
-      cleanUp();
-      createSources();
+      cleanUp(tname);
+      createSources(tname);
     }
 
     HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
@@ -955,11 +971,9 @@ public class QTestUtil {
     ss.setIsSilent(true);
     SessionState oldSs = SessionState.get();
 
-    if (oldSs != null && (clusterType == MiniClusterType.llap
-        || clusterType == MiniClusterType.spark || clusterType == MiniClusterType.miniSparkOnYarn))
{
-      sparkSession = oldSs.getSparkSession();
-      ss.setSparkSession(sparkSession);
-      oldSs.setSparkSession(null);
+    boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname);
+    if (oldSs != null && canReuseSession
+        && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap))
{
       // Copy the tezSessionState from the old CliSessionState.
       tezSessionState = oldSs.getTezSession();
       ss.setTezSession(tezSessionState);
@@ -967,6 +981,14 @@ public class QTestUtil {
       oldSs.close();
     }
 
+    if (oldSs != null && (clusterType == MiniClusterType.spark
+        || clusterType == MiniClusterType.miniSparkOnYarn)) {
+      sparkSession = oldSs.getSparkSession();
+      ss.setSparkSession(sparkSession);
+      oldSs.setSparkSession(null);
+      oldSs.close();
+    }
+
     if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
       oldSs.out.close();
     }
@@ -1008,7 +1030,7 @@ public class QTestUtil {
     };
   }
 
-  private CliSessionState startSessionState()
+  private CliSessionState startSessionState(boolean canReuseSession)
       throws IOException {
 
     HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
@@ -1023,17 +1045,22 @@ public class QTestUtil {
     ss.err = System.out;
 
     SessionState oldSs = SessionState.get();
-    if (oldSs != null && (clusterType == MiniClusterType.llap
-        || clusterType == MiniClusterType.miniSparkOnYarn || clusterType == MiniClusterType.miniSparkOnYarn))
{
-      sparkSession = oldSs.getSparkSession();
-      ss.setSparkSession(sparkSession);
-      oldSs.setSparkSession(null);
+    if (oldSs != null && canReuseSession
+        && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap))
{
       // Copy the tezSessionState from the old CliSessionState.
       tezSessionState = oldSs.getTezSession();
       ss.setTezSession(tezSessionState);
       oldSs.setTezSession(null);
       oldSs.close();
     }
+
+    if (oldSs != null && (clusterType == MiniClusterType.spark
+        || clusterType == MiniClusterType.miniSparkOnYarn)) {
+      sparkSession = oldSs.getSparkSession();
+      ss.setSparkSession(sparkSession);
+      oldSs.setSparkSession(null);
+      oldSs.close();
+    }
     if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
       oldSs.out.close();
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/queries/clientpositive/scriptfile1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/scriptfile1.q b/ql/src/test/queries/clientpositive/scriptfile1.q
index 49d2197..a075177 100644
--- a/ql/src/test/queries/clientpositive/scriptfile1.q
+++ b/ql/src/test/queries/clientpositive/scriptfile1.q
@@ -3,6 +3,9 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING);
 
 ADD FILE ../../ql/src/test/scripts/testgrep;

http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/results/clientpositive/scriptfile1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/scriptfile1.q.out b/ql/src/test/results/clientpositive/scriptfile1.q.out
index d72ff5c..bf202f9 100644
--- a/ql/src/test/results/clientpositive/scriptfile1.q.out
+++ b/ql/src/test/results/clientpositive/scriptfile1.q.out
@@ -1,6 +1,9 @@
 PREHOOK: query: -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING)
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
@@ -8,6 +11,9 @@ PREHOOK: Output: default@dest1
 POSTHOOK: query: -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING)
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default

http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/scriptfile1.q.out b/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
index d72ff5c..bf202f9 100644
--- a/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
+++ b/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
@@ -1,6 +1,9 @@
 PREHOOK: query: -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING)
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
@@ -8,6 +11,9 @@ PREHOOK: Output: default@dest1
 POSTHOOK: query: -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING)
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default

http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/scriptfile1.q.out b/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
index d72ff5c..bf202f9 100644
--- a/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
+++ b/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
@@ -1,6 +1,9 @@
 PREHOOK: query: -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING)
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
@@ -8,6 +11,9 @@ PREHOOK: Output: default@dest1
 POSTHOOK: query: -- SORT_QUERY_RESULTS
 
 -- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
 CREATE TABLE dest1(key INT, value STRING)
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default


Mime
View raw message