hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vgumas...@apache.org
Subject hive git commit: HIVE-14039: HiveServer2: Make the usage of server with JDBC thirft serde enabled, backward compatible for older clients (Ziyang Zhao reviewed by Vaibhav Gumashta)
Date Fri, 09 Sep 2016 09:54:35 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2.1 3318dcf8d -> 2a8f1ce1d


HIVE-14039: HiveServer2: Make the usage of server with JDBC thirft serde enabled, backward
compatible for older clients (Ziyang Zhao reviewed by Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2a8f1ce1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2a8f1ce1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2a8f1ce1

Branch: refs/heads/branch-2.1
Commit: 2a8f1ce1d8176202eeeee75ed5c9496ac47526ea
Parents: 3318dcf
Author: Vaibhav Gumashta <vgumashta@hortonworks.com>
Authored: Fri Sep 9 02:48:38 2016 -0700
Committer: Vaibhav Gumashta <vgumashta@hortonworks.com>
Committed: Fri Sep 9 02:54:28 2016 -0700

----------------------------------------------------------------------
 .../apache/hive/jdbc/TestJdbcWithMiniHS2.java   | 44 ++++++++++++++++++++
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  3 +-
 .../hadoop/hive/ql/parse/TaskCompiler.java      |  2 +-
 .../hadoop/hive/ql/session/SessionState.java    | 13 ++++++
 .../service/cli/session/HiveSessionImpl.java    |  9 +++-
 5 files changed, 67 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
index eeee0db..2b3ef65 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
@@ -441,6 +441,11 @@ public class TestJdbcWithMiniHS2 {
     conf.setInt("hive.server2.thrift.resultset.max.fetch.size", 1000);
   }
 
+  private void unsetSerializeInTasksInConf(HiveConf conf) {
+    conf.setBoolean("hive.server2.thrift.resultset.serialize.in.tasks", false);
+    conf.unset("hive.server2.thrift.resultset.max.fetch.size");
+  }
+
   @Test
   public void testMetadataQueriesWithSerializeThriftInTasks() throws Exception {
     //stop HiveServer2
@@ -603,6 +608,45 @@ public class TestJdbcWithMiniHS2 {
     stmt.close();
   }
 
+   @Test
+   public void testEnableThriftSerializeInTasks() throws Exception {
+     //stop HiveServer2
+     if (miniHS2.isStarted()) {
+       miniHS2.stop();
+     }
+
+     HiveConf conf = new HiveConf();
+     String userName;
+     setSerializeInTasksInConf(conf);
+     miniHS2 = new MiniHS2(conf);
+     Map<String, String> confOverlay = new HashMap<String, String>();
+     miniHS2.start(confOverlay);
+
+     userName = System.getProperty("user.name");
+     hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password");
+     Statement stmt = hs2Conn.createStatement();
+     stmt.execute("drop table if exists testThriftSerializeShow1");
+     stmt.execute("drop table if exists testThriftSerializeShow2");
+     stmt.execute("create table testThriftSerializeShow1 (a int)");
+     stmt.execute("create table testThriftSerializeShow2 (b int)");
+     stmt.execute("insert into testThriftSerializeShow1 values (1)");
+     stmt.execute("insert into testThriftSerializeShow2 values (2)");
+     ResultSet rs = stmt.executeQuery("select * from testThriftSerializeShow1 inner join
testThriftSerializeShow2 where testThriftSerializeShow1.a=testThriftSerializeShow2.b");
+     assertTrue(!rs.next());
+
+     unsetSerializeInTasksInConf(conf);
+     rs = stmt.executeQuery("select * from testThriftSerializeShow1 inner join testThriftSerializeShow2
where testThriftSerializeShow1.a=testThriftSerializeShow2.b");
+     assertTrue(!rs.next());
+
+     setSerializeInTasksInConf(conf);
+     rs = stmt.executeQuery("select * from testThriftSerializeShow1 inner join testThriftSerializeShow2
where testThriftSerializeShow1.a=testThriftSerializeShow2.b");
+     assertTrue(!rs.next());
+
+     stmt.execute("drop table testThriftSerializeShow1");
+     stmt.execute("drop table testThriftSerializeShow2");
+     stmt.close();
+   }
+
   /**
    * Tests the creation of the 3 scratch dirs: hdfs, local, downloaded resources (which is
also local).
    * 1. Test with doAs=false: open a new JDBC session and verify the presence of directories/permissions

http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index e437b7f..ede0b6e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6903,8 +6903,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       if (tblDesc == null) {
         if (qb.getIsQuery()) {
           String fileFormat;
-          if (SessionState.get().isHiveServerQuery() &&
-                   conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS))
{
+          if (SessionState.get().getIsUsingThriftJDBCBinarySerDe()) {
               fileFormat = "SequenceFile";
               HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT, fileFormat);
               table_desc=

http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 4b34ebf..80a5c32 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -152,7 +152,7 @@ public abstract class TaskCompiler {
       TableDesc resultTab = pCtx.getFetchTableDesc();
       if (resultTab == null) {
         resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
-        if (SessionState.get().isHiveServerQuery() && (conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS))
+        if (SessionState.get().getIsUsingThriftJDBCBinarySerDe()
             && (resFileFormat.equalsIgnoreCase("SequenceFile"))) {
           resultTab =
               PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat,

http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 08e58cc..784cd30 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -140,6 +140,11 @@ public class SessionState {
    */
   private boolean isHiveServerQuery = false;
 
+  /**
+   * The flag to indicate if the session using thrift jdbc binary serde or not.
+   */
+  private boolean isUsingThriftJDBCBinarySerDe = false;
+
   /*
    * HiveHistory Object
    */
@@ -342,6 +347,14 @@ public class SessionState {
     this.isVerbose = isVerbose;
   }
 
+  public void setIsUsingThriftJDBCBinarySerDe(boolean isUsingThriftJDBCBinarySerDe) {
+	this.isUsingThriftJDBCBinarySerDe = isUsingThriftJDBCBinarySerDe;
+  }
+
+  public boolean getIsUsingThriftJDBCBinarySerDe() {
+	return isUsingThriftJDBCBinarySerDe;
+  }
+
   public void setIsHiveServerQuery(boolean isHiveServerQuery) {
     this.isHiveServerQuery = isHiveServerQuery;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index ae6cb1a..0b8da36 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -164,6 +164,7 @@ public class HiveSessionImpl implements HiveSession {
     sessionState.setUserIpAddress(ipAddress);
     sessionState.setIsHiveServerQuery(true);
     sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses());
+    sessionState.setIsUsingThriftJDBCBinarySerDe(updateIsUsingThriftJDBCBinarySerDe());
     SessionState.start(sessionState);
     try {
       sessionState.reloadAuxJars();
@@ -186,7 +187,7 @@ public class HiveSessionImpl implements HiveSession {
     lastIdleTime = lastAccessTime;
   }
 
-  /**
+/**
    * It is used for processing hiverc file from HiveServer2 side.
    */
   private class GlobalHivercFileProcessor extends HiveFileProcessor {
@@ -265,6 +266,11 @@ public class HiveSessionImpl implements HiveSession {
     }
   }
 
+  private boolean updateIsUsingThriftJDBCBinarySerDe() {
+	return (8 <= getProtocolVersion().getValue())
+      && sessionConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS);
+  }
+
   @Override
   public void setOperationLogSessionDir(File operationLogRootDir) {
     if (!operationLogRootDir.exists()) {
@@ -352,6 +358,7 @@ public class HiveSessionImpl implements HiveSession {
     // stored in the thread local for the handler thread.
     SessionState.setCurrentSessionState(sessionState);
     sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses());
+    sessionState.setIsUsingThriftJDBCBinarySerDe(updateIsUsingThriftJDBCBinarySerDe());
     if (userAccess) {
       lastAccessTime = System.currentTimeMillis();
     }


Mime
View raw message