Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 03A05200B8A for ; Fri, 9 Sep 2016 11:54:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 028EA160AB6; Fri, 9 Sep 2016 09:54:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2ED38160AC2 for ; Fri, 9 Sep 2016 11:54:36 +0200 (CEST) Received: (qmail 36293 invoked by uid 500); 9 Sep 2016 09:54:35 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 36276 invoked by uid 99); 9 Sep 2016 09:54:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 Sep 2016 09:54:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 27C18DFE80; Fri, 9 Sep 2016 09:54:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vgumashta@apache.org To: commits@hive.apache.org Message-Id: <135684725abc4e658b3b61a4754a0b4a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-14039: HiveServer2: Make the usage of server with JDBC thirft serde enabled, backward compatible for older clients (Ziyang Zhao reviewed by Vaibhav Gumashta) Date: Fri, 9 Sep 2016 09:54:35 +0000 (UTC) archived-at: Fri, 09 Sep 2016 09:54:37 -0000 Repository: hive Updated Branches: refs/heads/branch-2.1 3318dcf8d -> 2a8f1ce1d HIVE-14039: HiveServer2: Make the usage of server with JDBC thirft serde enabled, backward compatible for older clients (Ziyang Zhao reviewed by Vaibhav Gumashta) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2a8f1ce1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2a8f1ce1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2a8f1ce1 Branch: refs/heads/branch-2.1 Commit: 2a8f1ce1d8176202eeeee75ed5c9496ac47526ea Parents: 3318dcf Author: Vaibhav Gumashta Authored: Fri Sep 9 02:48:38 2016 -0700 Committer: Vaibhav Gumashta Committed: Fri Sep 9 02:54:28 2016 -0700 ---------------------------------------------------------------------- .../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 44 ++++++++++++++++++++ .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +- .../hadoop/hive/ql/parse/TaskCompiler.java | 2 +- .../hadoop/hive/ql/session/SessionState.java | 13 ++++++ .../service/cli/session/HiveSessionImpl.java | 9 +++- 5 files changed, 67 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index eeee0db..2b3ef65 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -441,6 +441,11 @@ public class TestJdbcWithMiniHS2 { conf.setInt("hive.server2.thrift.resultset.max.fetch.size", 1000); } + private void unsetSerializeInTasksInConf(HiveConf conf) { + conf.setBoolean("hive.server2.thrift.resultset.serialize.in.tasks", false); + conf.unset("hive.server2.thrift.resultset.max.fetch.size"); + } + @Test public void testMetadataQueriesWithSerializeThriftInTasks() throws Exception { //stop HiveServer2 @@ -603,6 +608,45 @@ public class TestJdbcWithMiniHS2 { stmt.close(); } + @Test + public void testEnableThriftSerializeInTasks() throws Exception { + //stop HiveServer2 + if (miniHS2.isStarted()) { + miniHS2.stop(); + } + + HiveConf conf = new HiveConf(); + String userName; + setSerializeInTasksInConf(conf); + miniHS2 = new MiniHS2(conf); + Map confOverlay = new HashMap(); + miniHS2.start(confOverlay); + + userName = System.getProperty("user.name"); + hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password"); + Statement stmt = hs2Conn.createStatement(); + stmt.execute("drop table if exists testThriftSerializeShow1"); + stmt.execute("drop table if exists testThriftSerializeShow2"); + stmt.execute("create table testThriftSerializeShow1 (a int)"); + stmt.execute("create table testThriftSerializeShow2 (b int)"); + stmt.execute("insert into testThriftSerializeShow1 values (1)"); + stmt.execute("insert into testThriftSerializeShow2 values (2)"); + ResultSet rs = stmt.executeQuery("select * from testThriftSerializeShow1 inner join testThriftSerializeShow2 where testThriftSerializeShow1.a=testThriftSerializeShow2.b"); + assertTrue(!rs.next()); + + unsetSerializeInTasksInConf(conf); + rs = stmt.executeQuery("select * from testThriftSerializeShow1 inner join testThriftSerializeShow2 where testThriftSerializeShow1.a=testThriftSerializeShow2.b"); + assertTrue(!rs.next()); + + setSerializeInTasksInConf(conf); + rs = stmt.executeQuery("select * from testThriftSerializeShow1 inner join testThriftSerializeShow2 where testThriftSerializeShow1.a=testThriftSerializeShow2.b"); + assertTrue(!rs.next()); + + stmt.execute("drop table testThriftSerializeShow1"); + stmt.execute("drop table testThriftSerializeShow2"); + stmt.close(); + } + /** * Tests the creation of the 3 scratch dirs: hdfs, local, downloaded resources (which is also local). * 1. Test with doAs=false: open a new JDBC session and verify the presence of directories/permissions http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e437b7f..ede0b6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6903,8 +6903,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (tblDesc == null) { if (qb.getIsQuery()) { String fileFormat; - if (SessionState.get().isHiveServerQuery() && - conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) { + if (SessionState.get().getIsUsingThriftJDBCBinarySerDe()) { fileFormat = "SequenceFile"; HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT, fileFormat); table_desc= http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 4b34ebf..80a5c32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -152,7 +152,7 @@ public abstract class TaskCompiler { TableDesc resultTab = pCtx.getFetchTableDesc(); if (resultTab == null) { resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); - if (SessionState.get().isHiveServerQuery() && (conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) + if (SessionState.get().getIsUsingThriftJDBCBinarySerDe() && (resFileFormat.equalsIgnoreCase("SequenceFile"))) { resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat, http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 08e58cc..784cd30 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -140,6 +140,11 @@ public class SessionState { */ private boolean isHiveServerQuery = false; + /** + * The flag to indicate if the session using thrift jdbc binary serde or not. + */ + private boolean isUsingThriftJDBCBinarySerDe = false; + /* * HiveHistory Object */ @@ -342,6 +347,14 @@ public class SessionState { this.isVerbose = isVerbose; } + public void setIsUsingThriftJDBCBinarySerDe(boolean isUsingThriftJDBCBinarySerDe) { + this.isUsingThriftJDBCBinarySerDe = isUsingThriftJDBCBinarySerDe; + } + + public boolean getIsUsingThriftJDBCBinarySerDe() { + return isUsingThriftJDBCBinarySerDe; + } + public void setIsHiveServerQuery(boolean isHiveServerQuery) { this.isHiveServerQuery = isHiveServerQuery; } http://git-wip-us.apache.org/repos/asf/hive/blob/2a8f1ce1/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index ae6cb1a..0b8da36 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -164,6 +164,7 @@ public class HiveSessionImpl implements HiveSession { sessionState.setUserIpAddress(ipAddress); sessionState.setIsHiveServerQuery(true); sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses()); + sessionState.setIsUsingThriftJDBCBinarySerDe(updateIsUsingThriftJDBCBinarySerDe()); SessionState.start(sessionState); try { sessionState.reloadAuxJars(); @@ -186,7 +187,7 @@ public class HiveSessionImpl implements HiveSession { lastIdleTime = lastAccessTime; } - /** +/** * It is used for processing hiverc file from HiveServer2 side. */ private class GlobalHivercFileProcessor extends HiveFileProcessor { @@ -265,6 +266,11 @@ public class HiveSessionImpl implements HiveSession { } } + private boolean updateIsUsingThriftJDBCBinarySerDe() { + return (8 <= getProtocolVersion().getValue()) + && sessionConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS); + } + @Override public void setOperationLogSessionDir(File operationLogRootDir) { if (!operationLogRootDir.exists()) { @@ -352,6 +358,7 @@ public class HiveSessionImpl implements HiveSession { // stored in the thread local for the handler thread. SessionState.setCurrentSessionState(sessionState); sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses()); + sessionState.setIsUsingThriftJDBCBinarySerDe(updateIsUsingThriftJDBCBinarySerDe()); if (userAccess) { lastAccessTime = System.currentTimeMillis(); }