Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 80D8918500 for ; Tue, 1 Dec 2015 02:00:06 +0000 (UTC) Received: (qmail 47546 invoked by uid 500); 1 Dec 2015 02:00:01 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 47456 invoked by uid 500); 1 Dec 2015 02:00:01 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 47443 invoked by uid 99); 1 Dec 2015 02:00:01 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Dec 2015 02:00:01 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D176BE2C72; Tue, 1 Dec 2015 02:00:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Tue, 01 Dec 2015 02:00:00 -0000 Message-Id: <97b28c3d39754cd38d0a6e71ab2ab153@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hive git commit: HIVE-12490 : Metastore: Mysql ANSI_QUOTES is not there for some cases (Sergey Shelukhin, reviewed by Ashutosh Chauhan) Repository: hive Updated Branches: refs/heads/master e325eac9f -> 3a17d4230 HIVE-12490 : Metastore: Mysql ANSI_QUOTES is not there for some cases (Sergey Shelukhin, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef85cd62 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef85cd62 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef85cd62 Branch: refs/heads/master Commit: ef85cd62cb3bab4c4f59444433894f432fdabe8d Parents: e325eac Author: Sergey Shelukhin Authored: Mon Nov 30 17:46:58 2015 -0800 Committer: Sergey Shelukhin Committed: Mon Nov 30 17:46:58 2015 -0800 ---------------------------------------------------------------------- .../hive/metastore/MetaStoreDirectSql.java | 41 +++++++++----------- .../hadoop/hive/metastore/ObjectStore.java | 1 + 2 files changed, 19 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ef85cd62/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index d76e77f..36b316a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -232,7 +232,7 @@ class MetaStoreDirectSql { // Run a self-test query. If it doesn't work, we will self-disable. What a PITA... String selfTestQuery = "select \"DB_ID\" from \"DBS\""; try { - doDbSpecificInitializationsBeforeQuery(); + prepareTxn(); query = pm.newQuery("javax.jdo.query.SQL", selfTestQuery); query.execute(); return true; @@ -255,21 +255,6 @@ class MetaStoreDirectSql { return isCompatibleDatastore; } - /** - * This function is intended to be called by functions before they put together a query - * Thus, any query-specific instantiation to be done from within the transaction is done - * here - for eg., for MySQL, we signal that we want to use ANSI SQL quoting behaviour - */ - private void doDbSpecificInitializationsBeforeQuery() throws MetaException { - if (dbType != DB.MYSQL) return; - try { - assert pm.currentTransaction().isActive(); // must be inside tx together with queries - executeNoResult("SET @@session.sql_mode=ANSI_QUOTES"); - } catch (SQLException sqlEx) { - throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); - } - } - private void executeNoResult(final String queryText) throws SQLException { JDOConnection jdoConn = pm.getDataStoreConnection(); boolean doTrace = LOG.isDebugEnabled(); @@ -288,8 +273,6 @@ class MetaStoreDirectSql { try { dbName = dbName.toLowerCase(); - doDbSpecificInitializationsBeforeQuery(); - String queryTextDbSelector= "select " + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " + "\"OWNER_NAME\", \"OWNER_TYPE\" " @@ -461,8 +444,6 @@ class MetaStoreDirectSql { // We have to be mindful of order during filtering if we are not returning all partitions. String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : ""; - doDbSpecificInitializationsBeforeQuery(); - // Get all simple fields for partitions and related objects, which we can map one-on-one. // We will do this in 2 queries to use different existing indices for each one. // We do not get table and DB name, assuming they are the same as we are using to filter. @@ -1131,7 +1112,6 @@ class MetaStoreDirectSql { if (colNames.isEmpty()) { return null; } - doDbSpecificInitializationsBeforeQuery(); boolean doTrace = LOG.isDebugEnabled(); long start = doTrace ? System.nanoTime() : 0; String queryText = "select " + STATS_COLLIST + " from \"TAB_COL_STATS\" " @@ -1254,7 +1234,6 @@ class MetaStoreDirectSql { private List columnStatisticsObjForPartitions(String dbName, String tableName, List partNames, List colNames, long partsFound, boolean useDensityFunctionForNDVEstimation) throws MetaException { - doDbSpecificInitializationsBeforeQuery(); // TODO: all the extrapolation logic should be moved out of this class, // only mechanical data retrieval should remain here. String commonPrefix = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", " @@ -1571,7 +1550,6 @@ class MetaStoreDirectSql { return Lists.newArrayList(); } boolean doTrace = LOG.isDebugEnabled(); - doDbSpecificInitializationsBeforeQuery(); long start = doTrace ? System.nanoTime() : 0; String queryText = "select \"PARTITION_NAME\", " + STATS_COLLIST + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (" @@ -1668,4 +1646,21 @@ class MetaStoreDirectSql { throw new MetaException("See previous errors; " + ex.getMessage()); } } + + /** + * This run the necessary logic to prepare for queries. It should be called once, after the + * txn on DataNucleus connection is opened, and before any queries are issued. What it does + * currently is run db-specific logic, e.g. setting ansi quotes mode for MySQL. The reason it + * must be used inside of the txn is connection pooling; there's no way to guarantee that the + * effect will apply to the connection that is executing the queries otherwise. + */ + public void prepareTxn() throws MetaException { + if (dbType != DB.MYSQL) return; + try { + assert pm.currentTransaction().isActive(); // must be inside tx together with queries + executeNoResult("SET @@session.sql_mode=ANSI_QUOTES"); + } catch (SQLException sqlEx) { + throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/ef85cd62/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 803c6e7..1a8913c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2490,6 +2490,7 @@ public class ObjectStore implements RawStore, Configurable { start(initTable); if (doUseDirectSql) { try { + directSql.prepareTxn(); setResult(getSqlResult(this)); } catch (Exception ex) { handleDirectSqlError(ex);