Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9DDFD18839 for ; Mon, 4 Jan 2016 20:56:40 +0000 (UTC) Received: (qmail 55092 invoked by uid 500); 4 Jan 2016 20:56:40 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 54853 invoked by uid 500); 4 Jan 2016 20:56:40 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 54838 invoked by uid 99); 4 Jan 2016 20:56:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Jan 2016 20:56:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 33F3CE05D0; Mon, 4 Jan 2016 20:56:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Mon, 04 Jan 2016 20:56:41 -0000 Message-Id: <75cf0559841a4021b67065585033690c@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] hive git commit: HIVE-6113 : Upgrade DataNucleus [was: Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient] (Oleksiy Sayankin, reviewed by Sergey Shelukhin) HIVE-6113 : Upgrade DataNucleus [was: Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient] (Oleksiy Sayankin, reviewed by Sergey Shelukhin) Conflicts: ql/src/test/templates/TestPerfCliDriver.vm Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f338ded Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f338ded Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f338ded Branch: refs/heads/branch-2.0 Commit: 3f338ded987afe9bc6e0377d8c473eb2e88efdd6 Parents: 31c188a Author: Sergey Shelukhin Authored: Mon Jan 4 12:46:04 2016 -0800 Committer: Sergey Shelukhin Committed: Mon Jan 4 12:46:57 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hive/beeline/cli/TestHiveCli.java | 4 +--- beeline/src/test/resources/hive-site.xml | 12 +----------- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 16 +++++++--------- data/conf/hive-site.xml | 12 +----------- data/conf/llap/hive-site.xml | 12 +----------- data/conf/spark/standalone/hive-site.xml | 12 +----------- data/conf/spark/yarn-client/hive-site.xml | 12 +----------- data/conf/tez/hive-site.xml | 12 +----------- .../deployers/config/hive/hive-site.mssql.xml | 2 +- .../deployers/config/hive/hive-site.mysql.xml | 6 +----- .../hadoop/hive/metastore/TestMetastoreVersion.java | 9 +++------ .../org/apache/hive/jdbc/TestJdbcWithMiniHS2.java | 7 ++++--- metastore/pom.xml | 5 +++++ metastore/scripts/upgrade/mssql/README | 2 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 5 +++-- pom.xml | 12 +++++++++--- .../org/apache/hadoop/hive/ql/exec/Utilities.java | 2 ++ 17 files changed, 43 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java ---------------------------------------------------------------------- diff --git a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java index 53dfa1d..275036f 100644 --- a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java +++ b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java @@ -276,9 +276,7 @@ public class TestHiveCli { @Before public void setup() { - System.setProperty("datanucleus.fixedDatastore", "false"); - System.setProperty("datanucleus.autoCreateSchema", "true"); - System.setProperty("hive.metastore.schema.verification", "false"); + System.setProperty("datanucleus.schema.autoCreateAll", "true"); cli = new HiveCli(); redirectOutputStream(); initFromFile(); http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/beeline/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml index 615d4ed..453b0ea 100644 --- a/beeline/src/test/resources/hive-site.xml +++ b/beeline/src/test/resources/hive-site.xml @@ -25,20 +25,10 @@ - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true - - datanucleus.fixedDatastore - false - - - - hive.metastore.schema.verification - false - - javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/metastore_db;create=true http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e3cbb0b..a831d09 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -144,7 +144,7 @@ public class HiveConf extends Configuration { HiveConf.ConfVars.METASTORE_VALIDATE_COLUMNS, HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS, HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE, - HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA, + HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, HiveConf.ConfVars.METASTORE_AUTO_START_MECHANISM_MODE, HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION, HiveConf.ConfVars.METASTORE_CACHE_LEVEL2, @@ -530,17 +530,16 @@ public class HiveConf extends Configuration { "List of comma separated metastore object types that should be pinned in the cache"), METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", "Specify connection pool library for datanucleus"), - METASTORE_VALIDATE_TABLES("datanucleus.validateTables", false, + METASTORE_VALIDATE_TABLES("datanucleus.schema.validateTables", false, "validates existing schema against code. turn this on if you want to verify existing schema"), - METASTORE_VALIDATE_COLUMNS("datanucleus.validateColumns", false, + METASTORE_VALIDATE_COLUMNS("datanucleus.schema.validateColumns", false, "validates existing schema against code. turn this on if you want to verify existing schema"), - METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false, + METASTORE_VALIDATE_CONSTRAINTS("datanucleus.schema.validateConstraints", false, "validates existing schema against code. turn this on if you want to verify existing schema"), METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms", "metadata store type"), - METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", false, + METASTORE_AUTO_CREATE_ALL("datanucleus.schema.autoCreateAll", false, "creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once"), - METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", true, "Dictates whether to allow updates to schema or not."), - METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", true, + METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false, "Enforce metastore schema version consistency.\n" + "True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic\n" + " schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures\n" + @@ -3216,8 +3215,7 @@ public class HiveConf extends Configuration { } if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) { - setBoolVar(ConfVars.METASTORE_AUTO_CREATE_SCHEMA, false); - setBoolVar(ConfVars.METASTORE_FIXED_DATASTORE, true); + setBoolVar(ConfVars.METASTORE_AUTO_CREATE_ALL, false); } if (getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) { http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/data/conf/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index 84005b4..5a790e7 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -59,21 +59,11 @@ - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true - datanucleus.fixedDatastore - false - - - - hive.metastore.schema.verification - false - - - javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/data/conf/llap/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index e3425c8..f768601 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -69,21 +69,11 @@ - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true - datanucleus.fixedDatastore - false - - - - hive.metastore.schema.verification - false - - - javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/data/conf/spark/standalone/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml index 459b52e..ca6ec0f 100644 --- a/data/conf/spark/standalone/hive-site.xml +++ b/data/conf/spark/standalone/hive-site.xml @@ -45,21 +45,11 @@ - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true - datanucleus.fixedDatastore - false - - - - hive.metastore.schema.verification - false - - - javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/data/conf/spark/yarn-client/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index bdab294..aef7877 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -45,21 +45,11 @@ - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true - datanucleus.fixedDatastore - false - - - - hive.metastore.schema.verification - false - - - javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/data/conf/tez/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml index a056ced..a7129fc 100644 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@ -63,21 +63,11 @@ - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true - datanucleus.fixedDatastore - false - - - - hive.metastore.schema.verification - false - - - hive.exec.local.scratchdir ${test.tmp.dir}/localscratchdir/ Local scratch space for Hive jobs http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml ---------------------------------------------------------------------- diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml index 8473d99..5bef8c8 100644 --- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml +++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml @@ -45,7 +45,7 @@ com.microsoft.sqlserver.jdbc.SQLServerDriver - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll false hive.support.concurrency http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index fba315d..53f0d0e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -51,8 +51,7 @@ public class TestMetastoreVersion extends TestCase { // reset defaults ObjectStore.setSchemaVerified(false); System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); - System.setProperty(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA.toString(), "true"); - System.setProperty(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE.toString(), "false"); + System.setProperty(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true"); hiveConf = new HiveConf(this.getClass()); System.setProperty("hive.support.concurrency", "false"); System.setProperty("hive.metastore.event.listeners", @@ -81,8 +80,7 @@ public class TestMetastoreVersion extends TestCase { System.clearProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString()); hiveConf = new HiveConf(this.getClass()); assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION)); - assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA)); - assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE)); + assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL)); } /*** @@ -93,8 +91,7 @@ public class TestMetastoreVersion extends TestCase { System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true"); hiveConf = new HiveConf(this.getClass()); assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION)); - assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA)); - assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE)); + assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL)); // session creation should fail since the schema didn't get created try { http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 9c8cf02..10c8ff2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -58,6 +58,7 @@ import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.datanucleus.ClassLoaderResolver; import org.datanucleus.NucleusContext; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; +import org.datanucleus.AbstractNucleusContext; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -731,8 +732,8 @@ public class TestJdbcWithMiniHS2 { } /** - * Tests that DataNucleus' NucleusContext.classLoaderResolverMap clears cached class objects (& hence - * doesn't leak classloaders) on closing any session + * Tests that DataNucleus' NucleusContext.classLoaderResolverMap clears cached class objects + * (& hence doesn't leak classloaders) on closing any session * * @throws Exception */ @@ -795,7 +796,7 @@ public class TestJdbcWithMiniHS2 { } if (nc != null) { try { - classLoaderResolverMap = NucleusContext.class.getDeclaredField("classLoaderResolverMap"); + classLoaderResolverMap = AbstractNucleusContext.class.getDeclaredField("classLoaderResolverMap"); if (classLoaderResolverMap != null) { classLoaderResolverMap.setAccessible(true); cMap = (Map) classLoaderResolverMap.get(nc); http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/metastore/pom.xml ---------------------------------------------------------------------- diff --git a/metastore/pom.xml b/metastore/pom.xml index 9da3bfb..3ff6c39 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -122,6 +122,11 @@ ${jdo-api.version} + org.datanucleus + javax.jdo + ${datanucleus-jdo.version} + + org.antlr antlr-runtime ${antlr.version} http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/metastore/scripts/upgrade/mssql/README ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/README b/metastore/scripts/upgrade/mssql/README index 8e5a33e..0b62a01 100644 --- a/metastore/scripts/upgrade/mssql/README +++ b/metastore/scripts/upgrade/mssql/README @@ -77,7 +77,7 @@ Upgrading from 0.11.0 to 0.13.0 2) Make sure you have - datanucleus.autoCreateSchema + datanucleus.schema.autoCreateAll true in your hive-site.xml. This will cause DataNucleus to create http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 1c0ab6d..297b028 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -148,6 +148,7 @@ import org.apache.hive.common.util.HiveStringUtils; import org.apache.thrift.TException; import org.datanucleus.ClassLoaderResolver; import org.datanucleus.NucleusContext; +import org.datanucleus.AbstractNucleusContext; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; import org.datanucleus.store.rdbms.exceptions.MissingTableException; @@ -7757,8 +7758,8 @@ public class ObjectStore implements RawStore, Configurable { JDOPersistenceManagerFactory jdoPmf = (JDOPersistenceManagerFactory) pmf; NucleusContext nc = jdoPmf.getNucleusContext(); try { - Field classLoaderResolverMap = - NucleusContext.class.getDeclaredField("classLoaderResolverMap"); + Field classLoaderResolverMap = AbstractNucleusContext.class.getDeclaredField( + "classLoaderResolverMap"); classLoaderResolverMap.setAccessible(true); classLoaderResolverMap.set(nc, new HashMap()); LOG.debug("Removed cached classloaders from DataNucleus NucleusContext"); http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 31821a3..1393fba 100644 --- a/pom.xml +++ b/pom.xml @@ -107,9 +107,10 @@ 1.7.7 0.8.0.RELEASE 1.5.0 - 3.2.6 - 3.2.10 - 3.2.9 + 4.2.1 + 4.1.6 + 4.1.7 + 3.2.0-m3 1.2 1.4 3.2.2 @@ -576,6 +577,11 @@ ${datanucleus-rdbms.version} + org.datanucleus + javax.jdo + ${datanucleus-jdo.version} + + org.iq80.snappy snappy ${snappy.version} http://git-wip-us.apache.org/repos/asf/hive/blob/3f338ded/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index fce11c8..9a7d990 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3579,4 +3579,6 @@ public final class Utilities { return rowObjectInspector; } + + }