hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject hive git commit: HIVE-14152 : datanucleus.autoStartMechanismMode should set to Ignored to allow rolling downgrade (Thejas M Nair, reviewed by Sushanth Sowmyan)
Date Tue, 12 Jul 2016 07:12:49 GMT
Repository: hive
Updated Branches:
  refs/heads/master a61c35184 -> 5fb8a8b88


HIVE-14152 : datanucleus.autoStartMechanismMode should set to Ignored to allow rolling downgrade
(Thejas M Nair, reviewed by Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5fb8a8b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5fb8a8b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5fb8a8b8

Branch: refs/heads/master
Commit: 5fb8a8b8884d9b7318d61b77cc9b98ea6cadd570
Parents: a61c351
Author: Thejas Nair <thejas@hortonworks.com>
Authored: Tue Jul 12 00:12:40 2016 -0700
Committer: Thejas Nair <thejas@hortonworks.com>
Committed: Tue Jul 12 00:12:40 2016 -0700

----------------------------------------------------------------------
 beeline/src/test/resources/hive-site.xml        | 13 +++++++----
 .../org/apache/hadoop/hive/conf/HiveConf.java   | 13 +++++------
 data/conf/hive-site.xml                         |  5 ++++
 data/conf/llap/hive-site.xml                    |  5 ++++
 data/conf/spark/standalone/hive-site.xml        |  5 ++++
 data/conf/spark/yarn-client/hive-site.xml       |  5 ++++
 .../hive/metastore/TestMetastoreVersion.java    |  4 +++-
 .../hadoop/hive/metastore/ObjectStore.java      | 24 ++++++++++++++++++--
 ql/src/test/templates/TestPerfCliDriver.vm      |  1 +
 9 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/beeline/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml
index 453b0ea..5f310d6 100644
--- a/beeline/src/test/resources/hive-site.xml
+++ b/beeline/src/test/resources/hive-site.xml
@@ -24,10 +24,15 @@
     <description>Internal marker for test. Used for masking env-dependent values</description>
   </property>
 
-<property>
-  <name>datanucleus.schema.autoCreateAll</name>
-  <value>true</value>
-</property>
+  <property>
+    <name>datanucleus.schema.autoCreateAll</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>hive.metastore.schema.verification</name>
+    <value>false</value>
+  </property>
 
   <property>
     <name>javax.jdo.option.ConnectionURL</name>

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 283ef2e..bae1825 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -232,7 +232,6 @@ public class HiveConf extends Configuration {
       HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS,
       HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE,
       HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL,
-      HiveConf.ConfVars.METASTORE_AUTO_START_MECHANISM_MODE,
       HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION,
       HiveConf.ConfVars.METASTORE_CACHE_LEVEL2,
       HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE,
@@ -712,18 +711,18 @@ public class HiveConf extends Configuration {
         "validates existing schema against code. turn this on if you want to verify existing
schema"),
     METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms", "metadata store
type"),
     METASTORE_AUTO_CREATE_ALL("datanucleus.schema.autoCreateAll", false,
-        "creates necessary schema on a startup if one doesn't exist. set this to false, after
creating it once"),
-    METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false,
+        "Auto creates necessary schema on a startup if one doesn't exist. Set this to false,
after creating it once."
+        + "To enable auto create also set hive.metastore.schema.verification=false. Auto
creation is not "
+        + "recommended for production use cases, run schematool command instead." ),
+    METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", true,
         "Enforce metastore schema version consistency.\n" +
-        "True: Verify that version information stored in metastore matches with one from
Hive jars.  Also disable automatic\n" +
+        "True: Verify that version information stored in is compatible with one from Hive
jars.  Also disable automatic\n" +
         "      schema migration attempt. Users are required to manually migrate schema after
Hive upgrade which ensures\n" +
         "      proper metastore schema migration. (Default)\n" +
         "False: Warn if the version information stored in metastore doesn't match with one
from in Hive jars."),
-    METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version",
true,
+    METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version",
false,
       "When true the current MS version is recorded in the VERSION table. If this is disabled
and verification is\n" +
       " enabled the MS will be unusable."),
-    METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked",
-        "throw exception if metadata tables are incorrect"),
     METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed",
         "Default transaction isolation level for identity generation."),
     METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false,

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml
index 041b67d..775f5c2 100644
--- a/data/conf/hive-site.xml
+++ b/data/conf/hive-site.xml
@@ -64,6 +64,11 @@
 </property>
 
 <property>
+  <name>hive.metastore.schema.verification</name>
+  <value>false</value>
+</property>
+
+<property>
   <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
 </property>

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/llap/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml
index 72bdcfb..8b6dc95 100644
--- a/data/conf/llap/hive-site.xml
+++ b/data/conf/llap/hive-site.xml
@@ -74,6 +74,11 @@
 </property>
 
 <property>
+  <name>hive.metastore.schema.verification</name>
+  <value>false</value>
+</property>
+
+<property>
   <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
 </property>

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/spark/standalone/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml
index bc8c0e5..989e65e 100644
--- a/data/conf/spark/standalone/hive-site.xml
+++ b/data/conf/spark/standalone/hive-site.xml
@@ -50,6 +50,11 @@
 </property>
 
 <property>
+  <name>hive.metastore.schema.verification</name>
+  <value>false</value>
+</property>
+
+<property>
   <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
 </property>

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/spark/yarn-client/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml
index f1d9ddc..8c66f87 100644
--- a/data/conf/spark/yarn-client/hive-site.xml
+++ b/data/conf/spark/yarn-client/hive-site.xml
@@ -50,6 +50,11 @@
 </property>
 
 <property>
+  <name>hive.metastore.schema.verification</name>
+  <value>false</value>
+</property>
+
+<property>
   <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
 </property>

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 1352198..6d1673d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -107,12 +107,14 @@ public class TestMetastoreVersion extends TestCase {
   }
 
   /***
-   * Test that with no verification, hive populates the schema and version correctly
+   * Test that with no verification, and record verification enabled, hive populates the
schema
+   * and version correctly
    * @throws Exception
    */
   public void testMetastoreVersion () throws Exception {
     // let the schema and version be auto created
     System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false");
+    System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION.toString(),
"true");
     hiveConf = new HiveConf(this.getClass());
     SessionState.start(new CliSessionState(hiveConf));
     driver = new Driver(hiveConf);

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 82de857..5adfa02 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -385,6 +385,7 @@ public class ObjectStore implements RawStore, Configurable {
   @SuppressWarnings("nls")
   private static Properties getDataSourceProps(Configuration conf) {
     Properties prop = new Properties();
+    correctAutoStartMechanism(conf);
 
     Iterator<Map.Entry<String, String>> iter = conf.iterator();
     while (iter.hasNext()) {
@@ -420,6 +421,24 @@ public class ObjectStore implements RawStore, Configurable {
     return prop;
   }
 
+  /**
+   * Update conf to set datanucleus.autoStartMechanismMode=ignored.
+   * This is necessary to able to use older version of hive against
+   * an upgraded but compatible metastore schema in db from new version
+   * of hive
+   * @param conf
+   */
+  private static void correctAutoStartMechanism(Configuration conf) {
+    final String autoStartKey = "datanucleus.autoStartMechanismMode";
+    final String autoStartIgnore = "ignored";
+    String currentAutoStartVal = conf.get(autoStartKey);
+    if(currentAutoStartVal != null && !currentAutoStartVal.equalsIgnoreCase(autoStartIgnore))
{
+      LOG.warn(autoStartKey + " is set to unsupported value " + conf.get(autoStartKey) +
+          " . Setting it to value " + autoStartIgnore);
+    }
+    conf.set(autoStartKey, autoStartIgnore);
+  }
+
   private static synchronized PersistenceManagerFactory getPMF() {
     if (pmf == null) {
       pmf = JDOHelper.getPersistenceManagerFactory(prop);
@@ -7769,10 +7788,10 @@ public class ObjectStore implements RawStore, Configurable {
           "Set by MetaStore " + USER + "@" + HOSTNAME);
       }
     } else {
-      // metastore schema version is different than Hive distribution needs
       if (MetaStoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) {
         LOG.debug("Found expected HMS version of " + dbSchemaVer);
       } else {
+        // metastore schema version is different than Hive distribution needs
         if (strictValidation) {
           throw new MetaException("Hive Schema version " + hiveSchemaVer +
               " does not match metastore's schema version " + dbSchemaVer +
@@ -7781,7 +7800,7 @@ public class ObjectStore implements RawStore, Configurable {
           LOG.error("Version information found in metastore differs " + dbSchemaVer +
               " from expected schema version " + hiveSchemaVer +
               ". Schema verififcation is disabled " +
-              HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION + " so setting version.");
+              HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION);
           setMetaStoreSchemaVersion(hiveSchemaVer,
             "Set by MetaStore " + USER + "@" + HOSTNAME);
         }
@@ -7858,6 +7877,7 @@ public class ObjectStore implements RawStore, Configurable {
         "version = " + schemaVersion + ", comment = " + comment);
       return;
     }
+    LOG.warn("Setting metastore schema version in db to " + schemaVersion);
 
     try {
       mSchemaVer = getMSchemaVersion();

http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/ql/src/test/templates/TestPerfCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestPerfCliDriver.vm b/ql/src/test/templates/TestPerfCliDriver.vm
index 97dde2c..d2946cb 100644
--- a/ql/src/test/templates/TestPerfCliDriver.vm
+++ b/ql/src/test/templates/TestPerfCliDriver.vm
@@ -57,6 +57,7 @@ public class $className extends TestCase {
 
   static {
     System.setProperty("datanucleus.schema.autoCreateAll", "true");
+    System.setProperty("hive.metastore.schema.verification", "false");
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
     String hiveConfDir = "$hiveConfDir";
     String initScript = "$initScript";


Mime
View raw message