hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject hive git commit: HIVE-14080 : hive.metastore.schema.verification should check for schema compatiblity (Thejas M Nair, reviewed by Sushanth Sowmyan)
Date Mon, 04 Jul 2016 19:47:39 GMT
Repository: hive
Updated Branches:
  refs/heads/master fb35bae5a -> dbb76ead4


HIVE-14080 : hive.metastore.schema.verification should check for schema compatiblity (Thejas
M Nair, reviewed by Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dbb76ead
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dbb76ead
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dbb76ead

Branch: refs/heads/master
Commit: dbb76ead42a94560bc53b78781bf2174aad58d3e
Parents: fb35bae
Author: Thejas Nair <thejas@hortonworks.com>
Authored: Mon Jul 4 12:46:42 2016 -0700
Committer: Thejas Nair <thejas@hortonworks.com>
Committed: Mon Jul 4 12:47:20 2016 -0700

----------------------------------------------------------------------
 .../hive/metastore/TestMetastoreVersion.java    | 23 +++++++++
 .../hadoop/hive/metastore/ObjectStore.java      | 26 ++++++-----
 .../hive/metastore/TestMetaStoreSchemaInfo.java | 49 ++++++++++++++++++++
 3 files changed, 86 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/dbb76ead/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 5ceb3d2..1352198 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -135,6 +135,7 @@ public class TestMetastoreVersion extends TestCase {
     driver = new Driver(hiveConf);
     driver.run("show tables");
 
+    ObjectStore.setSchemaVerified(false);
     hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, true);
     setVersion(hiveConf, MetaStoreSchemaInfo.getHiveSchemaVersion());
     driver = new Driver(hiveConf);
@@ -153,12 +154,34 @@ public class TestMetastoreVersion extends TestCase {
     driver = new Driver(hiveConf);
     driver.run("show tables");
 
+    ObjectStore.setSchemaVerified(false);
     System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true");
     hiveConf = new HiveConf(this.getClass());
     setVersion(hiveConf, "fooVersion");
     SessionState.start(new CliSessionState(hiveConf));
     driver = new Driver(hiveConf);
     CommandProcessorResponse proc = driver.run("show tables");
+    assertTrue(proc.getResponseCode() != 0);
+  }
+
+  /**
+   * Store higher version in metastore and verify that hive works with the compatible
+   * version
+   * @throws Exception
+   */
+  public void testVersionCompatibility () throws Exception {
+    System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false");
+    hiveConf = new HiveConf(this.getClass());
+    SessionState.start(new CliSessionState(hiveConf));
+    driver = new Driver(hiveConf);
+    driver.run("show tables");
+
+    System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true");
+    hiveConf = new HiveConf(this.getClass());
+    setVersion(hiveConf, "3.9000.0");
+    SessionState.start(new CliSessionState(hiveConf));
+    driver = new Driver(hiveConf);
+    CommandProcessorResponse proc = driver.run("show tables");
     assertEquals(0, proc.getResponseCode());
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/dbb76ead/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b409c07..718fa3b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -7749,34 +7749,36 @@ public class ObjectStore implements RawStore, Configurable {
     boolean strictValidation =
       HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION);
     // read the schema version stored in metastore db
-    String schemaVer = getMetaStoreSchemaVersion();
-    if (schemaVer == null) {
+    String dbSchemaVer = getMetaStoreSchemaVersion();
+    // version of schema for this version of hive
+    String hiveSchemaVer = MetaStoreSchemaInfo.getHiveSchemaVersion();
+
+    if (dbSchemaVer == null) {
       if (strictValidation) {
         throw new MetaException("Version information not found in metastore. ");
       } else {
         LOG.warn("Version information not found in metastore. "
             + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() +
             " is not enabled so recording the schema version " +
-            MetaStoreSchemaInfo.getHiveSchemaVersion());
-        setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+            hiveSchemaVer);
+        setMetaStoreSchemaVersion(hiveSchemaVer,
           "Set by MetaStore " + USER + "@" + HOSTNAME);
       }
     } else {
       // metastore schema version is different than Hive distribution needs
-      if (schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) {
-        LOG.debug("Found expected HMS version of " + schemaVer);
+      if (MetaStoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) {
+        LOG.debug("Found expected HMS version of " + dbSchemaVer);
       } else {
         if (strictValidation) {
-          throw new MetaException("Hive Schema version "
-              + MetaStoreSchemaInfo.getHiveSchemaVersion() +
-              " does not match metastore's schema version " + schemaVer +
+          throw new MetaException("Hive Schema version " + hiveSchemaVer +
+              " does not match metastore's schema version " + dbSchemaVer +
               " Metastore is not upgraded or corrupt");
         } else {
-          LOG.error("Version information found in metastore differs " + schemaVer +
-              " from expected schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion()
+
+          LOG.error("Version information found in metastore differs " + dbSchemaVer +
+              " from expected schema version " + hiveSchemaVer +
               ". Schema verififcation is disabled " +
               HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION + " so setting version.");
-          setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+          setMetaStoreSchemaVersion(hiveSchemaVer,
             "Set by MetaStore " + USER + "@" + HOSTNAME);
         }
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/dbb76ead/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
new file mode 100644
index 0000000..7142001
--- /dev/null
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test MetaStoreSchemaInfo
+ */
+public class TestMetaStoreSchemaInfo {
+
+  @Test
+  public void testIsVersionCompatible() throws Exception {
+    // first argument is hiveVersion, it is compatible if 2nd argument - dbVersion is
+    // greater than or equal to it
+    // check the compatible case
+    Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1"));
+    Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2"));
+    Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1"));
+    Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.0.9", "9.0.0"));
+
+    // check equivalent versions, should be compatible
+    Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.13.0", "0.13.1"));
+    Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.13.1", "0.13.0"));
+
+    // check incompatible versions
+    Assert.assertFalse(MetaStoreSchemaInfo.isVersionCompatible("0.1.1", "0.1.0"));
+    Assert.assertFalse(MetaStoreSchemaInfo.isVersionCompatible("4.0.1", "0.1.0"));
+
+  }
+
+}


Mime
View raw message