hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [1/4] hive git commit: HIVE-19753: Strict managed tables mode in Hive (Jason Dere, reviewed by Eugene Koifman)
Date Sun, 24 Jun 2018 17:10:33 GMT
Repository: hive
Updated Branches:
  refs/heads/master 4b7f88ad1 -> 091186484


http://git-wip-us.apache.org/repos/asf/hive/blob/09118648/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 33cf542..d9fdd29 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.events.PreEventContext;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.utils.HiveStrictManagedUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -78,12 +79,14 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
   private void handle(PreAlterTableEvent context) throws MetaException {
     if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) {
       handleAlterTableTransactionalProp(context);
+      HiveStrictManagedUtils.validateStrictManagedTableWithThrow(getConf(), context.getNewTable());
     }
   }
 
   private void handle(PreCreateTableEvent context) throws MetaException {
     if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) {
       handleCreateTableTransactionalProp(context);
+      HiveStrictManagedUtils.validateStrictManagedTableWithThrow(getConf(), context.getTable());
     }
   }
 
@@ -329,7 +332,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
         }
       }
 
-      if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
+      if (MetaStoreUtils.isExternalTable(newTable)) {
         throw new MetaException(Warehouse.getQualifiedName(newTable) +
             " cannot be declared transactional because it's an external table");
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/09118648/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 54d8830..f3b909c 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -748,6 +748,9 @@ public class MetastoreConf {
         "The class to use to read schemas from storage.  It must implement " +
         "org.apache.hadoop.hive.metastore.StorageSchemaReader"),
     STORE_MANAGER_TYPE("datanucleus.storeManagerType", "datanucleus.storeManagerType", "rdbms",
"metadata store type"),
+    STRICT_MANAGED_TABLES("metastore.strict.managed.tables", "hive.strict.managed.tables",
false,
+            "Whether strict managed tables mode is enabled. With this mode enabled, " +
+            "only transactional tables (both full and insert-only) are allowed to be created
as managed tables"),
     SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES("metastore.support.special.characters.tablename",
         "hive.support.special.characters.tablename", true,
         "This flag should be set to true to enable support for special characters in table
names.\n"

http://git-wip-us.apache.org/repos/asf/hive/blob/09118648/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
new file mode 100644
index 0000000..fac3c22
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.utils;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.TableType;
+
+public class HiveStrictManagedUtils {
+
+  public static void validateStrictManagedTableWithThrow(Configuration conf, Table table)
+      throws MetaException {
+    String reason = validateStrictManagedTable(conf, table);
+    if (reason != null) {
+      throw new MetaException(reason);
+    }
+  }
+
+  /**
+   * Checks if the table is valid based on the rules for strict managed tables.
+   * @param conf
+   * @param table
+   * @return  Null if the table is valid, otherwise a string message indicating why the table
is invalid.
+   */
+  public static String validateStrictManagedTable(Configuration conf,
+      Table table) {
+    if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STRICT_MANAGED_TABLES)) {
+      if (table.isTemporary()) {
+        // temp tables exempted from checks.
+        return null;
+      }
+
+      TableType tableType = TableType.valueOf(table.getTableType());
+      if (tableType == TableType.MANAGED_TABLE) {
+        if (!MetaStoreUtils.isTransactionalTable(table.getParameters())) {
+          return createValidationError(table, "Table is marked as a managed table but is
not transactional.");
+        }
+        if (MetaStoreUtils.isNonNativeTable(table)) {
+          return createValidationError(table, "Table is marked as a managed table but is
non-native.");
+        }
+        if (isAvroTableWithExternalSchema(table)) {
+          return createValidationError(table, "Managed Avro table has externally defined
schema.");
+        }
+      }
+    }
+
+    // Table is valid
+    return null;
+  }
+
+  private static final String AVRO_SERDE_CLASSNAME = "org.apache.hadoop.hive.serde2.avro.AvroSerDe";
+  private static final String AVRO_SCHEMA_URL_PROPERTY = "avro.schema.url";
+
+  public static boolean isAvroTableWithExternalSchema(Table table) {
+    if (table.getSd().getSerdeInfo().getSerializationLib().equals(AVRO_SERDE_CLASSNAME))
{
+      String schemaUrl = table.getParameters().get(AVRO_SCHEMA_URL_PROPERTY);
+      if (schemaUrl != null && !schemaUrl.isEmpty()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public static boolean isListBucketedTable(Table table) {
+    return table.getSd().isStoredAsSubDirectories();
+  }
+
+  private static String createValidationError(Table table, String message) {
+    StringBuilder sb = new StringBuilder();
+    sb.append("Table ");
+    sb.append(table.getDbName());
+    sb.append(".");
+    sb.append(table.getTableName());
+    sb.append(" failed strict managed table checks due to the following reason: ");
+    sb.append(message);
+    return sb.toString();
+  }
+}


Mime
View raw message