hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-15714 : backport HIVE-11985 (and HIVE-12601) to branch-1 (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Date Tue, 31 Jan 2017 02:22:18 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 f746dc6ab -> 70eb1f6e2


HIVE-15714 : backport HIVE-11985 (and HIVE-12601) to branch-1 (Sergey Shelukhin, reviewed
by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/70eb1f6e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/70eb1f6e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/70eb1f6e

Branch: refs/heads/branch-1
Commit: 70eb1f6e2b9b15f74c65ee480d07e3ef35b672dc
Parents: f746dc6
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Mon Jan 30 18:22:18 2017 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Mon Jan 30 18:22:18 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   | 14 +++++---
 .../hadoop/hive/metastore/MetaStoreUtils.java   | 24 ++++++++++---
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 25 +++++++++----
 .../apache/hadoop/hive/ql/metadata/Hive.java    | 17 +++++++--
 .../hadoop/hive/ql/metadata/Partition.java      | 15 +++++++-
 .../apache/hadoop/hive/ql/metadata/Table.java   | 38 +++++++++++++++++---
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  5 +++
 .../hadoop/hive/serde2/AbstractSerDe.java       |  9 +++++
 .../hadoop/hive/serde2/avro/AvroSerDe.java      | 19 ++++++++--
 9 files changed, 141 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 93cf71c..10b0644 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -874,11 +874,15 @@ public class HiveConf extends Configuration {
         "The default SerDe Hive will use for storage formats that do not specify a SerDe."),
 
     SERDESUSINGMETASTOREFORSCHEMA("hive.serdes.using.metastore.for.schema",
-        "org.apache.hadoop.hive.ql.io.orc.OrcSerde,org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,"
+
-        "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe,org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe,"
+
-        "org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe,"
+
-        "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe,org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe",
-        "SerDes retriving schema from metastore. This an internal parameter. Check with the
hive dev. team"),
+        "org.apache.hadoop.hive.ql.io.orc.OrcSerde," +
+        "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe," +
+        "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe," +
+        "org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe," +
+        "org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe," +
+        "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," +
+        "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," +
+        "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe",
+        "SerDes retrieving schema from metastore. This is an internal parameter."),
 
     HIVEHISTORYFILELOC("hive.querylog.location",
         "${system:java.io.tmpdir}" + File.separator + "${system:user.name}",

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 961359e..7878f16 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -385,6 +385,12 @@ public class MetaStoreUtils {
     if (lib == null) {
       return null;
     }
+    return getDeserializer(conf, table, skipConfError, lib);
+  }
+
+  public static Deserializer getDeserializer(Configuration conf,
+      org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError,
+      String lib) throws MetaException {
     try {
       Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib).
               asSubclass(Deserializer.class), conf);
@@ -549,8 +555,9 @@ public class MetaStoreUtils {
       if (!validateColumnName(fieldSchema.getName())) {
         return "name: " + fieldSchema.getName();
       }
-      if (!validateColumnType(fieldSchema.getType())) {
-        return "type: " + fieldSchema.getType();
+      String typeError = validateColumnType(fieldSchema.getType());
+      if (typeError != null) {
+        return typeError;
       }
     }
     return null;
@@ -625,6 +632,9 @@ public class MetaStoreUtils {
     return false;
   }
 
+  public static final int MAX_MS_TYPENAME_LENGTH = 2000; // 4000/2, for an unlikely unicode
case
+
+  public static final String TYPE_FROM_DESERIALIZER = "<derived from deserializer>";
   /**
    * validate column type
    *
@@ -632,7 +642,11 @@ public class MetaStoreUtils {
    * @param name
    * @return
    */
-  static public boolean validateColumnType(String type) {
+  static public String validateColumnType(String type) {
+    if (type.equals(TYPE_FROM_DESERIALIZER)) return null;
+    if (type.length() > MAX_MS_TYPENAME_LENGTH) {
+      return "type name is too long: " + type;
+    }
     int last = 0;
     boolean lastAlphaDigit = isValidTypeChar(type.charAt(last));
     for (int i = 1; i <= type.length(); i++) {
@@ -641,12 +655,12 @@ public class MetaStoreUtils {
         String token = type.substring(last, i);
         last = i;
         if (!hiveThriftTypeMap.contains(token)) {
-          return false;
+          return "type: " + type;
         }
         break;
       }
     }
-    return true;
+    return null;
   }
 
   private static boolean isValidTypeChar(char c) {

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index cb667fc..7d972e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -204,6 +204,7 @@ import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeSpec;
 import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
 import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe;
@@ -3112,7 +3113,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
           cols.addAll(tbl.getPartCols());
         }
       } else {
-        cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
+        cols = Hive.getFieldsFromDeserializer(colPath, deserializer); // TODO#: here - desc
         if (descTbl.isFormatted()) {
           // when column name is specified in describe table DDL, colPath will
           // will be table_name.column_name
@@ -3366,7 +3367,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
       tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName()));
       tbl.setTableName(Utilities.getTableName(alterTbl.getNewName()));
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDCOLS) {
-      List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
+      List<FieldSchema> oldCols = (part == null
+          ? tbl.getColsForMetastore() : part.getColsForMetastore());
       StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       List<FieldSchema> newCols = alterTbl.getNewCols();
       String serializationLib = sd.getSerdeInfo().getSerializationLib();
@@ -3394,7 +3396,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
         sd.setCols(oldCols);
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAMECOLUMN) {
-      List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
+      List<FieldSchema> oldCols = (part == null
+          ? tbl.getColsForMetastore() : part.getColsForMetastore());
       StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       List<FieldSchema> newCols = new ArrayList<FieldSchema>();
       Iterator<FieldSchema> iterOldCols = oldCols.iterator();
@@ -3536,11 +3539,21 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
         sd.getSerdeInfo().getParameters().putAll(alterTbl.getProps());
       }
       if (part != null) {
+        // TODO: wtf? This doesn't do anything.
         part.getTPartition().getSd().setCols(part.getTPartition().getSd().getCols());
       } else {
-        if (!Table.hasMetastoreBasedSchema(conf, serdeName)) {
-          tbl.setFields(Hive.getFieldsFromDeserializer(tbl.getTableName(), tbl.
-              getDeserializer()));
+        if (Table.shouldStoreFieldsInMetastore(conf, serdeName, tbl.getParameters())
+            && !Table.hasMetastoreBasedSchema(conf, oldSerdeName)) {
+          // If new SerDe needs to store fields in metastore, but the old serde doesn't,
save
+          // the fields so that new SerDe could operate. Note that this may fail if some
fields
+          // from old SerDe are too long to be stored in metastore, but there's nothing we
can do.
+          try {
+            Deserializer oldSerde = MetaStoreUtils.getDeserializer(
+                conf, tbl.getTTable(), false, oldSerdeName);
+            tbl.setFields(Hive.getFieldsFromDeserializer(tbl.getTableName(), oldSerde));
+          } catch (MetaException ex) {
+            throw new HiveException(ex);
+          }
         }
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDFILEFORMAT) {

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index c2ab4bf..48b781d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -724,6 +724,16 @@ public class Hive {
     }
   }
 
+  public static List<FieldSchema> getFieldsFromDeserializerForMsStorage(
+      Table tbl, Deserializer deserializer) throws SerDeException, MetaException {
+    List<FieldSchema> schema = MetaStoreUtils.getFieldsFromDeserializer(
+        tbl.getTableName(), deserializer);
+    for (FieldSchema field : schema) {
+      field.setType(MetaStoreUtils.TYPE_FROM_DESERIALIZER);
+    }
+    return schema;
+  }
+
   /**
    *
    * @param tableName
@@ -848,8 +858,11 @@ public class Hive {
       List<Order> sortCols = new ArrayList<Order>();
       int k = 0;
       Table metaBaseTbl = new Table(baseTbl);
-      for (int i = 0; i < metaBaseTbl.getCols().size(); i++) {
-        FieldSchema col = metaBaseTbl.getCols().get(i);
+      // Even though we are storing these in metastore, get regular columns. Indexes on lengthy
+      // types from e.g. Avro schema will just fail to create the index table (by design).
+      List<FieldSchema> cols = metaBaseTbl.getCols();
+      for (int i = 0; i < cols.size(); i++) {
+        FieldSchema col = cols.get(i);
         if (indexedCols.contains(col.getName())) {
           indexTblCols.add(col);
           sortCols.add(new Order(col.getName(), 1));

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
index 08ff2e9..21a55dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
@@ -492,10 +492,23 @@ public class Partition implements Serializable {
   }
 
   public List<FieldSchema> getCols() {
+    return getColsInternal(false);
+  }
+
+  public List<FieldSchema> getColsForMetastore() {
+    return getColsInternal(true);
+  }
+
+  private List<FieldSchema> getColsInternal(boolean forMs) {
 
     try {
-      if (Table.hasMetastoreBasedSchema(SessionState.getSessionConf(), tPartition.getSd()))
{
+      String serializationLib = tPartition.getSd().getSerdeInfo().getSerializationLib();
+      // Do the lightweight check for general case.
+      if (Table.hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib))
{
         return tPartition.getSd().getCols();
+      } else if (forMs && !Table.shouldStoreFieldsInMetastore(
+          SessionState.getSessionConf(), serializationLib, table.getParameters())) {
+        return Hive.getFieldsFromDeserializerForMsStorage(table, getDeserializer());
       }
       return MetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), getDeserializer());
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index e53933e..fddb6d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
@@ -64,6 +65,7 @@ import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.OutputFormat;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * A Hive Table: is a fundamental unit of data in Hive that shares a common schema/DDL.
@@ -596,11 +598,22 @@ public class Table implements Serializable {
   }
 
   public List<FieldSchema> getCols() {
+    return getColsInternal(false);
+  }
+
+  public List<FieldSchema> getColsForMetastore() {
+    return getColsInternal(true);
+  }
 
+  private List<FieldSchema> getColsInternal(boolean forMs) {
     String serializationLib = getSerializationLib();
     try {
+      // Do the lightweight check for general case.
       if (hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib)) {
         return tTable.getSd().getCols();
+      } else if (forMs && !shouldStoreFieldsInMetastore(
+          SessionState.getSessionConf(), serializationLib, tTable.getParameters())) {
+        return Hive.getFieldsFromDeserializerForMsStorage(this, getDeserializer());
       } else {
         return MetaStoreUtils.getFieldsFromDeserializer(getTableName(), getDeserializer());
       }
@@ -934,15 +947,28 @@ public class Table implements Serializable {
     return tTable.isTemporary();
   }
 
-  public static boolean hasMetastoreBasedSchema(HiveConf conf, StorageDescriptor serde) {
-    return hasMetastoreBasedSchema(conf, serde.getSerdeInfo().getSerializationLib());
-  }
-
   public static boolean hasMetastoreBasedSchema(HiveConf conf, String serdeLib) {
     return StringUtils.isEmpty(serdeLib) ||
         conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(serdeLib);
   }
 
+  public static boolean shouldStoreFieldsInMetastore(
+      HiveConf conf, String serdeLib, Map<String, String> tableParams) {
+    if (hasMetastoreBasedSchema(conf, serdeLib))  return true;
+    // Table may or may not be using metastore. Only the SerDe can tell us.
+    AbstractSerDe deserializer = null;
+    try {
+      Class<?> clazz = conf.getClassByName(serdeLib);
+      if (!AbstractSerDe.class.isAssignableFrom(clazz)) return true; // The default.
+      deserializer = ReflectionUtil.newInstance(
+          conf.getClassByName(serdeLib).asSubclass(AbstractSerDe.class), conf);
+    } catch (Exception ex) {
+      LOG.warn("Cannot initialize SerDe: " + serdeLib + ", ignoring", ex);
+      return true;
+    }
+    return deserializer.shouldStoreFieldsInMetastore(tableParams);
+  }
+
   public static void validateColumns(List<FieldSchema> columns, List<FieldSchema>
partCols)
       throws HiveException {
     List<String> colNames = new ArrayList<String>();
@@ -982,4 +1008,8 @@ public class Table implements Serializable {
     this.tableSpec = tableSpec;
   }
 
+  public boolean hasDeserializer() {
+    return deserializer != null;
+  }
+
 };

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 74f5867..98d3fc5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10433,6 +10433,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         FieldSchema fieldSchema = derivedSchema.get(i);
         // Modify a copy, not the original
         fieldSchema = new FieldSchema(fieldSchema);
+        // TODO: there's a potential problem here if some table uses external schema like
Avro,
+        //       with a very large type name. It seems like the view does not derive the
SerDe from
+        //       the table, so it won't be able to just get the type from the deserializer
like the
+        //       table does; we won't be able to properly store the type in the RDBMS metastore.
+        //       Not sure if these large cols could be in resultSchema. Ignore this for now
0_o
         derivedSchema.set(i, fieldSchema);
         sb.append(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf));
         sb.append(" AS ");

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
index c5e78c5..9269ff4 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.serde2;
 
+import java.util.Map;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
@@ -115,4 +116,12 @@ public abstract class AbstractSerDe implements SerDe {
   public String getConfigurationErrors() {
     return configErrors == null ? "" : configErrors;
   }
+
+  /**
+   * @rturn Whether the SerDe that can store schema both inside and outside of metastore
+   *        does, in fact, store it inside metastore, based on table parameters.
+   */
+  public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) {
+    return false; // The default, unless SerDe overrides it. TODO#
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
index efff663..7271095 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.avro;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 import org.apache.avro.Schema;
@@ -96,8 +97,7 @@ public class AvroSerDe extends AbstractSerDe {
     final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
     final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS,"");
 
-    if (properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName())
!= null
-        || properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName())
!= null
+    if (hasExternalSchema(properties)
         || columnNameProperty == null || columnNameProperty.isEmpty()
         || columnTypeProperty == null || columnTypeProperty.isEmpty()) {
       schema = determineSchemaOrReturnErrorSchema(configuration, properties);
@@ -127,6 +127,16 @@ public class AvroSerDe extends AbstractSerDe {
     this.oi = aoig.getObjectInspector();
   }
 
+  private boolean hasExternalSchema(Properties properties) {
+    return properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName())
!= null
+        || properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName())
!= null;
+  }
+
+  private boolean hasExternalSchema(Map<String, String> tableParams) {
+    return tableParams.containsKey(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName())
+        || tableParams.containsKey(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName());
+  }
+
   public static Schema getSchemaFromCols(Properties properties,
           List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty)
{
     List<String> columnComments;
@@ -227,4 +237,9 @@ public class AvroSerDe extends AbstractSerDe {
 
     return avroSerializer;
   }
+
+  @Override
+  public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) {
+    return !hasExternalSchema(tableParams);
+  }
 }


Mime
View raw message