hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1614214 [9/9] - in /hive/branches/cbo: ./ common/ common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src/test/results/clientpositive/ hbase-handler/src/test/results/positive/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/...
Date Mon, 28 Jul 2014 23:20:59 GMT
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java?rev=1614214&r1=1614213&r2=1614214&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java Mon
Jul 28 23:20:54 2014
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.serde2.avro;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 
@@ -29,6 +31,7 @@ import org.apache.hadoop.hive.serde2.Ser
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.io.Writable;
 
 /**
@@ -50,6 +53,8 @@ public class AvroSerDe extends AbstractS
   private AvroSerializer avroSerializer = null;
 
   private boolean badSchema = false;
+  private static String TABLE_NAME = "name";
+  private static String TABLE_COMMENT = "comment";
 
   @Override
   public void initialize(Configuration configuration, Properties tableProperties,
@@ -61,17 +66,56 @@ public class AvroSerDe extends AbstractS
   @Override
   public void initialize(Configuration configuration, Properties properties) throws SerDeException
{
     // Reset member variables so we don't get in a half-constructed state
-    if(schema != null) {
+    if (schema != null) {
       LOG.info("Resetting already initialized AvroSerDe");
     }
 
     schema = null;
     oi = null;
-    columnNames  = null;
+    columnNames = null;
     columnTypes = null;
 
-    schema =  AvroSerdeUtils.determineSchemaOrReturnErrorSchema(properties);
-    if(configuration == null) {
+    final String columnNameProperty = properties.getProperty("columns");
+    final String columnTypeProperty = properties.getProperty("columns.types");
+    final String columnCommentProperty = properties.getProperty("columns.comments");
+
+    if (properties.getProperty(AvroSerdeUtils.SCHEMA_LITERAL) != null
+        || properties.getProperty(AvroSerdeUtils.SCHEMA_URL) != null
+        || columnNameProperty == null || columnNameProperty.isEmpty()
+        || columnTypeProperty == null || columnTypeProperty.isEmpty()) {
+      schema = AvroSerdeUtils.determineSchemaOrReturnErrorSchema(properties);
+    } else {
+      // Get column names and sort order
+      columnNames = Arrays.asList(columnNameProperty.split(","));
+      columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
+
+      List<String> columnComments;
+      if (columnCommentProperty.isEmpty()) {
+        columnComments = new ArrayList<String>();
+      } else {
+        columnComments = Arrays.asList(columnCommentProperty.split(","));
+        LOG.info("columnComments is " + columnCommentProperty);
+      }
+      if (columnNames.size() != columnTypes.size()) {
+        throw new IllegalArgumentException("AvroSerde initialization failed. Number of column
" +
+            "name and column type differs. columnNames = " + columnNames + ", columnTypes
= " +
+            columnTypes);
+      }
+
+      final String tableName = properties.getProperty(TABLE_NAME);
+      final String tableComment = properties.getProperty(TABLE_COMMENT);
+      TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema();
+      schema = typeInfoToSchema.convert(columnNames, columnTypes, columnComments,
+          properties.getProperty(AvroSerdeUtils.SCHEMA_NAMESPACE),
+          properties.getProperty(AvroSerdeUtils.SCHEMA_NAME, tableName),
+          properties.getProperty(AvroSerdeUtils.SCHEMA_DOC, tableComment));
+
+      properties.setProperty(AvroSerdeUtils.SCHEMA_LITERAL, schema.toString());
+    }
+
+    LOG.info("Avro schema is " + schema);
+
+    if (configuration == null) {
       LOG.info("Configuration null, not inserting schema");
     } else {
       configuration.set(AvroSerdeUtils.AVRO_SERDE_SCHEMA, schema.toString(false));

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java?rev=1614214&r1=1614213&r2=1614214&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
(original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
Mon Jul 28 23:20:54 2014
@@ -49,6 +49,9 @@ public class AvroSerdeUtils {
   public static final String SCHEMA_LITERAL = "avro.schema.literal";
   public static final String SCHEMA_URL = "avro.schema.url";
   public static final String SCHEMA_NONE = "none";
+  public static final String SCHEMA_NAMESPACE = "avro.schema.namespace";
+  public static final String SCHEMA_NAME = "avro.schema.name";
+  public static final String SCHEMA_DOC = "avro.schema.doc";
   public static final String EXCEPTION_MESSAGE = "Neither " + SCHEMA_LITERAL + " nor "
           + SCHEMA_URL + " specified, can't determine table schema";
   public static final String AVRO_SERDE_SCHEMA = "avro.serde.schema";

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveDecimalObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveDecimalObjectInspector.java?rev=1614214&r1=1614213&r2=1614214&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveDecimalObjectInspector.java
(original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveDecimalObjectInspector.java
Mon Jul 28 23:20:54 2014
@@ -35,11 +35,17 @@ implements SettableHiveDecimalObjectInsp
 
   @Override
   public HiveDecimalWritable getPrimitiveWritableObject(Object o) {
+    if (o == null) {
+      return null;
+    }
     return enforcePrecisionScale(((HiveDecimalWritable) o));
   }
 
   @Override
   public HiveDecimal getPrimitiveJavaObject(Object o) {
+    if (o == null) {
+      return null;
+    }
     return enforcePrecisionScale(((HiveDecimalWritable)o).getHiveDecimal());
   }
 



Mime
View raw message