hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject svn commit: r1157224 - in /hive/trunk: ivy/ metastore/src/java/org/apache/hadoop/hive/metastore/ serde/ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/ serde/sr...
Date Fri, 12 Aug 2011 19:26:29 GMT
Author: jvs
Date: Fri Aug 12 19:26:29 2011
New Revision: 1157224

URL: http://svn.apache.org/viewvc?rev=1157224&view=rev
Log:
HIVE-2171. Allow custom serdes to set field comments
(Jakob Homan via jvs)


Added:
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
Modified:
    hive/trunk/ivy/libraries.properties
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    hive/trunk/serde/ivy.xml
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java

Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Fri Aug 12 19:26:29 2011
@@ -44,6 +44,7 @@ junit.version=3.8.1
 ivy.version=2.1.0
 log4j.version=1.2.15
 maven-ant-tasks.version=2.1.0
+mockito-all.version=1.8.2
 slf4j-api.version=1.6.1
 slf4j-log4j12.version=1.6.1
 thrift.version=0.5.0

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Fri
Aug 12 19:26:29 2011
@@ -823,21 +823,27 @@ public class MetaStoreUtils {
     // rules on how to recurse the ObjectInspector based on its type
     if (oi.getCategory() != Category.STRUCT) {
       str_fields.add(new FieldSchema(last_name, oi.getTypeName(),
-          "from deserializer"));
+          FROM_SERIALIZER));
     } else {
       List<? extends StructField> fields = ((StructObjectInspector) oi)
           .getAllStructFieldRefs();
       for (int i = 0; i < fields.size(); i++) {
-        String fieldName = fields.get(i).getFieldName();
-        String fieldTypeName = fields.get(i).getFieldObjectInspector()
-            .getTypeName();
-        str_fields.add(new FieldSchema(fieldName, fieldTypeName,
-            "from deserializer"));
+        StructField structField = fields.get(i);
+        String fieldName = structField.getFieldName();
+        String fieldTypeName = structField.getFieldObjectInspector().getTypeName();
+        String fieldComment = determineFieldComment(structField.getFieldComment());
+
+        str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment));
       }
     }
     return str_fields;
   }
 
+  private static final String FROM_SERIALIZER = "from deserializer";
+  private static String determineFieldComment(String comment) {
+    return (comment == null || comment.isEmpty()) ? FROM_SERIALIZER : comment;
+  }
+
   /**
    * Convert TypeInfo to FieldSchema.
    */

Modified: hive/trunk/serde/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/serde/ivy.xml?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/ivy.xml (original)
+++ hive/trunk/serde/ivy.xml Fri Aug 12 19:26:29 2011
@@ -28,5 +28,6 @@
         </dependency>
         <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}"/>
         <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"/>
+        <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}"/>
     </dependencies>
 </ivy-module>

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
Fri Aug 12 19:26:29 2011
@@ -46,6 +46,16 @@ public final class LazyObjectInspectorFa
       List<ObjectInspector> structFieldObjectInspectors, byte separator,
       Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
       byte escapeChar) {
+    return getLazySimpleStructObjectInspector(structFieldNames,
+      structFieldObjectInspectors, null, separator, nullSequence,
+      lastColumnTakesRest, escaped, escapeChar);
+  }
+
+  public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
+      List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments,
+      byte separator, Text nullSequence, boolean lastColumnTakesRest,
+      boolean escaped,byte escapeChar) {
     ArrayList<Object> signature = new ArrayList<Object>();
     signature.add(structFieldNames);
     signature.add(structFieldObjectInspectors);
@@ -54,12 +64,15 @@ public final class LazyObjectInspectorFa
     signature.add(Boolean.valueOf(lastColumnTakesRest));
     signature.add(Boolean.valueOf(escaped));
     signature.add(Byte.valueOf(escapeChar));
+    if(structFieldComments != null) {
+      signature.add(structFieldComments);
+    }
     LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector
         .get(signature);
     if (result == null) {
       result = new LazySimpleStructObjectInspector(structFieldNames,
-          structFieldObjectInspectors, separator, nullSequence,
-          lastColumnTakesRest, escaped, escapeChar);
+          structFieldObjectInspectors, structFieldComments, separator,
+          nullSequence, lastColumnTakesRest, escaped, escapeChar);
       cachedLazySimpleStructObjectInspector.put(signature, result);
     }
     return result;

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -49,6 +49,7 @@ public class LazySimpleStructObjectInspe
     protected int fieldID;
     protected String fieldName;
     protected ObjectInspector fieldObjectInspector;
+    protected String fieldComment;
 
     public MyField(int fieldID, String fieldName,
         ObjectInspector fieldObjectInspector) {
@@ -57,6 +58,11 @@ public class LazySimpleStructObjectInspe
       this.fieldObjectInspector = fieldObjectInspector;
     }
 
+    public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector, String
fieldComment) {
+      this(fieldID, fieldName, fieldObjectInspector);
+      this.fieldComment = fieldComment;
+    }
+
     public int getFieldID() {
       return fieldID;
     }
@@ -69,6 +75,10 @@ public class LazySimpleStructObjectInspe
       return fieldObjectInspector;
     }
 
+    public String getFieldComment() {
+      return fieldComment;
+    }
+
     @Override
     public String toString() {
       return "" + fieldID + ":" + fieldName;
@@ -95,15 +105,26 @@ public class LazySimpleStructObjectInspe
       List<ObjectInspector> structFieldObjectInspectors, byte separator,
       Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
       byte escapeChar) {
-    init(structFieldNames, structFieldObjectInspectors, separator,
+    init(structFieldNames, structFieldObjectInspectors, null, separator,
         nullSequence, lastColumnTakesRest, escaped, escapeChar);
   }
 
+  public LazySimpleStructObjectInspector(List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments, byte separator, Text nullSequence,
+      boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
+    init(structFieldNames, structFieldObjectInspectors, structFieldComments,
+        separator, nullSequence, lastColumnTakesRest, escaped, escapeChar);
+  }
+
   protected void init(List<String> structFieldNames,
-      List<ObjectInspector> structFieldObjectInspectors, byte separator,
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments, byte separator,
       Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
       byte escapeChar) {
     assert (structFieldNames.size() == structFieldObjectInspectors.size());
+    assert (structFieldComments == null ||
+            structFieldNames.size() == structFieldComments.size());
 
     this.separator = separator;
     this.nullSequence = nullSequence;
@@ -114,7 +135,8 @@ public class LazySimpleStructObjectInspe
     fields = new ArrayList<MyField>(structFieldNames.size());
     for (int i = 0; i < structFieldNames.size(); i++) {
       fields.add(new MyField(i, structFieldNames.get(i),
-          structFieldObjectInspectors.get(i)));
+          structFieldObjectInspectors.get(i),
+          structFieldComments == null ? null : structFieldComments.get(i)));
     }
   }
 
@@ -131,7 +153,7 @@ public class LazySimpleStructObjectInspe
     this.fields = new ArrayList<MyField>(fields.size());
     for (int i = 0; i < fields.size(); i++) {
       this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
-          .get(i).getFieldObjectInspector()));
+          .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment()));
     }
   }
 

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
Fri Aug 12 19:26:29 2011
@@ -42,14 +42,24 @@ public final class LazyBinaryObjectInspe
   public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
       List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors) {
-    ArrayList<Object> signature = new ArrayList<Object>();
+    return getLazyBinaryStructObjectInspector(structFieldNames,
+                                              structFieldObjectInspectors, null);
+  }
+
+  public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
+      List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments)
{
+    ArrayList<Object> signature = new ArrayList<Object>(3);
     signature.add(structFieldNames);
     signature.add(structFieldObjectInspectors);
+    if(structFieldComments != null) {
+      signature.add(structFieldComments);
+    }
     LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector
         .get(signature);
     if (result == null) {
       result = new LazyBinaryStructObjectInspector(structFieldNames,
-          structFieldObjectInspectors);
+          structFieldObjectInspectors, structFieldComments);
       cachedLazyBinaryStructObjectInspector.put(signature, result);
     }
     return result;

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -37,6 +37,12 @@ public class LazyBinaryStructObjectInspe
     super(structFieldNames, structFieldObjectInspectors);
   }
 
+  protected LazyBinaryStructObjectInspector(List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments) {
+    super(structFieldNames, structFieldObjectInspectors, structFieldComments);
+  }
+
   protected LazyBinaryStructObjectInspector(List<StructField> fields) {
     super(fields);
   }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -44,6 +44,7 @@ class ColumnarStructObjectInspector exte
     protected int fieldID;
     protected String fieldName;
     protected ObjectInspector fieldObjectInspector;
+    protected String fieldComment;
 
     public MyField(int fieldID, String fieldName,
         ObjectInspector fieldObjectInspector) {
@@ -52,6 +53,12 @@ class ColumnarStructObjectInspector exte
       this.fieldObjectInspector = fieldObjectInspector;
     }
 
+    public MyField(int fieldID, String fieldName,
+        ObjectInspector fieldObjectInspector, String fieldComment) {
+      this(fieldID, fieldName, fieldObjectInspector);
+      this.fieldComment = fieldComment;
+    }
+
     public int getFieldID() {
       return fieldID;
     }
@@ -64,6 +71,9 @@ class ColumnarStructObjectInspector exte
       return fieldObjectInspector;
     }
 
+    public String getFieldComment() {
+      return fieldComment;
+    }
     @Override
     public String toString() {
       return "" + fieldID + ":" + fieldName;
@@ -82,17 +92,27 @@ class ColumnarStructObjectInspector exte
    */
   public ColumnarStructObjectInspector(List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors) {
-    init(structFieldNames, structFieldObjectInspectors);
+    init(structFieldNames, structFieldObjectInspectors, null);
+  }
+
+  public ColumnarStructObjectInspector(List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments) {
+    init(structFieldNames, structFieldObjectInspectors, structFieldComments);
   }
 
   protected void init(List<String> structFieldNames,
-      List<ObjectInspector> structFieldObjectInspectors) {
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments) {
     assert (structFieldNames.size() == structFieldObjectInspectors.size());
+    assert (structFieldComments == null ||
+           (structFieldNames.size() == structFieldComments.size()));
 
     fields = new ArrayList<MyField>(structFieldNames.size());
     for (int i = 0; i < structFieldNames.size(); i++) {
       fields.add(new MyField(i, structFieldNames.get(i),
-          structFieldObjectInspectors.get(i)));
+          structFieldObjectInspectors.get(i),
+          structFieldComments == null ? null : structFieldComments.get(i)));
     }
   }
 
@@ -104,7 +124,7 @@ class ColumnarStructObjectInspector exte
     this.fields = new ArrayList<MyField>(fields.size());
     for (int i = 0; i < fields.size(); i++) {
       this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
-          .get(i).getFieldObjectInspector()));
+          .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment()));
     }
   }
 

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.serde2.objectinspector;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 
@@ -38,17 +39,33 @@ import org.apache.hadoop.hive.serde2.obj
 public class MetadataListStructObjectInspector extends
     StandardStructObjectInspector {
 
-  static HashMap<List<String>, MetadataListStructObjectInspector> cached = new
HashMap<List<String>, MetadataListStructObjectInspector>();
+  static HashMap<List<List<String>>, MetadataListStructObjectInspector>
+      cached = new HashMap<List<List<String>>, MetadataListStructObjectInspector>();
 
   // public static MetadataListStructObjectInspector getInstance(int fields) {
   // return getInstance(ObjectInspectorUtils.getIntegerArray(fields));
   // }
   public static MetadataListStructObjectInspector getInstance(
       List<String> columnNames) {
+    ArrayList<List<String>> key = new ArrayList<List<String>>(1);
+    key.add(columnNames);
     MetadataListStructObjectInspector result = cached.get(columnNames);
     if (result == null) {
       result = new MetadataListStructObjectInspector(columnNames);
-      cached.put(columnNames, result);
+      cached.put(key, result);
+    }
+    return result;
+  }
+
+  public static MetadataListStructObjectInspector getInstance(
+      List<String> columnNames, List<String> columnComments) {
+    ArrayList<List<String>> key = new ArrayList<List<String>>(2);
+    Collections.addAll(key, columnNames, columnComments);
+
+    MetadataListStructObjectInspector result = cached.get(key);
+    if (result == null) {
+      result = new MetadataListStructObjectInspector(columnNames, columnComments);
+      cached.put(key, result);
     }
     return result;
   }
@@ -66,6 +83,12 @@ public class MetadataListStructObjectIns
     super(columnNames, getFieldObjectInspectors(columnNames.size()));
   }
 
+  public MetadataListStructObjectInspector(List<String> columnNames,
+                                           List<String> columnComments) {
+    super(columnNames, getFieldObjectInspectors(columnNames.size()),
+          columnComments);
+  }
+
   // Get col object out
   @Override
   public Object getStructFieldData(Object data, StructField fieldRef) {

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
Fri Aug 12 19:26:29 2011
@@ -242,14 +242,22 @@ public final class ObjectInspectorFactor
   public static StandardStructObjectInspector getStandardStructObjectInspector(
       List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors) {
-    ArrayList<List<?>> signature = new ArrayList<List<?>>();
+    return getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors,
null);
+  }
+
+  public static StandardStructObjectInspector getStandardStructObjectInspector(
+      List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structComments) {
+    ArrayList<List<?>> signature = new ArrayList<List<?>>(3);
     signature.add(structFieldNames);
     signature.add(structFieldObjectInspectors);
-    StandardStructObjectInspector result = cachedStandardStructObjectInspector
-        .get(signature);
-    if (result == null) {
-      result = new StandardStructObjectInspector(structFieldNames,
-          structFieldObjectInspectors);
+    if(structComments != null) {
+      signature.add(structComments);
+    }
+    StandardStructObjectInspector result = cachedStandardStructObjectInspector.get(signature);
+    if(result == null) {
+      result = new StandardStructObjectInspector(structFieldNames, structFieldObjectInspectors,
structComments);
       cachedStandardStructObjectInspector.put(signature, result);
     }
     return result;
@@ -273,14 +281,23 @@ public final class ObjectInspectorFactor
   public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
       List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors) {
-    ArrayList<Object> signature = new ArrayList<Object>();
+    return getColumnarStructObjectInspector(structFieldNames, structFieldObjectInspectors,
null);
+  }
+
+  public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
+      List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments)
{
+    ArrayList<Object> signature = new ArrayList<Object>(3);
     signature.add(structFieldNames);
     signature.add(structFieldObjectInspectors);
+    if(structFieldComments != null) {
+      signature.add(structFieldComments);
+    }
     ColumnarStructObjectInspector result = cachedColumnarStructObjectInspector
         .get(signature);
     if (result == null) {
       result = new ColumnarStructObjectInspector(structFieldNames,
-          structFieldObjectInspectors);
+          structFieldObjectInspectors, structFieldComments);
       cachedColumnarStructObjectInspector.put(signature, result);
     }
     return result;

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -58,6 +58,10 @@ public class ReflectionStructObjectInspe
       return fieldObjectInspector;
     }
 
+    public String getFieldComment() {
+      return null;
+    }
+
     @Override
     public String toString() {
       return field.toString();

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -46,6 +46,7 @@ public class StandardStructObjectInspect
     protected int fieldID;
     protected String fieldName;
     protected ObjectInspector fieldObjectInspector;
+    protected String fieldComment;
 
     public MyField(int fieldID, String fieldName,
         ObjectInspector fieldObjectInspector) {
@@ -54,6 +55,12 @@ public class StandardStructObjectInspect
       this.fieldObjectInspector = fieldObjectInspector;
     }
 
+    public MyField(int fieldID, String fieldName,
+        ObjectInspector fieldObjectInspector, String fieldComment) {
+      this(fieldID, fieldName, fieldObjectInspector);
+      this.fieldComment = fieldComment;
+    }
+
     public int getFieldID() {
       return fieldID;
     }
@@ -66,6 +73,10 @@ public class StandardStructObjectInspect
       return fieldObjectInspector;
     }
 
+    public String getFieldComment() {
+      return fieldComment;
+    }
+
     @Override
     public String toString() {
       return "" + fieldID + ":" + fieldName;
@@ -83,17 +94,30 @@ public class StandardStructObjectInspect
    */
   protected StandardStructObjectInspector(List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors) {
-    init(structFieldNames, structFieldObjectInspectors);
+    init(structFieldNames, structFieldObjectInspectors, null);
+  }
+
+  /**
+  * Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+  */
+  protected StandardStructObjectInspector(List<String> structFieldNames,
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments) {
+    init(structFieldNames, structFieldObjectInspectors, structFieldComments);
   }
 
   protected void init(List<String> structFieldNames,
-      List<ObjectInspector> structFieldObjectInspectors) {
+      List<ObjectInspector> structFieldObjectInspectors,
+      List<String> structFieldComments) {
     assert (structFieldNames.size() == structFieldObjectInspectors.size());
+    assert (structFieldComments == null ||
+            (structFieldNames.size() == structFieldComments.size()));
 
     fields = new ArrayList<MyField>(structFieldNames.size());
     for (int i = 0; i < structFieldNames.size(); i++) {
       fields.add(new MyField(i, structFieldNames.get(i),
-          structFieldObjectInspectors.get(i)));
+          structFieldObjectInspectors.get(i),
+          structFieldComments == null ? null : structFieldComments.get(i)));
     }
   }
 

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
Fri Aug 12 19:26:29 2011
@@ -36,4 +36,8 @@ public interface StructField {
    */
   ObjectInspector getFieldObjectInspector();
 
+  /**
+   * Get the comment for the field. May be null if no comment provided.
+   */
+  String getFieldComment();
 }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
Fri Aug 12 19:26:29 2011
@@ -55,6 +55,10 @@ public class UnionStructObjectInspector 
     public ObjectInspector getFieldObjectInspector() {
       return structField.getFieldObjectInspector();
     }
+
+    public String getFieldComment() {
+      return structField.getFieldComment();
+    }
   }
 
   List<StructObjectInspector> unionObjectInspectors;

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java?rev=1157224&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
(added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
Fri Aug 12 19:26:29 2011
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestSerdeWithFieldComments extends TestCase {
+
+  private StructField mockedStructField(String name, String oiTypeName,
+                                        String comment) {
+    StructField m = mock(StructField.class);
+    when(m.getFieldName()).thenReturn(name);
+
+    ObjectInspector oi = mock(ObjectInspector.class);
+    when(oi.getTypeName()).thenReturn(oiTypeName);
+
+    when(m.getFieldObjectInspector()).thenReturn(oi);
+    when(m.getFieldComment()).thenReturn(comment);
+
+    return m;
+  }
+
+  public void testFieldComments() throws MetaException, SerDeException {
+    StructObjectInspector mockSOI = mock(StructObjectInspector.class);
+    when(mockSOI.getCategory()).thenReturn(ObjectInspector.Category.STRUCT);
+    List fieldRefs = new ArrayList<StructField>();
+    // Add field with a comment...
+    fieldRefs.add(mockedStructField("first", "type name 1", "this is a comment"));
+    // ... and one without
+    fieldRefs.add(mockedStructField("second", "type name 2", null));
+
+    when(mockSOI.getAllStructFieldRefs()).thenReturn(fieldRefs);
+
+    Deserializer mockDe = mock(Deserializer.class);
+    when(mockDe.getObjectInspector()).thenReturn(mockSOI);
+    List<FieldSchema> result =
+        MetaStoreUtils.getFieldsFromDeserializer("testTable", mockDe);
+
+    assertEquals(2, result.size());
+    assertEquals("first", result.get(0).getName());
+    assertEquals("this is a comment", result.get(0).getComment());
+    assertEquals("second", result.get(1).getName());
+    assertEquals("from deserializer", result.get(1).getComment());
+  }
+}

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java?rev=1157224&r1=1157223&r2=1157224&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
Fri Aug 12 19:26:29 2011
@@ -263,75 +263,9 @@ public class TestStandardObjectInspector
   @SuppressWarnings("unchecked")
   public void testStandardStructObjectInspector() throws Throwable {
     try {
-      ArrayList<String> fieldNames = new ArrayList<String>();
-      fieldNames.add("firstInteger");
-      fieldNames.add("secondString");
-      fieldNames.add("thirdBoolean");
-      ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
-      fieldObjectInspectors
-          .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
-      fieldObjectInspectors
-          .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
-      fieldObjectInspectors
-          .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
-
-      StandardStructObjectInspector soi1 = ObjectInspectorFactory
-          .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
-      StandardStructObjectInspector soi2 = ObjectInspectorFactory
-          .getStandardStructObjectInspector((ArrayList<String>) fieldNames
-          .clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors
-          .clone());
-      assertEquals(soi1, soi2);
-
-      // metadata
-      assertEquals(Category.STRUCT, soi1.getCategory());
-      List<? extends StructField> fields = soi1.getAllStructFieldRefs();
-      assertEquals(3, fields.size());
-      for (int i = 0; i < 3; i++) {
-        assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i)
-            .getFieldName());
-        assertEquals(fieldObjectInspectors.get(i), fields.get(i)
-            .getFieldObjectInspector());
-      }
-      assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
-      StringBuilder structTypeName = new StringBuilder();
-      structTypeName.append("struct<");
-      for (int i = 0; i < fields.size(); i++) {
-        if (i > 0) {
-          structTypeName.append(",");
-        }
-        structTypeName.append(fields.get(i).getFieldName());
-        structTypeName.append(":");
-        structTypeName.append(fields.get(i).getFieldObjectInspector()
-            .getTypeName());
-      }
-      structTypeName.append(">");
-      assertEquals(structTypeName.toString(), soi1.getTypeName());
-
-      // null
-      assertNull(soi1.getStructFieldData(null, fields.get(0)));
-      assertNull(soi1.getStructFieldData(null, fields.get(1)));
-      assertNull(soi1.getStructFieldData(null, fields.get(2)));
-      assertNull(soi1.getStructFieldsDataAsList(null));
-
-      // HashStruct
-      ArrayList<Object> struct = new ArrayList<Object>(3);
-      struct.add(1);
-      struct.add("two");
-      struct.add(true);
-
-      assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
-      assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
-      assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
-
-      // Settable
-      Object struct3 = soi1.create();
-      System.out.println(struct3);
-      soi1.setStructFieldData(struct3, fields.get(0), 1);
-      soi1.setStructFieldData(struct3, fields.get(1), "two");
-      soi1.setStructFieldData(struct3, fields.get(2), true);
-      assertEquals(struct, struct3);
-
+      // Test StandardObjectInspector both with field comments and without
+      doStandardObjectInspectorTest(true);
+      doStandardObjectInspectorTest(false);
     } catch (Throwable e) {
       e.printStackTrace();
       throw e;
@@ -339,6 +273,95 @@ public class TestStandardObjectInspector
 
   }
 
+  private void doStandardObjectInspectorTest(boolean testComments) {
+    ArrayList<String> fieldNames = new ArrayList<String>();
+    fieldNames.add("firstInteger");
+    fieldNames.add("secondString");
+    fieldNames.add("thirdBoolean");
+    ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
+    fieldObjectInspectors
+        .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+    fieldObjectInspectors
+        .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+    fieldObjectInspectors
+        .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
+    ArrayList<String> fieldComments = new ArrayList<String>(3);
+    if(testComments) {
+      fieldComments.add("firstInteger comment");
+      fieldComments.add("secondString comment");
+      fieldComments.add("thirdBoolean comment");
+    } else { // should have null for non-specified comments
+      for(int i = 0; i < 3; i++) fieldComments.add(null);
+    }
+
+    StandardStructObjectInspector soi1 = testComments ?
+        ObjectInspectorFactory
+        .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors,
+            fieldComments)
+      : ObjectInspectorFactory
+        .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
+    StandardStructObjectInspector soi2 = testComments ?
+        ObjectInspectorFactory
+        .getStandardStructObjectInspector((ArrayList<String>) fieldNames
+        .clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors
+        .clone(), (ArrayList<String>)fieldComments.clone())
+        : ObjectInspectorFactory
+        .getStandardStructObjectInspector((ArrayList<String>) fieldNames
+        .clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors
+        .clone());
+    assertEquals(soi1, soi2);
+
+    // metadata
+    assertEquals(Category.STRUCT, soi1.getCategory());
+    List<? extends StructField> fields = soi1.getAllStructFieldRefs();
+    assertEquals(3, fields.size());
+    for (int i = 0; i < 3; i++) {
+      assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i)
+          .getFieldName());
+      assertEquals(fieldObjectInspectors.get(i), fields.get(i)
+          .getFieldObjectInspector());
+      assertEquals(fieldComments.get(i), fields.get(i).getFieldComment());
+    }
+    assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
+    StringBuilder structTypeName = new StringBuilder();
+    structTypeName.append("struct<");
+    for (int i = 0; i < fields.size(); i++) {
+      if (i > 0) {
+        structTypeName.append(",");
+      }
+      structTypeName.append(fields.get(i).getFieldName());
+      structTypeName.append(":");
+      structTypeName.append(fields.get(i).getFieldObjectInspector()
+          .getTypeName());
+    }
+    structTypeName.append(">");
+    assertEquals(structTypeName.toString(), soi1.getTypeName());
+
+    // null
+    assertNull(soi1.getStructFieldData(null, fields.get(0)));
+    assertNull(soi1.getStructFieldData(null, fields.get(1)));
+    assertNull(soi1.getStructFieldData(null, fields.get(2)));
+    assertNull(soi1.getStructFieldsDataAsList(null));
+
+    // HashStruct
+    ArrayList<Object> struct = new ArrayList<Object>(3);
+    struct.add(1);
+    struct.add("two");
+    struct.add(true);
+
+    assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
+    assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
+    assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
+
+    // Settable
+    Object struct3 = soi1.create();
+    System.out.println(struct3);
+    soi1.setStructFieldData(struct3, fields.get(0), 1);
+    soi1.setStructFieldData(struct3, fields.get(1), "two");
+    soi1.setStructFieldData(struct3, fields.get(2), true);
+    assertEquals(struct, struct3);
+  }
+
   @SuppressWarnings("unchecked")
   public void testStandardUnionObjectInspector() throws Throwable {
     try {



Mime
View raw message