hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmccl...@apache.org
Subject hive git commit: HIVE-16730: Vectorization: Schema Evolution for Text Vectorization / Complex Types
Date Wed, 12 Jul 2017 06:18:02 GMT
Repository: hive
Updated Branches:
  refs/heads/master 607077a90 -> 26d6de7e2


HIVE-16730: Vectorization: Schema Evolution for Text Vectorization / Complex Types


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/26d6de7e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/26d6de7e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/26d6de7e

Branch: refs/heads/master
Commit: 26d6de7e2108149c1be411ce2c1b63e4fae230ad
Parents: 607077a
Author: Teddy Choi <tchoi@hortonworks.com>
Authored: Wed Jul 12 01:17:46 2017 -0500
Committer: Matt McCline <mmccline@hortonworks.com>
Committed: Wed Jul 12 01:17:46 2017 -0500

----------------------------------------------------------------------
 .../hive/ql/exec/vector/VectorAssignRow.java    | 277 +++++++----
 .../ql/exec/vector/VectorDeserializeRow.java    | 492 +++++++++++++------
 .../hive/ql/exec/vector/VectorExtractRow.java   |  40 +-
 .../schema_evol_text_vec_part_all_complex.q     |   3 -
 .../schema_evol_text_vecrow_part_all_complex.q  |   3 -
 .../schema_evol_text_vec_part_all_complex.q.out |  72 ++-
 ...hema_evol_text_vecrow_part_all_complex.q.out |  72 ++-
 .../lazy/fast/LazySimpleDeserializeRead.java    |  14 +-
 .../PrimitiveObjectInspectorUtils.java          |   2 +-
 9 files changed, 706 insertions(+), 269 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
index c7aa93e..f02a300 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
@@ -23,7 +23,10 @@ import java.sql.Timestamp;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
@@ -53,7 +56,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -106,7 +108,7 @@ public class VectorAssignRow {
    * These members have information for data type conversion.
    * Not defined if there is no conversion.
    */
-  PrimitiveObjectInspector[] convertSourcePrimitiveObjectInspectors;
+  ObjectInspector[] convertSourceOI;
                 // The primitive object inspector of the source data type for any column being
                 // converted.  Otherwise, null.
 
@@ -128,7 +130,7 @@ public class VectorAssignRow {
    * Allocate the source conversion related arrays (optional).
    */
   private void allocateConvertArrays(int count) {
-    convertSourcePrimitiveObjectInspectors = new PrimitiveObjectInspector[count];
+    convertSourceOI = new ObjectInspector[count];
     convertTargetWritables = new Writable[count];
   }
 
@@ -163,12 +165,10 @@ public class VectorAssignRow {
   private void initConvertSourceEntry(int logicalColumnIndex, TypeInfo convertSourceTypeInfo) {
     isConvert[logicalColumnIndex] = true;
     final Category convertSourceCategory = convertSourceTypeInfo.getCategory();
-    if (convertSourceCategory == Category.PRIMITIVE) {
-      final PrimitiveTypeInfo convertSourcePrimitiveTypeInfo = (PrimitiveTypeInfo) convertSourceTypeInfo;
-      convertSourcePrimitiveObjectInspectors[logicalColumnIndex] =
-        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
-            convertSourcePrimitiveTypeInfo);
+    convertSourceOI[logicalColumnIndex] =
+        TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(convertSourceTypeInfo);
 
+    if (convertSourceCategory == Category.PRIMITIVE) {
       // These need to be based on the target.
       final PrimitiveCategory targetPrimitiveCategory =
           ((PrimitiveTypeInfo) targetTypeInfos[logicalColumnIndex]).getPrimitiveCategory();
@@ -298,36 +298,29 @@ public class VectorAssignRow {
 
       } else {
         final TypeInfo targetTypeInfo = targetTypeInfos[i];
+        final TypeInfo sourceTypeInfo = sourceTypeInfos[i];
 
-        if (targetTypeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
+        if (!sourceTypeInfo.equals(targetTypeInfo)) {
 
-          // For now, we don't have an assigner for complex types...
-
-        } else {
-          final TypeInfo sourceTypeInfo = sourceTypeInfos[i];
+          if (VectorPartitionConversion.isImplicitVectorColumnConversion(
+              sourceTypeInfo, targetTypeInfo)) {
 
-          if (!sourceTypeInfo.equals(targetTypeInfo)) {
+            // Do implicit conversion accepting the source type and putting it in the same
+            // target type ColumnVector type.
+            initTargetEntry(i, i, sourceTypeInfo);
 
-            if (VectorPartitionConversion.isImplicitVectorColumnConversion(
-                sourceTypeInfo, targetTypeInfo)) {
-
-              // Do implicit conversion accepting the source type and putting it in the same
-              // target type ColumnVector type.
-              initTargetEntry(i, i, sourceTypeInfo);
-
-            } else {
-
-              // Do formal conversion...
-              initTargetEntry(i, i, targetTypeInfo);
-              initConvertSourceEntry(i, sourceTypeInfo);
-
-            }
           } else {
 
-            // No conversion.
+            // Do formal conversion...
             initTargetEntry(i, i, targetTypeInfo);
+            initConvertSourceEntry(i, sourceTypeInfo);
 
           }
+        } else {
+
+          // No conversion.
+          initTargetEntry(i, i, targetTypeInfo);
+
         }
       }
     }
@@ -594,18 +587,18 @@ public class VectorAssignRow {
     case STRUCT:
       {
         final StructColumnVector structColumnVector = (StructColumnVector) columnVector;
-        final StructTypeInfo structTypeInfo = (StructTypeInfo) targetTypeInfo;
-        final List<TypeInfo> fieldStructTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
-        final int size = fieldStructTypeInfos.size();
+        final StructTypeInfo targetStructTypeInfo = (StructTypeInfo) targetTypeInfo;
+        final List<TypeInfo> targetFieldTypeInfos = targetStructTypeInfo.getAllStructFieldTypeInfos();
+        final int size = targetFieldTypeInfos.size();
         if (object instanceof List) {
           final List struct = (List) object;
           for (int i = 0; i < size; i++) {
-            assignRowColumn(structColumnVector.fields[i], batchIndex, fieldStructTypeInfos.get(i), struct.get(i));
+            assignRowColumn(structColumnVector.fields[i], batchIndex, targetFieldTypeInfos.get(i), struct.get(i));
           }
         } else {
           final Object[] array = (Object[]) object;
           for (int i = 0; i < size; i++) {
-            assignRowColumn(structColumnVector.fields[i], batchIndex, fieldStructTypeInfos.get(i), array[i]);
+            assignRowColumn(structColumnVector.fields[i], batchIndex, targetFieldTypeInfos.get(i), array[i]);
           }
         }
       }
@@ -647,113 +640,134 @@ public class VectorAssignRow {
   public void assignConvertRowColumn(VectorizedRowBatch batch, int batchIndex,
       int logicalColumnIndex, Object object) {
     Preconditions.checkState(isConvert[logicalColumnIndex]);
-    final Category targetCategory = targetTypeInfos[logicalColumnIndex].getCategory();
+    final int projectionColumnNum = projectionColumnNums[logicalColumnIndex];
+    assignConvertRowColumn(
+        batch.cols[projectionColumnNum],
+        batchIndex,
+        targetTypeInfos[logicalColumnIndex],
+        convertSourceOI[logicalColumnIndex],
+        convertTargetWritables[logicalColumnIndex],
+        object);
+  }
+
+  private void assignConvertRowColumn(ColumnVector columnVector, int batchIndex,
+      TypeInfo targetTypeInfo, ObjectInspector sourceObjectInspector,
+      Writable convertTargetWritable, Object object) {
+
+    final Category targetCategory = targetTypeInfo.getCategory();
     if (targetCategory == null) {
       /*
        * This is a column that we don't want (i.e. not included) -- we are done.
        */
       return;
     }
-    final int projectionColumnNum = projectionColumnNums[logicalColumnIndex];
     if (object == null) {
-      VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+      VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
       return;
     }
     try {
       switch (targetCategory) {
       case PRIMITIVE:
+        final PrimitiveObjectInspector sourcePrimitiveOI =
+            (PrimitiveObjectInspector) sourceObjectInspector;
         final PrimitiveCategory targetPrimitiveCategory =
-            ((PrimitiveTypeInfo) targetTypeInfos[logicalColumnIndex]).getPrimitiveCategory();
+            ((PrimitiveTypeInfo) targetTypeInfo).getPrimitiveCategory();
         switch (targetPrimitiveCategory) {
         case VOID:
-          VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+          VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
           return;
         case BOOLEAN:
-          ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((LongColumnVector) columnVector).vector[batchIndex] =
               (PrimitiveObjectInspectorUtils.getBoolean(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]) ? 1 : 0);
+                  object, sourcePrimitiveOI) ? 1 : 0);
           break;
         case BYTE:
-          ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((LongColumnVector) columnVector).vector[batchIndex] =
               PrimitiveObjectInspectorUtils.getByte(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
           break;
         case SHORT:
-          ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((LongColumnVector) columnVector).vector[batchIndex] =
               PrimitiveObjectInspectorUtils.getShort(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
           break;
         case INT:
-          ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((LongColumnVector) columnVector).vector[batchIndex] =
               PrimitiveObjectInspectorUtils.getInt(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
           break;
         case LONG:
-          ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((LongColumnVector) columnVector).vector[batchIndex] =
               PrimitiveObjectInspectorUtils.getLong(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
           break;
         case TIMESTAMP:
           {
             final Timestamp timestamp =
               PrimitiveObjectInspectorUtils.getTimestamp(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
             if (timestamp == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            ((TimestampColumnVector) batch.cols[projectionColumnNum]).set(
+            ((TimestampColumnVector) columnVector).set(
                 batchIndex, timestamp);
           }
           break;
         case DATE:
           {
             final Date date = PrimitiveObjectInspectorUtils.getDate(
-                object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                object, sourcePrimitiveOI);
             if (date == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            final DateWritable dateWritable = (DateWritable) convertTargetWritables[logicalColumnIndex];
+            DateWritable dateWritable = (DateWritable) convertTargetWritable;
+            if (dateWritable == null) {
+              dateWritable = new DateWritable();
+            }
             dateWritable.set(date);
-            ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+            ((LongColumnVector) columnVector).vector[batchIndex] =
                 dateWritable.getDays();
           }
           break;
         case FLOAT:
-          ((DoubleColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((DoubleColumnVector) columnVector).vector[batchIndex] =
               PrimitiveObjectInspectorUtils.getFloat(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
           break;
         case DOUBLE:
-          ((DoubleColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+          ((DoubleColumnVector) columnVector).vector[batchIndex] =
               PrimitiveObjectInspectorUtils.getDouble(
-                  object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                  object, sourcePrimitiveOI);
           break;
         case BINARY:
           {
             final BytesWritable bytesWritable =
                 PrimitiveObjectInspectorUtils.getBinary(
-                    object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                    object, sourcePrimitiveOI);
             if (bytesWritable == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(
+            ((BytesColumnVector) columnVector).setVal(
                 batchIndex, bytesWritable.getBytes(), 0, bytesWritable.getLength());
           }
           break;
         case STRING:
           {
             final String string = PrimitiveObjectInspectorUtils.getString(
-                object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                object, sourcePrimitiveOI);
             if (string == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            Text text = (Text) convertTargetWritables[logicalColumnIndex];
+            Text text = (Text) convertTargetWritable;
+            if (text == null) {
+              text = new Text();
+            }
             text.set(string);
-            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(
+            ((BytesColumnVector) columnVector).setVal(
                 batchIndex, text.getBytes(), 0, text.getLength());
           }
           break;
@@ -763,16 +777,16 @@ public class VectorAssignRow {
 
             final HiveVarchar hiveVarchar =
                 PrimitiveObjectInspectorUtils.getHiveVarchar(
-                    object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                    object, sourcePrimitiveOI);
             if (hiveVarchar == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
 
             // TODO: Do we need maxLength checking?
 
             byte[] bytes = hiveVarchar.getValue().getBytes();
-            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(
+            ((BytesColumnVector) columnVector).setVal(
                 batchIndex, bytes, 0, bytes.length);
           }
           break;
@@ -782,9 +796,9 @@ public class VectorAssignRow {
 
             final HiveChar hiveChar =
                 PrimitiveObjectInspectorUtils.getHiveChar(
-                    object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                    object, sourcePrimitiveOI);
             if (hiveChar == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
             // We store CHAR in vector row batch with padding stripped.
@@ -792,7 +806,7 @@ public class VectorAssignRow {
             // TODO: Do we need maxLength checking?
 
             final byte[] bytes = hiveChar.getStrippedValue().getBytes();
-            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(
+            ((BytesColumnVector) columnVector).setVal(
                 batchIndex, bytes, 0, bytes.length);
           }
           break;
@@ -800,12 +814,12 @@ public class VectorAssignRow {
           {
             final HiveDecimal hiveDecimal =
                 PrimitiveObjectInspectorUtils.getHiveDecimal(
-                    object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                    object, sourcePrimitiveOI);
             if (hiveDecimal == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            ((DecimalColumnVector) batch.cols[projectionColumnNum]).set(
+            ((DecimalColumnVector) columnVector).set(
                 batchIndex, hiveDecimal);
           }
           break;
@@ -813,12 +827,12 @@ public class VectorAssignRow {
           {
             final HiveIntervalYearMonth intervalYearMonth =
                 PrimitiveObjectInspectorUtils.getHiveIntervalYearMonth(
-                    object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                    object, sourcePrimitiveOI);
             if (intervalYearMonth == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =
+            ((LongColumnVector) columnVector).vector[batchIndex] =
                 intervalYearMonth.getTotalMonths();
           }
           break;
@@ -826,12 +840,12 @@ public class VectorAssignRow {
           {
             final HiveIntervalDayTime intervalDayTime =
                 PrimitiveObjectInspectorUtils.getHiveIntervalDayTime(
-                    object, convertSourcePrimitiveObjectInspectors[logicalColumnIndex]);
+                    object, sourcePrimitiveOI);
             if (intervalDayTime == null) {
-              VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+              VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
               return;
             }
-            ((IntervalDayTimeColumnVector) batch.cols[projectionColumnNum]).set(
+            ((IntervalDayTimeColumnVector) columnVector).set(
                 batchIndex, intervalDayTime);
           }
           break;
@@ -840,18 +854,113 @@ public class VectorAssignRow {
               " not supported");
         }
         break;
+      case LIST:
+        {
+          final ListColumnVector listColumnVector = (ListColumnVector) columnVector;
+          final ListObjectInspector sourceListOI = (ListObjectInspector) sourceObjectInspector;
+          final ObjectInspector sourceElementOI = sourceListOI.getListElementObjectInspector();
+          final int size = sourceListOI.getListLength(object);
+          final TypeInfo targetElementTypeInfo = ((ListTypeInfo) targetTypeInfo).getListElementTypeInfo();
+
+          listColumnVector.offsets[batchIndex] = listColumnVector.childCount;
+          listColumnVector.childCount += size;
+          listColumnVector.ensureSize(listColumnVector.childCount, true);
+          listColumnVector.lengths[batchIndex] = size;
+
+          for (int i = 0; i < size; i++) {
+            final Object element = sourceListOI.getListElement(object, i);
+            final int offset = (int) (listColumnVector.offsets[batchIndex] + i);
+            assignConvertRowColumn(
+                listColumnVector.child,
+                offset,
+                targetElementTypeInfo,
+                sourceElementOI,
+                null,
+                element);
+          }
+        }
+        break;
+      case MAP:
+        {
+          final MapColumnVector mapColumnVector = (MapColumnVector) columnVector;
+          final MapObjectInspector mapObjectInspector = (MapObjectInspector) sourceObjectInspector;
+          final MapTypeInfo mapTypeInfo = (MapTypeInfo) targetTypeInfo;
+
+          final Map<?, ?> map = mapObjectInspector.getMap(object);
+          for (Map.Entry<?, ?> entry : map.entrySet()) {
+            assignConvertRowColumn(
+                mapColumnVector.keys,
+                batchIndex,
+                mapTypeInfo.getMapKeyTypeInfo(),
+                mapObjectInspector.getMapKeyObjectInspector(),
+                null,
+                entry.getKey());
+            assignConvertRowColumn(
+                mapColumnVector.values,
+                batchIndex,
+                mapTypeInfo.getMapValueTypeInfo(),
+                mapObjectInspector.getMapValueObjectInspector(),
+                null,
+                entry.getValue());
+          }
+        }
+        break;
+      case STRUCT:
+        {
+          final StructColumnVector structColumnVector = (StructColumnVector) columnVector;
+          final StructObjectInspector sourceStructOI = (StructObjectInspector) sourceObjectInspector;
+          final List<? extends StructField> sourceFields = sourceStructOI.getAllStructFieldRefs();
+          final StructTypeInfo targetStructTypeInfo = (StructTypeInfo) targetTypeInfo;
+          final List<TypeInfo> targetTypeInfos = targetStructTypeInfo.getAllStructFieldTypeInfos();
+          final int size = targetTypeInfos.size();
+
+          for (int i = 0; i < size; i++) {
+            if (i < sourceFields.size()) {
+              final StructField sourceStructField = sourceFields.get(i);
+              final ObjectInspector sourceFieldOI = sourceStructField.getFieldObjectInspector();
+              final Object sourceData = sourceStructOI.getStructFieldData(object, sourceStructField);
+              assignConvertRowColumn(
+                  structColumnVector.fields[i],
+                  batchIndex,
+                  targetTypeInfos.get(i),
+                  sourceFieldOI,
+                  null,
+                  sourceData);
+            } else {
+              final ColumnVector fieldColumnVector = structColumnVector.fields[i];
+              VectorizedBatchUtil.setNullColIsNullValue(fieldColumnVector, batchIndex);
+            }
+          }
+        }
+        break;
+      case UNION:
+        {
+          final UnionColumnVector unionColumnVector = (UnionColumnVector) columnVector;
+          final UnionObjectInspector unionObjectInspector = (UnionObjectInspector) sourceObjectInspector;
+          final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) targetTypeInfo;
+          final int tag = unionObjectInspector.getTag(object);
+
+          assignConvertRowColumn(
+              unionColumnVector.fields[tag],
+              batchIndex,
+              unionTypeInfo.getAllUnionObjectTypeInfos().get(tag),
+              unionObjectInspector.getObjectInspectors().get(tag),
+              null,
+              unionObjectInspector.getField(tag));
+        }
+        break;
       default:
         throw new RuntimeException("Category " + targetCategory.name() + " not supported");
       }
     } catch (NumberFormatException e) {
 
       // Some of the conversion methods throw this exception on numeric parsing errors.
-      VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
+      VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
       return;
     }
 
     // We always set the null flag to false when there is a value.
-    batch.cols[projectionColumnNum].isNull[batchIndex] = false;
+    columnVector.isNull[batchIndex] = false;
   }
 
   /*

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index e37816f..2ad06fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -40,7 +40,16 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableUnionObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@@ -55,7 +64,6 @@ import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
@@ -112,7 +120,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
      * This member has information for data type conversion.
      * Not defined if there is no conversion.
      */
-    Writable conversionWritable;
+    private Object conversionWritable;
                   // Conversion requires source be placed in writable so we can call upon
                   // VectorAssignRow to convert and assign the row column.
 
@@ -120,6 +128,8 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
                   // For a complex type, a helper object that describes elements, key/value pairs,
                   // or fields.
 
+    private ObjectInspector objectInspector;
+
     public Field(PrimitiveCategory primitiveCategory, int maxLength) {
       this.category = Category.PRIMITIVE;
       this.primitiveCategory = primitiveCategory;
@@ -127,10 +137,13 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
       this.isConvert = false;
       this.conversionWritable = null;
       this.complexTypeHelper = null;
+      this.objectInspector = PrimitiveObjectInspectorFactory.
+          getPrimitiveWritableObjectInspector(primitiveCategory);
     }
 
-    public Field(Category category, ComplexTypeHelper complexTypeHelper) {
+    public Field(Category category, ComplexTypeHelper complexTypeHelper, TypeInfo typeInfo) {
       this.category = category;
+      this.objectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
       this.primitiveCategory = null;
       this.maxLength = 0;
       this.isConvert = false;
@@ -158,17 +171,21 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
       return isConvert;
     }
 
-    public void setConversionWritable(Writable conversionWritable) {
+    public void setConversionWritable(Object conversionWritable) {
       this.conversionWritable = conversionWritable;
     }
 
-    public Writable getConversionWritable() {
+    public Object getConversionWritable() {
       return conversionWritable;
     }
 
     public ComplexTypeHelper getComplexHelper() {
       return complexTypeHelper;
     }
+
+    public ObjectInspector getObjectInspector() {
+      return objectInspector;
+    }
   }
 
   /*
@@ -191,7 +208,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
 
   private Field topLevelFields[];
 
-  VectorAssignRow convertVectorAssignRow;
+  private VectorAssignRow convertVectorAssignRow;
                 // Use its conversion ability.
 
   /*
@@ -231,7 +248,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
         final ListComplexTypeHelper listHelper =
             new ListComplexTypeHelper(
                 allocateField(listTypeInfo.getListElementTypeInfo()));
-        return new Field(category, listHelper);
+        return new Field(category, listHelper, sourceTypeInfo);
       }
     case MAP:
       {
@@ -240,7 +257,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
             new MapComplexTypeHelper(
                 allocateField(mapTypeInfo.getMapKeyTypeInfo()),
                 allocateField(mapTypeInfo.getMapValueTypeInfo()));
-        return new Field(category, mapHelper);
+        return new Field(category, mapHelper, sourceTypeInfo);
       }
     case STRUCT:
       {
@@ -253,7 +270,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
         }
         final StructComplexTypeHelper structHelper =
             new StructComplexTypeHelper(fields);
-        return new Field(category, structHelper);
+        return new Field(category, structHelper, sourceTypeInfo);
       }
     case UNION:
       {
@@ -266,7 +283,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
         }
         final UnionComplexTypeHelper unionHelper =
             new UnionComplexTypeHelper(fields);
-        return new Field(category, unionHelper);
+        return new Field(category, unionHelper, sourceTypeInfo);
       }
     default:
       throw new RuntimeException("Category " + category + " not supported");
@@ -711,8 +728,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
       Field field, int batchIndex, boolean canRetainByteRef) throws IOException {
 
     if (!deserializeRead.readComplexField()) {
-      fieldColVector.isNull[batchIndex] = true;
-      fieldColVector.noNulls = false;
+      VectorizedBatchUtil.setNullColIsNullValue(fieldColVector, batchIndex);
       return;
     }
 
@@ -735,6 +751,8 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
     default:
       throw new RuntimeException("Category " + field.getCategory() + " not supported");
     }
+
+    fieldColVector.isNull[batchIndex] = false;
   }
 
   private void storeListRowColumn(ColumnVector colVector,
@@ -867,7 +885,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
       Field field, int logicalColumnIndex, boolean canRetainByteRef) throws IOException {
 
     final int projectionColumnNum = projectionColumnNums[logicalColumnIndex];
-    ColumnVector colVector = batch.cols[projectionColumnNum];
+    final ColumnVector colVector = batch.cols[projectionColumnNum];
 
     switch (field.getCategory()) {
     case PRIMITIVE:
@@ -890,7 +908,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
     }
 
     // We always set the null flag to false when there is a value.
-    batch.cols[projectionColumnNum].isNull[batchIndex] = false;
+    colVector.isNull[batchIndex] = false;
   }
 
   /**
@@ -907,156 +925,336 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
   private void convertRowColumn(VectorizedRowBatch batch, int batchIndex,
       Field field, int logicalColumnIndex) throws IOException {
 
-    Writable convertSourceWritable = field.getConversionWritable();
+    final int projectionColumnIndex = projectionColumnNums[logicalColumnIndex];
+    final ColumnVector colVector = batch.cols[projectionColumnIndex];
+
+    final Object convertSourceWritable;
+    switch (field.getCategory()) {
+    case PRIMITIVE:
+      convertSourceWritable = convertPrimitiveRowColumn(batchIndex, field);
+      break;
+    case LIST:
+      convertSourceWritable = convertListRowColumn(colVector, batchIndex, field);
+      break;
+    case MAP:
+      convertSourceWritable = convertMapRowColumn(colVector, batchIndex, field);
+      break;
+    case STRUCT:
+      convertSourceWritable = convertStructRowColumn(colVector, batchIndex, field);
+      break;
+    case UNION:
+      convertSourceWritable = convertUnionRowColumn(colVector, batchIndex, field);
+      break;
+    default:
+      throw new RuntimeException();
+    }
+
+    /*
+     * Convert our source object we just read into the target object and store that in the
+     * VectorizedRowBatch.
+     */
+    convertVectorAssignRow.assignConvertRowColumn(
+        batch, batchIndex, logicalColumnIndex, convertSourceWritable);
+  }
+
+  private Object convertComplexFieldRowColumn(ColumnVector colVector, int batchIndex,
+      Field field) throws IOException {
+
+    if (!deserializeRead.readComplexField()) {
+      VectorizedBatchUtil.setNullColIsNullValue(colVector, batchIndex);
+      return null;
+    }
+
+    colVector.isNull[batchIndex] = false;
+
     switch (field.getCategory()) {
     case PRIMITIVE:
+      return convertPrimitiveRowColumn(batchIndex, field);
+    case LIST:
+      return convertListRowColumn(colVector, batchIndex, field);
+    case MAP:
+      return convertMapRowColumn(colVector, batchIndex, field);
+    case STRUCT:
+      return convertStructRowColumn(colVector, batchIndex, field);
+    case UNION:
+      return convertUnionRowColumn(colVector, batchIndex, field);
+    default:
+      throw new RuntimeException();
+    }
+  }
+
+  private Object convertPrimitiveRowColumn(int batchIndex, Field field) throws IOException {
+
+    Object writable = field.getConversionWritable();
+    switch (field.getPrimitiveCategory()) {
+    case VOID:
+      writable = null;
+      break;
+    case BOOLEAN:
       {
-        switch (field.getPrimitiveCategory()) {
-        case VOID:
-          convertSourceWritable = null;
-          break;
-        case BOOLEAN:
-          ((BooleanWritable) convertSourceWritable).set(deserializeRead.currentBoolean);
-          break;
-        case BYTE:
-          ((ByteWritable) convertSourceWritable).set(deserializeRead.currentByte);
-          break;
-        case SHORT:
-          ((ShortWritable) convertSourceWritable).set(deserializeRead.currentShort);
-          break;
-        case INT:
-          ((IntWritable) convertSourceWritable).set(deserializeRead.currentInt);
-          break;
-        case LONG:
-          ((LongWritable) convertSourceWritable).set(deserializeRead.currentLong);
-          break;
-        case TIMESTAMP:
-          ((TimestampWritable) convertSourceWritable).set(deserializeRead.currentTimestampWritable);
-          break;
-        case DATE:
-          ((DateWritable) convertSourceWritable).set(deserializeRead.currentDateWritable);
-          break;
-        case FLOAT:
-          ((FloatWritable) convertSourceWritable).set(deserializeRead.currentFloat);
-          break;
-        case DOUBLE:
-          ((DoubleWritable) convertSourceWritable).set(deserializeRead.currentDouble);
-          break;
-        case BINARY:
-          if (deserializeRead.currentBytes == null) {
-            LOG.info(
-                "null binary entry: batchIndex " + batchIndex + " projection column num " +
-                projectionColumnNums[logicalColumnIndex]);
-          }
+        if (writable == null) {
+          writable = new BooleanWritable();
+        }
+        ((BooleanWritable) writable).set(deserializeRead.currentBoolean);
+      }
+      break;
+    case BYTE:
+      {
+        if (writable == null) {
+          writable = new ByteWritable();
+        }
+        ((ByteWritable) writable).set(deserializeRead.currentByte);
+      }
+      break;
+    case SHORT:
+      {
+        if (writable == null) {
+          writable = new ShortWritable();
+        }
+        ((ShortWritable) writable).set(deserializeRead.currentShort);
+      }
+      break;
+    case INT:
+      {
+        if (writable == null) {
+          writable = new IntWritable();
+        }
+        ((IntWritable) writable).set(deserializeRead.currentInt);
+      }
+      break;
+    case LONG:
+      {
+        if (writable == null) {
+          writable = new LongWritable();
+        }
+        ((LongWritable) writable).set(deserializeRead.currentLong);
+      }
+      break;
+    case TIMESTAMP:
+      {
+        if (writable == null) {
+          writable = new TimestampWritable();
+        }
+        ((TimestampWritable) writable).set(deserializeRead.currentTimestampWritable);
+      }
+      break;
+    case DATE:
+      {
+        if (writable == null) {
+          writable = new DateWritable();
+        }
+        ((DateWritable) writable).set(deserializeRead.currentDateWritable);
+      }
+      break;
+    case FLOAT:
+      {
+        if (writable == null) {
+          writable = new FloatWritable();
+        }
+        ((FloatWritable) writable).set(deserializeRead.currentFloat);
+      }
+      break;
+    case DOUBLE:
+      {
+        if (writable == null) {
+          writable = new DoubleWritable();
+        }
+        ((DoubleWritable) writable).set(deserializeRead.currentDouble);
+      }
+      break;
+    case BINARY:
+      {
+        if (writable == null) {
+          writable = new BytesWritable();
+        }
+        if (deserializeRead.currentBytes == null) {
+          LOG.info(
+              "null binary entry: batchIndex " + batchIndex);
+        }
+
+        ((BytesWritable) writable).set(
+            deserializeRead.currentBytes,
+            deserializeRead.currentBytesStart,
+            deserializeRead.currentBytesLength);
+        break;
+      }
+    case STRING:
+      {
+        if (writable == null) {
+          writable = new Text();
+        }
+        if (deserializeRead.currentBytes == null) {
+          throw new RuntimeException(
+              "null string entry: batchIndex " + batchIndex);
+        }
 
-          ((BytesWritable) convertSourceWritable).set(
+        // Use org.apache.hadoop.io.Text as our helper to go from byte[] to String.
+        ((Text) writable).set(
+            deserializeRead.currentBytes,
+            deserializeRead.currentBytesStart,
+            deserializeRead.currentBytesLength);
+      }
+      break;
+    case VARCHAR:
+      {
+        if (writable == null) {
+          writable = new HiveVarcharWritable();
+        }
+        // Use the basic STRING bytes read to get access, then use our optimal truncate/trim method
+        // that does not use Java String objects.
+        if (deserializeRead.currentBytes == null) {
+          throw new RuntimeException(
+              "null varchar entry: batchIndex " + batchIndex);
+        }
+
+        int adjustedLength = StringExpr.truncate(
+            deserializeRead.currentBytes,
+            deserializeRead.currentBytesStart,
+            deserializeRead.currentBytesLength,
+            field.getMaxLength());
+
+        ((HiveVarcharWritable) writable).set(
+            new String(
               deserializeRead.currentBytes,
               deserializeRead.currentBytesStart,
-              deserializeRead.currentBytesLength);
-          break;
-        case STRING:
-          if (deserializeRead.currentBytes == null) {
-            throw new RuntimeException(
-                "null string entry: batchIndex " + batchIndex + " projection column num " +
-                projectionColumnNums[logicalColumnIndex]);
-          }
+              adjustedLength,
+              Charsets.UTF_8),
+            -1);
+      }
+      break;
+    case CHAR:
+      {
+        if (writable == null) {
+          writable = new HiveCharWritable();
+        }
+        // Use the basic STRING bytes read to get access, then use our optimal truncate/trim method
+        // that does not use Java String objects.
+        if (deserializeRead.currentBytes == null) {
+          throw new RuntimeException(
+              "null char entry: batchIndex " + batchIndex);
+        }
+
+        int adjustedLength = StringExpr.rightTrimAndTruncate(
+            deserializeRead.currentBytes,
+            deserializeRead.currentBytesStart,
+            deserializeRead.currentBytesLength,
+            field.getMaxLength());
 
-          // Use org.apache.hadoop.io.Text as our helper to go from byte[] to String.
-          ((Text) convertSourceWritable).set(
+        ((HiveCharWritable) writable).set(
+            new String(
               deserializeRead.currentBytes,
               deserializeRead.currentBytesStart,
-              deserializeRead.currentBytesLength);
-          break;
-        case VARCHAR:
-          {
-            // Use the basic STRING bytes read to get access, then use our optimal truncate/trim method
-            // that does not use Java String objects.
-            if (deserializeRead.currentBytes == null) {
-              throw new RuntimeException(
-                  "null varchar entry: batchIndex " + batchIndex + " projection column num " +
-                  projectionColumnNums[logicalColumnIndex]);
-            }
-
-            int adjustedLength = StringExpr.truncate(
-                deserializeRead.currentBytes,
-                deserializeRead.currentBytesStart,
-                deserializeRead.currentBytesLength,
-                field.getMaxLength());
-
-            ((HiveVarcharWritable) convertSourceWritable).set(
-                new String(
-                  deserializeRead.currentBytes,
-                  deserializeRead.currentBytesStart,
-                  adjustedLength,
-                  Charsets.UTF_8),
-                -1);
-          }
-          break;
-        case CHAR:
-          {
-            // Use the basic STRING bytes read to get access, then use our optimal truncate/trim method
-            // that does not use Java String objects.
-            if (deserializeRead.currentBytes == null) {
-              throw new RuntimeException(
-                  "null char entry: batchIndex " + batchIndex + " projection column num " +
-                  projectionColumnNums[logicalColumnIndex]);
-            }
-
-            int adjustedLength = StringExpr.rightTrimAndTruncate(
-                deserializeRead.currentBytes,
-                deserializeRead.currentBytesStart,
-                deserializeRead.currentBytesLength,
-                field.getMaxLength());
-
-            ((HiveCharWritable) convertSourceWritable).set(
-                new String(
-                  deserializeRead.currentBytes,
-                  deserializeRead.currentBytesStart,
-                  adjustedLength, Charsets.UTF_8),
-                -1);
-          }
-          break;
-        case DECIMAL:
-          ((HiveDecimalWritable) convertSourceWritable).set(
-              deserializeRead.currentHiveDecimalWritable);
-          break;
-        case INTERVAL_YEAR_MONTH:
-          ((HiveIntervalYearMonthWritable) convertSourceWritable).set(
-              deserializeRead.currentHiveIntervalYearMonthWritable);
-          break;
-        case INTERVAL_DAY_TIME:
-          ((HiveIntervalDayTimeWritable) convertSourceWritable).set(
-              deserializeRead.currentHiveIntervalDayTimeWritable);
-          break;
-        default:
-          throw new RuntimeException("Primitive category " + field.getPrimitiveCategory() +
-              " not supported");
+              adjustedLength, Charsets.UTF_8),
+            -1);
+      }
+      break;
+    case DECIMAL:
+      {
+        if (writable == null) {
+          writable = new HiveDecimalWritable();
+        }
+        ((HiveDecimalWritable) writable).set(
+            deserializeRead.currentHiveDecimalWritable);
+      }
+      break;
+    case INTERVAL_YEAR_MONTH:
+      {
+        if (writable == null) {
+          writable = new HiveIntervalYearMonthWritable();
+        }
+        ((HiveIntervalYearMonthWritable) writable).set(
+            deserializeRead.currentHiveIntervalYearMonthWritable);
+      }
+      break;
+    case INTERVAL_DAY_TIME:
+      {
+        if (writable == null) {
+          writable = new HiveIntervalDayTimeWritable();
         }
+        ((HiveIntervalDayTimeWritable) writable).set(
+            deserializeRead.currentHiveIntervalDayTimeWritable);
       }
       break;
+    default:
+      throw new RuntimeException("Primitive category " + field.getPrimitiveCategory() +
+          " not supported");
+    }
+    return writable;
+  }
 
-    case STRUCT:
-    case UNION:
-      // The only aspect of conversion to Struct / Union themselves is add fields as NULL on the end
-      // (no removal from end? which would mean skipping fields...)
+  private Object convertListRowColumn(
+      ColumnVector colVector, int batchIndex, Field field) throws IOException {
 
-      // UNDONE
-      break;
+    final SettableListObjectInspector listOI = (SettableListObjectInspector) field.objectInspector;
+    final ListComplexTypeHelper listHelper = (ListComplexTypeHelper) field.getComplexHelper();
+    final Field elementField = listHelper.getElementField();
+    final List<Object> tempList = new ArrayList<>();
+    final ListColumnVector listColumnVector = (ListColumnVector) colVector;
 
-    case LIST:
-    case MAP:
-      // Conversion only happens below to List elements or Map key and/or values and not to the
-      // List or Map itself.
-    default:
-      throw new RuntimeException("Category " + field.getCategory() + " not supported");
+    while (deserializeRead.isNextComplexMultiValue()) {
+      tempList.add(
+          convertComplexFieldRowColumn(listColumnVector.child, batchIndex, elementField));
     }
 
-    /*
-     * Convert our source object we just read into the target object and store that in the
-     * VectorizedRowBatch.
-     */
-    convertVectorAssignRow.assignConvertRowColumn(batch, batchIndex, logicalColumnIndex,
-        convertSourceWritable);
+    final int size = tempList.size();
+    final Object list = listOI.create(size);
+    for (int i = 0; i < size; i++) {
+      listOI.set(list, i, tempList.get(i));
+    }
+    return list;
+  }
+
+  private Object convertMapRowColumn(
+      ColumnVector colVector, int batchIndex, Field field) throws IOException {
+
+    final SettableMapObjectInspector mapOI = (SettableMapObjectInspector) field.objectInspector;
+    final MapComplexTypeHelper mapHelper = (MapComplexTypeHelper) field.getComplexHelper();
+    final Field keyField = mapHelper.getKeyField();
+    final Field valueField = mapHelper.getValueField();
+    final MapColumnVector mapColumnVector = (MapColumnVector) colVector;
+
+    final Object map = mapOI.create();
+    while (deserializeRead.isNextComplexMultiValue()) {
+      final Object key = convertComplexFieldRowColumn(mapColumnVector.keys, batchIndex, keyField);
+      final Object value = convertComplexFieldRowColumn(mapColumnVector.values, batchIndex, valueField);
+      mapOI.put(map, key, value);
+    }
+    return map;
+  }
+
+  private Object convertStructRowColumn(
+      ColumnVector colVector, int batchIndex, Field field) throws IOException {
+
+    final SettableStructObjectInspector structOI = (SettableStructObjectInspector) field.objectInspector;
+    final List<? extends StructField> structFields = structOI.getAllStructFieldRefs();
+    final StructComplexTypeHelper structHelper = (StructComplexTypeHelper) field.getComplexHelper();
+    final Field[] fields = structHelper.getFields();
+    final StructColumnVector structColumnVector = (StructColumnVector) colVector;
+
+    final Object struct = structOI.create();
+    for (int i = 0; i < fields.length; i++) {
+      final Object fieldObject =
+          convertComplexFieldRowColumn(structColumnVector.fields[i], batchIndex, fields[i]);
+      structOI.setStructFieldData(struct, structFields.get(i), fieldObject);
+    }
+    deserializeRead.finishComplexVariableFieldsType();
+    return struct;
+  }
+
+  private Object convertUnionRowColumn(
+      ColumnVector colVector, int batchIndex, Field field) throws IOException {
+
+    final SettableUnionObjectInspector unionOI = (SettableUnionObjectInspector) field.objectInspector;
+    final UnionComplexTypeHelper unionHelper = (UnionComplexTypeHelper) field.getComplexHelper();
+    final Field[] fields = unionHelper.getFields();
+    final UnionColumnVector unionColumnVector = (UnionColumnVector) colVector;
+
+    final Object union = unionOI.create();
+    final int tag = deserializeRead.currentInt;
+    unionOI.addField(union, new StandardUnion((byte) tag,
+        convertComplexFieldRowColumn(unionColumnVector.fields[tag], batchIndex, fields[tag])));
+    deserializeRead.finishComplexVariableFieldsType();
+    return union;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
index 1201499..23fdaa5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
@@ -257,8 +257,14 @@ public class VectorExtractRow {
             final int start = bytesColVector.start[adjustedIndex];
             final int length = bytesColVector.length[adjustedIndex];
 
-            if (bytes == null) {
-              LOG.info("null binary entry: batchIndex " + batchIndex);
+            if (bytesColVector.isRepeating) {
+              if (!bytesColVector.isNull[0] && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
+            } else {
+              if ((bytesColVector.noNulls || !bytesColVector.isNull[batchIndex]) && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
             }
 
             BytesWritable bytesWritable = (BytesWritable) primitiveWritable;
@@ -273,8 +279,14 @@ public class VectorExtractRow {
             final int start = bytesColVector.start[adjustedIndex];
             final int length = bytesColVector.length[adjustedIndex];
 
-            if (bytes == null) {
-              nullBytesReadError(primitiveCategory, batchIndex);
+            if (bytesColVector.isRepeating) {
+              if (!bytesColVector.isNull[0] && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
+            } else {
+              if ((bytesColVector.noNulls || !bytesColVector.isNull[batchIndex]) && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
             }
 
             // Use org.apache.hadoop.io.Text as our helper to go from byte[] to String.
@@ -289,8 +301,14 @@ public class VectorExtractRow {
             final int start = bytesColVector.start[adjustedIndex];
             final int length = bytesColVector.length[adjustedIndex];
 
-            if (bytes == null) {
-              nullBytesReadError(primitiveCategory, batchIndex);
+            if (bytesColVector.isRepeating) {
+              if (!bytesColVector.isNull[0] && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
+            } else {
+              if ((bytesColVector.noNulls || !bytesColVector.isNull[batchIndex]) && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
             }
 
             final int adjustedLength = StringExpr.truncate(bytes, start, length,
@@ -308,8 +326,14 @@ public class VectorExtractRow {
             final int start = bytesColVector.start[adjustedIndex];
             final int length = bytesColVector.length[adjustedIndex];
 
-            if (bytes == null) {
-              nullBytesReadError(primitiveCategory, batchIndex);
+            if (bytesColVector.isRepeating) {
+              if (!bytesColVector.isNull[0] && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
+            } else {
+              if ((bytesColVector.noNulls || !bytesColVector.isNull[batchIndex]) && bytes == null) {
+                nullBytesReadError(primitiveCategory, batchIndex);
+              }
             }
 
             final int adjustedLength = StringExpr.rightTrimAndTruncate(bytes, start, length,

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q
index 02f7c75..131a1af 100644
--- a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q
+++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q
@@ -12,9 +12,6 @@ set hive.metastore.disallow.incompatible.col.type.changes=false;
 set hive.default.fileformat=textfile;
 set hive.llap.io.enabled=false;
 
--- TEMPORARY UNTIL Vectorized Text Schema Evolution works.
-set hive.vectorized.complex.types.enabled=false;
-
 -- SORT_QUERY_RESULTS
 --
 -- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned --> all complex conversions

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/test/queries/clientpositive/schema_evol_text_vecrow_part_all_complex.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vecrow_part_all_complex.q b/ql/src/test/queries/clientpositive/schema_evol_text_vecrow_part_all_complex.q
index d780074..b4a9d66 100644
--- a/ql/src/test/queries/clientpositive/schema_evol_text_vecrow_part_all_complex.q
+++ b/ql/src/test/queries/clientpositive/schema_evol_text_vecrow_part_all_complex.q
@@ -12,9 +12,6 @@ set hive.metastore.disallow.incompatible.col.type.changes=false;
 set hive.default.fileformat=textfile;
 set hive.llap.io.enabled=false;
 
--- TEMPORARY UNTIL Vectorized Text Schema Evolution works.
-set hive.vectorized.complex.types.enabled=false;
-
 -- SORT_QUERY_RESULTS
 --
 -- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned --> all complex conversions

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out
index 0ea2972..5e0c8f7 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out
@@ -156,24 +156,42 @@ STAGE PLANS:
                 TableScan
                   alias: part_change_various_various_struct1
                   Statistics: Num rows: 6 Data size: 931 Basic stats: COMPLETE Column stats: PARTIAL
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3]
                   Select Operator
                     expressions: insert_num (type: int), part (type: int), s1 (type: struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>), b (type: string)
                     outputColumnNames: _col0, _col1, _col2, _col3
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 3, 1, 2]
                     Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
+                      File Sink Vectorization:
+                          className: VectorFileSinkOperator
+                          native: false
                       Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
-                notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type STRUCT not enabled (struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>) since hive.vectorized.complex.types.enabled IS false
-                vectorized: false
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+                rowBatchContext:
+                    dataColumnCount: 3
+                    includeColumns: [0, 1, 2]
+                    dataColumns: insert_num:int, s1:struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>, b:string
+                    partitionColumnCount: 1
+                    partitionColumns: part:int
 
   Stage: Stage-0
     Fetch Operator
@@ -438,24 +456,42 @@ STAGE PLANS:
                 TableScan
                   alias: part_add_various_various_struct2
                   Statistics: Num rows: 8 Data size: 939 Basic stats: COMPLETE Column stats: PARTIAL
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3]
                   Select Operator
                     expressions: insert_num (type: int), part (type: int), b (type: string), s2 (type: struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>)
                     outputColumnNames: _col0, _col1, _col2, _col3
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 3, 1, 2]
                     Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
+                      File Sink Vectorization:
+                          className: VectorFileSinkOperator
+                          native: false
                       Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
-                notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type STRUCT not enabled (struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>) since hive.vectorized.complex.types.enabled IS false
-                vectorized: false
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+                rowBatchContext:
+                    dataColumnCount: 3
+                    includeColumns: [0, 1, 2]
+                    dataColumns: insert_num:int, b:string, s2:struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>
+                    partitionColumnCount: 1
+                    partitionColumns: part:int
 
   Stage: Stage-0
     Fetch Operator
@@ -648,24 +684,42 @@ STAGE PLANS:
                 TableScan
                   alias: part_add_to_various_various_struct4
                   Statistics: Num rows: 4 Data size: 353 Basic stats: COMPLETE Column stats: PARTIAL
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3]
                   Select Operator
                     expressions: insert_num (type: int), part (type: int), b (type: string), s3 (type: struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:decimal(38,18),c9:char(25),c10:varchar(25),c11:timestamp,c12:date,c13:binary>)
                     outputColumnNames: _col0, _col1, _col2, _col3
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 3, 1, 2]
                     Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
+                      File Sink Vectorization:
+                          className: VectorFileSinkOperator
+                          native: false
                       Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
-                notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type STRUCT not enabled (struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:decimal(38,18),c9:char(25),c10:varchar(25),c11:timestamp,c12:date,c13:binary>) since hive.vectorized.complex.types.enabled IS false
-                vectorized: false
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+                rowBatchContext:
+                    dataColumnCount: 3
+                    includeColumns: [0, 1, 2]
+                    dataColumns: insert_num:int, b:string, s3:struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:decimal(38,18),c9:char(25),c10:varchar(25),c11:timestamp,c12:date,c13:binary>
+                    partitionColumnCount: 1
+                    partitionColumns: part:int
 
   Stage: Stage-0
     Fetch Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out
index 70be462..afaf69f 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out
@@ -156,24 +156,42 @@ STAGE PLANS:
                 TableScan
                   alias: part_change_various_various_struct1
                   Statistics: Num rows: 6 Data size: 931 Basic stats: COMPLETE Column stats: PARTIAL
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3]
                   Select Operator
                     expressions: insert_num (type: int), part (type: int), s1 (type: struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>), b (type: string)
                     outputColumnNames: _col0, _col1, _col2, _col3
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 3, 1, 2]
                     Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
+                      File Sink Vectorization:
+                          className: VectorFileSinkOperator
+                          native: false
                       Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
-                notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type STRUCT not enabled (struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>) since hive.vectorized.complex.types.enabled IS false
-                vectorized: false
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+                rowBatchContext:
+                    dataColumnCount: 3
+                    includeColumns: [0, 1, 2]
+                    dataColumns: insert_num:int, s1:struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>, b:string
+                    partitionColumnCount: 1
+                    partitionColumns: part:int
 
   Stage: Stage-0
     Fetch Operator
@@ -438,24 +456,42 @@ STAGE PLANS:
                 TableScan
                   alias: part_add_various_various_struct2
                   Statistics: Num rows: 8 Data size: 939 Basic stats: COMPLETE Column stats: PARTIAL
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3]
                   Select Operator
                     expressions: insert_num (type: int), part (type: int), b (type: string), s2 (type: struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>)
                     outputColumnNames: _col0, _col1, _col2, _col3
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 3, 1, 2]
                     Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
+                      File Sink Vectorization:
+                          className: VectorFileSinkOperator
+                          native: false
                       Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
-                notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type STRUCT not enabled (struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>) since hive.vectorized.complex.types.enabled IS false
-                vectorized: false
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+                rowBatchContext:
+                    dataColumnCount: 3
+                    includeColumns: [0, 1, 2]
+                    dataColumns: insert_num:int, b:string, s2:struct<c1:string,c2:string,c3:string,c4:string,c5:string,c6:string,c7:string,c8:string,c9:string,c10:string,c11:string,c12:string,c13:string>
+                    partitionColumnCount: 1
+                    partitionColumns: part:int
 
   Stage: Stage-0
     Fetch Operator
@@ -648,24 +684,42 @@ STAGE PLANS:
                 TableScan
                   alias: part_add_to_various_various_struct4
                   Statistics: Num rows: 4 Data size: 353 Basic stats: COMPLETE Column stats: PARTIAL
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3]
                   Select Operator
                     expressions: insert_num (type: int), part (type: int), b (type: string), s3 (type: struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:decimal(38,18),c9:char(25),c10:varchar(25),c11:timestamp,c12:date,c13:binary>)
                     outputColumnNames: _col0, _col1, _col2, _col3
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 3, 1, 2]
                     Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
+                      File Sink Vectorization:
+                          className: VectorFileSinkOperator
+                          native: false
                       Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
-                notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type STRUCT not enabled (struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:decimal(38,18),c9:char(25),c10:varchar(25),c11:timestamp,c12:date,c13:binary>) since hive.vectorized.complex.types.enabled IS false
-                vectorized: false
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+                rowBatchContext:
+                    dataColumnCount: 3
+                    includeColumns: [0, 1, 2]
+                    dataColumns: insert_num:int, b:string, s3:struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:decimal(38,18),c9:char(25),c10:varchar(25),c11:timestamp,c12:date,c13:binary>
+                    partitionColumnCount: 1
+                    partitionColumns: part:int
 
   Stage: Stage-0
     Fetch Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
index 9b3f3d0..8cf7c47 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
@@ -1060,6 +1060,13 @@ public final class LazySimpleDeserializeRead extends DeserializeRead {
           return doReadField(fields[structHelper.nextFieldIndex++]);
         } else {
 
+          // Parse until field separator (currentLevel).
+          fieldEnd = parseComplexField(fieldPosition, complexFieldEnd, currentLevel);
+          currentFieldLength = fieldEnd - fieldPosition;
+
+          structHelper.nextFieldIndex = 0;
+          boolean result = doReadField(fields[fields.length - 1]);
+
           if (!isEscaped) {
 
             // No parsing necessary -- the end is the parent's end.
@@ -1067,13 +1074,10 @@ public final class LazySimpleDeserializeRead extends DeserializeRead {
             currentEscapeCount = 0;
           } else {
             // We must parse to get the escape count.
-            fieldEnd = parseComplexField(fieldPosition, complexFieldEnd, currentLevel - 1);
+            parseComplexField(fieldPosition, complexFieldEnd, currentLevel - 1);
           }
 
-          currentFieldLength = complexFieldEnd - fieldPosition;
-
-          structHelper.nextFieldIndex = 0;
-          return doReadField(fields[fields.length - 1]);
+          return result;
         }
       }
     case UNION:

http://git-wip-us.apache.org/repos/asf/hive/blob/26d6de7e/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
index a6cda4e..886c298 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
@@ -842,7 +842,7 @@ public final class PrimitiveObjectInspectorUtils {
       }
       break;
     case BOOLEAN:
-      result = String.valueOf((((BooleanObjectInspector) oi).get(o)));
+      result = (((BooleanObjectInspector) oi).get(o)) ? "TRUE" : "FALSE";
       break;
     case BYTE:
       result = String.valueOf((((ByteObjectInspector) oi).get(o)));


Mime
View raw message