hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1544002 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/vector/expressions/ java/org/apache/hadoop/hive/ql/io/orc/ test/org/apache/hadoop/hive/ql/exec/vector/expressions/
Date Thu, 21 Nov 2013 02:20:49 GMT
Author: hashutosh
Date: Thu Nov 21 02:20:49 2013
New Revision: 1544002

URL: http://svn.apache.org/r1544002
Log:
HIVE-5845 : CTAS failed on vectorized code path (Remus Rusanu via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java Thu Nov 21 02:20:49 2013
@@ -33,4 +33,6 @@ public interface VectorExpressionWriter 
   Object writeValue(long value) throws HiveException;
   Object writeValue(double value) throws HiveException;
   Object writeValue(byte[] value, int start, int length) throws HiveException;
+  Object setValue(Object row, ColumnVector column, int columnRow) throws HiveException;
+  Object initValue(Object ost) throws HiveException;
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java Thu Nov 21 02:20:49 2013
@@ -20,8 +20,12 @@ package org.apache.hadoop.hive.ql.exec.v
 
 import java.sql.Timestamp;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -35,16 +39,39 @@ import org.apache.hadoop.hive.serde2.io.
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
 
 /**
  * VectorExpressionWritableFactory helper class for generating VectorExpressionWritable objects.
@@ -67,20 +94,11 @@ public final class VectorExpressionWrite
       return objectInspector;
     }
 
-    public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-      objectInspector = nodeDesc.getWritableObjectInspector();
-      if (null == objectInspector) {
-        objectInspector = TypeInfoUtils
-            .getStandardWritableObjectInspectorFromTypeInfo(nodeDesc.getTypeInfo());
-      }
-      if (null == objectInspector) {
-        throw new HiveException(String.format(
-            "Failed to initialize VectorExpressionWriter for expr: %s",
-            nodeDesc.getExprString()));
-      }
+    public VectorExpressionWriter init(ObjectInspector objectInspector) throws HiveException {
+      this.objectInspector = objectInspector;
       return this;
     }
-
+    
     /**
      * The base implementation must be overridden by the Long specialization
      */
@@ -88,6 +106,13 @@ public final class VectorExpressionWrite
     public Object writeValue(long value) throws HiveException {
       throw new HiveException("Internal error: should not reach here");
     }
+    
+    /**
+     * The base implementation must be overridden by the Long specialization
+     */
+    public Object setValue(Object field, long value) throws HiveException {
+      throw new HiveException("Internal error: should not reach here");
+    }
 
     /**
      * The base implementation must be overridden by the Double specialization
@@ -98,12 +123,26 @@ public final class VectorExpressionWrite
     }
 
     /**
+     * The base implementation must be overridden by the Double specialization
+     */
+    public Object setValue(Object field, double value) throws HiveException {
+      throw new HiveException("Internal error: should not reach here");
+    }
+    
+    /**
      * The base implementation must be overridden by the Bytes specialization
      */
     @Override
     public Object writeValue(byte[] value, int start, int length) throws HiveException {
       throw new HiveException("Internal error: should not reach here");
     }
+    
+    /**
+     * The base implementation must be overridden by the Bytes specialization
+     */
+    public Object setValue(Object field, byte[] value, int start, int length) throws HiveException {
+      throw new HiveException("Internal error: should not reach here");
+    }
   }
 
   /**
@@ -133,7 +172,29 @@ public final class VectorExpressionWrite
           "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
           row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0]));
     }
-   }
+    
+    @Override
+    public Object setValue(Object field, ColumnVector column, int row) throws HiveException {
+      LongColumnVector lcv = (LongColumnVector) column;
+      if (lcv.noNulls && !lcv.isRepeating) {
+        return setValue(field, lcv.vector[row]);
+      } else if (lcv.noNulls && lcv.isRepeating) {
+        return setValue(field, lcv.vector[0]);
+      } else if (!lcv.noNulls && !lcv.isRepeating && !lcv.isNull[row]) {
+        return setValue(field, lcv.vector[row]);
+      } else if (!lcv.noNulls && !lcv.isRepeating && lcv.isNull[row]) {
+        return null;
+      } else if (!lcv.noNulls && lcv.isRepeating && !lcv.isNull[0]) {
+        return setValue(field, lcv.vector[0]);
+      } else if (!lcv.noNulls && lcv.isRepeating && lcv.isNull[0]) {
+        return null;
+      }
+      throw new HiveException(
+        String.format(
+          "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+          row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0]));
+    }    
+  }
 
   /**
    * Specialized writer for DoubleColumnVector. Will throw cast exception
@@ -161,6 +222,28 @@ public final class VectorExpressionWrite
           "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
           row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
     }
+    
+    @Override
+    public Object setValue(Object field, ColumnVector column, int row) throws HiveException {
+      DoubleColumnVector dcv = (DoubleColumnVector) column;
+      if (dcv.noNulls && !dcv.isRepeating) {
+        return setValue(field, dcv.vector[row]);
+      } else if (dcv.noNulls && dcv.isRepeating) {
+        return setValue(field, dcv.vector[0]);
+      } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) {
+        return setValue(field, dcv.vector[row]);
+      } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) {
+        return null;
+      } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) {
+        return setValue(field, dcv.vector[0]);
+      } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) {
+        return null;
+      }
+      throw new HiveException(
+        String.format(
+          "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+          row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
+    }    
    }
 
   /**
@@ -189,176 +272,528 @@ public final class VectorExpressionWrite
           "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
           row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0]));
     }
+    
+    @Override
+    public Object setValue(Object field, ColumnVector column, int row) throws HiveException {
+      BytesColumnVector bcv = (BytesColumnVector) column;
+      if (bcv.noNulls && !bcv.isRepeating) {
+        return setValue(field, bcv.vector[row], bcv.start[row], bcv.length[row]);
+      } else if (bcv.noNulls && bcv.isRepeating) {
+        return setValue(field, bcv.vector[0], bcv.start[0], bcv.length[0]);
+      } else if (!bcv.noNulls && !bcv.isRepeating && !bcv.isNull[row]) {
+        return setValue(field, bcv.vector[row], bcv.start[row], bcv.length[row]);
+      } else if (!bcv.noNulls && !bcv.isRepeating && bcv.isNull[row]) {
+        return null;
+      } else if (!bcv.noNulls && bcv.isRepeating && !bcv.isNull[0]) {
+        return setValue(field, bcv.vector[0], bcv.start[0], bcv.length[0]);
+      } else if (!bcv.noNulls && bcv.isRepeating && bcv.isNull[0]) {
+        return null;
+      }
+      throw new HiveException(
+        String.format(
+          "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+          row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0]));
+    }    
    }
 
-  /**
-   * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc)
-   */
-  public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc)
-    throws HiveException {
-    String nodeType = nodeDesc.getTypeString();
-    if (nodeType.equalsIgnoreCase("tinyint")) {
-      return new VectorExpressionWriterLong()
-      {
-        private ByteWritable writable;
-
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new ByteWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(long value) {
-          writable.set((byte) value);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("smallint")) {
-      return new VectorExpressionWriterLong()
-      {
-        private ShortWritable writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new ShortWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(long value) {
-          writable.set((short) value);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("int")) {
-      return new VectorExpressionWriterLong()
-      {
-        private IntWritable writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new IntWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(long value) {
-          writable.set((int) value);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("bigint")) {
-      return new VectorExpressionWriterLong()
-      {
-        private LongWritable writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new LongWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(long value) {
-          writable.set(value);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("boolean")) {
-      return new VectorExpressionWriterLong()
-      {
-        private BooleanWritable writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new BooleanWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(long value) {
-          writable.set(value != 0 ? true : false);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("timestamp")) {
-      return new VectorExpressionWriterLong()
-      {
-        private TimestampWritable writable;
-        private Timestamp timestamp;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new TimestampWritable();
-          timestamp = new Timestamp(0);
-          return this;
-        }
-
-        @Override
-        public Object writeValue(long value) {
-          TimestampUtils.assignTimeInNanoSec(value, timestamp);
-          writable.set(timestamp);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("string")) {
-      return new VectorExpressionWriterBytes()
-      {
-        private Text writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new Text();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(byte[] value, int start, int length) throws HiveException {
-          writable.set(value, start, length);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("float")) {
-      return new VectorExpressionWriterDouble()
-      {
-        private FloatWritable writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new FloatWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(double value) {
-          writable.set((float)value);
-          return writable;
-        }
-      }.init(nodeDesc);
-    } else if (nodeType.equalsIgnoreCase("double")) {
-      return new VectorExpressionWriterDouble()
-      {
-        private DoubleWritable writable;
-        @Override
-        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
-          super.init(nodeDesc);
-          writable = new DoubleWritable();
-          return this;
-        }
-
-        @Override
-        public Object writeValue(double value) {
-          writable.set(value);
-          return writable;
-        }
-      }.init(nodeDesc);
-    }
-
-    throw new HiveException(String.format(
-        "Unimplemented genVectorExpressionWritable type: %s for expression: %s",
-        nodeType, nodeDesc));
+    /**
+     * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc)
+     */
+    public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc)
+      throws HiveException {
+      String nodeType = nodeDesc.getTypeString();
+      ObjectInspector objectInspector = nodeDesc.getWritableObjectInspector();
+      if (null == objectInspector) {
+        objectInspector = TypeInfoUtils
+            .getStandardWritableObjectInspectorFromTypeInfo(nodeDesc.getTypeInfo());
+      }
+      if (null == objectInspector) {
+        throw new HiveException(String.format(
+            "Failed to initialize VectorExpressionWriter for expr: %s",
+            nodeDesc.getExprString()));
+      }
+      return genVectorExpressionWritable(objectInspector);
+    }
+
+    /**
+     * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc)
+     */
+    public static VectorExpressionWriter genVectorExpressionWritable(
+        ObjectInspector fieldObjInspector) throws HiveException {
+
+      switch (fieldObjInspector.getCategory()) {
+      case PRIMITIVE:
+        switch (((PrimitiveObjectInspector) fieldObjInspector).getPrimitiveCategory()) {
+          case FLOAT:
+            return genVectorExpressionWritableFloat(
+                (SettableFloatObjectInspector) fieldObjInspector);
+          case DOUBLE:
+            return genVectorExpressionWritableDouble(
+                (SettableDoubleObjectInspector) fieldObjInspector);
+          case BOOLEAN:
+            return genVectorExpressionWritableBoolean(
+                (SettableBooleanObjectInspector) fieldObjInspector);
+          case BYTE:
+            return genVectorExpressionWritableByte(
+                (SettableByteObjectInspector) fieldObjInspector);
+          case SHORT:
+            return genVectorExpressionWritableShort(
+                (SettableShortObjectInspector) fieldObjInspector);
+          case INT:
+            return genVectorExpressionWritableInt(
+                (SettableIntObjectInspector) fieldObjInspector);
+          case LONG:
+            return genVectorExpressionWritableLong(
+                (SettableLongObjectInspector) fieldObjInspector);
+          case BINARY:
+            return genVectorExpressionWritableBinary(
+                (SettableBinaryObjectInspector) fieldObjInspector);
+          case STRING:
+            return genVectorExpressionWritableString(
+                (SettableStringObjectInspector) fieldObjInspector);
+          case VARCHAR:
+            return genVectorExpressionWritableVarchar(
+                (SettableHiveVarcharObjectInspector) fieldObjInspector);
+          case TIMESTAMP:
+            return genVectorExpressionWritableTimestamp(
+                (SettableTimestampObjectInspector) fieldObjInspector);
+          case DATE:
+            return genVectorExpressionWritableDate(
+                (SettableDateObjectInspector) fieldObjInspector);
+          case DECIMAL:
+            return genVectorExpressionWritableDecimal(
+                (SettableHiveDecimalObjectInspector) fieldObjInspector);
+          default:
+            throw new IllegalArgumentException("Unknown primitive type: " +
+              ((PrimitiveObjectInspector) fieldObjInspector).getPrimitiveCategory());
+        }
+        
+      case STRUCT:
+      case UNION:
+      case MAP:
+      case LIST:
+        throw new IllegalArgumentException("Unsupported complex type: " +
+            fieldObjInspector.getCategory());
+      default:
+        throw new IllegalArgumentException("Unknown type " +
+            fieldObjInspector.getCategory());      
+      }
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableDecimal(
+        SettableHiveDecimalObjectInspector fieldObjInspector) throws HiveException {
+    
+      // We should never reach this, the compile validation should guard us
+      throw new HiveException("DECIMAL primitive type not supported in vectorization.");
+    }
+
+  private static VectorExpressionWriter genVectorExpressionWritableDate(
+        SettableDateObjectInspector fieldObjInspector) throws HiveException {
+    // We should never reach this, the compile validation should guard us
+    throw new HiveException("DATE primitive type not supported in vectorization.");
+    }
+
+  private static VectorExpressionWriter genVectorExpressionWritableTimestamp(
+        SettableTimestampObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterLong() {
+      private Object obj;
+      private Timestamp ts;
+
+      public VectorExpressionWriter init(SettableTimestampObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        ts = new Timestamp(0);
+        obj = initValue(null);
+        return this;
+      }
+
+      @Override
+      public Object writeValue(long value) {
+        TimestampUtils.assignTimeInNanoSec(value, ts);
+        ((SettableTimestampObjectInspector) this.objectInspector).set(obj, ts);
+        return obj;
+      }
+
+      @Override
+      public Object setValue(Object field, long value) {
+        if (null == field) {
+          field = initValue(null);
+        }
+        TimestampUtils.assignTimeInNanoSec(value, ts);
+        ((SettableTimestampObjectInspector) this.objectInspector).set(field, ts);
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableTimestampObjectInspector) this.objectInspector).create(new Timestamp(0));
+      }
+   }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableVarchar(
+        SettableHiveVarcharObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterBytes() {
+      private Object obj;
+      private Text text;
+      
+      public VectorExpressionWriter init(SettableHiveVarcharObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.text = new Text();
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(byte[] value, int start, int length) throws HiveException {
+        text.set(value, start, length);
+        ((SettableHiveVarcharObjectInspector) this.objectInspector).set(this.obj, text.toString());
+        return this.obj;
+      }
+
+      @Override
+      public Object setValue(Object field, byte[] value, int start, int length) 
+          throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        text.set(value, start, length);
+        ((SettableHiveVarcharObjectInspector) this.objectInspector).set(field, text.toString());
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableHiveVarcharObjectInspector) this.objectInspector)
+            .create(new HiveVarchar(StringUtils.EMPTY, -1));
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableString(
+      SettableStringObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterBytes() {
+      private Object obj;
+      private Text text;
+      
+      public VectorExpressionWriter init(SettableStringObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.text = new Text();
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(byte[] value, int start, int length) throws HiveException {
+        this.text.set(value, start, length);
+        ((SettableStringObjectInspector) this.objectInspector).set(this.obj, this.text.toString());
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, byte[] value, int start, int length) 
+          throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        this.text.set(value, start, length);
+        ((SettableStringObjectInspector) this.objectInspector).set(field, this.text.toString());
+        return field;
+      }      
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableStringObjectInspector) this.objectInspector).create(StringUtils.EMPTY);
+      }      
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableBinary(
+      SettableBinaryObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterBytes() {
+      private Object obj;
+      private byte[] bytes;
+      
+      public VectorExpressionWriter init(SettableBinaryObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.bytes = ArrayUtils.EMPTY_BYTE_ARRAY;
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(byte[] value, int start, int length) throws HiveException {
+        bytes = Arrays.copyOfRange(value, start, start + length);
+        ((SettableBinaryObjectInspector) this.objectInspector).set(this.obj, bytes);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, byte[] value, int start, int length) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        bytes = Arrays.copyOfRange(value, start, start + length);
+        ((SettableBinaryObjectInspector) this.objectInspector).set(field, bytes);
+        return field;
+      }      
+
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableBinaryObjectInspector) this.objectInspector)
+            .create(ArrayUtils.EMPTY_BYTE_ARRAY);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableLong(
+      SettableLongObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterLong() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableLongObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(long value) throws HiveException {
+        ((SettableLongObjectInspector) this.objectInspector).set(this.obj, value);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, long value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableLongObjectInspector) this.objectInspector).set(field, value);
+        return field;
+      }
+
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableLongObjectInspector) this.objectInspector)
+            .create(0L);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableInt(
+      SettableIntObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterLong() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableIntObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(long value) throws HiveException {
+        ((SettableIntObjectInspector) this.objectInspector).set(this.obj, (int) value);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, long value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableIntObjectInspector) this.objectInspector).set(field, (int) value);
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableIntObjectInspector) this.objectInspector)
+            .create(0);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableShort(
+      SettableShortObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterLong() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableShortObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(long value) throws HiveException {
+        ((SettableShortObjectInspector) this.objectInspector).set(this.obj, (short) value);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, long value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableShortObjectInspector) this.objectInspector).set(field, (short) value);
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableShortObjectInspector) this.objectInspector)
+            .create((short) 0);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableByte(
+      SettableByteObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterLong() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableByteObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(long value) throws HiveException {
+        ((SettableByteObjectInspector) this.objectInspector).set(this.obj, (byte) value);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, long value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableByteObjectInspector) this.objectInspector).set(field, (byte) value);
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableByteObjectInspector) this.objectInspector)
+            .create((byte) 0);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableBoolean(
+      SettableBooleanObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterLong() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableBooleanObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(long value) throws HiveException {
+        ((SettableBooleanObjectInspector) this.objectInspector).set(this.obj, 
+            value == 0 ? false : true);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, long value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableBooleanObjectInspector) this.objectInspector).set(field, 
+            value == 0 ? false : true);
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableBooleanObjectInspector) this.objectInspector)
+            .create(false);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableDouble(
+      SettableDoubleObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterDouble() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableDoubleObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(double value) throws HiveException {
+        ((SettableDoubleObjectInspector) this.objectInspector).set(this.obj, value);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, double value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableDoubleObjectInspector) this.objectInspector).set(field, value);
+        return field;
+      }      
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableDoubleObjectInspector) this.objectInspector)
+            .create(0f);
+      }
+    }.init(fieldObjInspector);
+  }
+
+  private static VectorExpressionWriter genVectorExpressionWritableFloat(
+      SettableFloatObjectInspector fieldObjInspector) throws HiveException {
+    return new VectorExpressionWriterDouble() {
+      private Object obj;
+      
+      public VectorExpressionWriter init(SettableFloatObjectInspector objInspector) 
+          throws HiveException {
+        super.init(objInspector);
+        this.obj = initValue(null);
+        return this;
+      }
+      
+      @Override
+      public Object writeValue(double value) throws HiveException {
+        ((SettableFloatObjectInspector) this.objectInspector).set(this.obj, (float) value);
+        return this.obj;
+      }
+      
+      @Override
+      public Object setValue(Object field, double value) throws HiveException {
+        if (null == field) {
+          field = initValue(null);
+        }
+        ((SettableFloatObjectInspector) this.objectInspector).set(field, (float) value);
+        return field;
+      }
+      
+      @Override
+      public Object initValue(Object ignored) {
+        return ((SettableFloatObjectInspector) this.objectInspector)
+            .create(0f);
+      }
+    }.init(fieldObjInspector);
   }
 
   /**
@@ -432,25 +867,87 @@ public final class VectorExpressionWrite
    */
   public static VectorExpressionWriter[] getExpressionWriters(StructObjectInspector objInspector)
       throws HiveException {
-    List<ExprNodeDesc> outputNodeDescs = new ArrayList<ExprNodeDesc>();
-    for (StructField fieldRef : (objInspector)
-        .getAllStructFieldRefs()) {
-      String typeName = fieldRef.getFieldObjectInspector().getTypeName();
-      TypeInfo ti = TypeInfoFactory.getPrimitiveTypeInfo(typeName);
-      outputNodeDescs.add(new ExprNodeDesc(ti) {
-        private static final long serialVersionUID = 1L;
-
-        @Override
-        public ExprNodeDesc clone() { /* Not needed */
-          return null;
-        }
-
-        @Override
-        public boolean isSame(Object o) { /* Not needed */
-          return false;
-        }
-      });
+    
+    if (objInspector.isSettable()) {
+      return getSettableExpressionWriters((SettableStructObjectInspector) objInspector);
+    }
+    
+    List<? extends StructField> allFieldRefs = objInspector.getAllStructFieldRefs();
+    
+    VectorExpressionWriter[] expressionWriters = new VectorExpressionWriter[allFieldRefs.size()];
+    
+    for(int i=0; i<expressionWriters.length; ++i) {
+      expressionWriters[i] = genVectorExpressionWritable(allFieldRefs.get(i).getFieldObjectInspector());
+    }
+    
+    return expressionWriters;
+  }
+
+  public static VectorExpressionWriter[] getSettableExpressionWriters(
+      SettableStructObjectInspector objInspector) throws HiveException {
+    List<? extends StructField> fieldsRef = objInspector.getAllStructFieldRefs(); 
+    VectorExpressionWriter[] writers = new VectorExpressionWriter[fieldsRef.size()];
+    for(int i=0; i<writers.length; ++i) {
+      StructField fieldRef = fieldsRef.get(i);
+      VectorExpressionWriter baseWriter = genVectorExpressionWritable(
+          fieldRef.getFieldObjectInspector());
+      writers[i] = genVectorExpressionWritable(objInspector, fieldRef, baseWriter);
+    }
+    return writers;
+    
+  }
+  
+  /**
+   * VectorExpressionWriterSetter helper for vector expression writers that use 
+   * settable ObjectInspector fields to assign the values.
+   * This is used by the OrcStruct serialization (eg. CREATE TABLE ... AS ...) 
+   */
+  private static class VectorExpressionWriterSetter extends VectorExpressionWriterBase {
+    private SettableStructObjectInspector settableObjInspector;
+    private StructField fieldRef;
+    private VectorExpressionWriter baseWriter;
+    
+    public VectorExpressionWriterSetter init(
+        SettableStructObjectInspector objInspector,
+        StructField fieldRef,
+        VectorExpressionWriter baseWriter) {
+      this.fieldRef = fieldRef;
+      this.settableObjInspector = objInspector;
+      this.objectInspector = fieldRef.getFieldObjectInspector();
+      this.baseWriter = baseWriter;
+      return this;
+    }
+
+    @Override
+    public Object writeValue(ColumnVector column, int row)
+        throws HiveException {
+      throw new HiveException("Should never reach here");
     }
-    return VectorExpressionWriterFactory.getExpressionWriters(outputNodeDescs);
+
+    @Override
+    public Object setValue(Object row, ColumnVector column, int columnRow)
+        throws HiveException {
+      
+      // NULLs are handled by each individual base writer setter
+      // We could handle NULLs centrally here but that would result in spurious allocs
+      
+      Object fieldValue = this.settableObjInspector.getStructFieldData(row, fieldRef);
+      fieldValue = baseWriter.setValue(fieldValue, column, columnRow);
+      return this.settableObjInspector.setStructFieldData(row, fieldRef, fieldValue);
+    }
+    
+    @Override
+    public Object initValue(Object struct) throws HiveException {
+      Object initValue = this.baseWriter.initValue(null);
+      this.settableObjInspector.setStructFieldData(struct, fieldRef, initValue);
+      return struct;
+    }
+  }
+  
+  private static VectorExpressionWriter genVectorExpressionWritable(
+      SettableStructObjectInspector objInspector,
+      StructField fieldRef,
+      VectorExpressionWriter baseWriter) throws HiveException {
+    return new VectorExpressionWriterSetter().init(objInspector, fieldRef, baseWriter);
   }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java Thu Nov 21 02:20:49 2013
@@ -143,9 +143,9 @@ public class OrcSerde implements SerDe, 
   public Writable serializeVector(VectorizedRowBatch vrg, ObjectInspector objInspector)
       throws SerDeException {
     if (vos == null) {
-      vos = new VectorizedOrcSerde(objInspector);
+      vos = new VectorizedOrcSerde(getObjectInspector());
     }
-    return vos.serialize(vrg, objInspector);
+    return vos.serialize(vrg, getObjectInspector());
   }
 
   @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcSerde.java Thu Nov 21 02:20:49 2013
@@ -69,13 +69,11 @@ public class VectorizedOrcSerde extends 
         }
         for (int p = 0; p < batch.projectionSize; p++) {
           int k = batch.projectedColumns[p];
-          Writable w;
           if (batch.cols[k].isRepeating) {
-            w = (Writable) valueWriters[p].writeValue(batch.cols[k], 0);
+            valueWriters[p].setValue(ost, batch.cols[k], 0);
           } else {
-            w = (Writable) valueWriters[p].writeValue(batch.cols[k], index);
+            valueWriters[p].setValue(ost, batch.cols[k], index);
           }
-          ost.setFieldValue(k, w);
         }
         OrcSerdeRow row = (OrcSerdeRow) orcRowArray[i];
         row.realRow = ost;

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java?rev=1544002&r1=1544001&r2=1544002&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java Thu Nov 21 02:20:49 2013
@@ -20,10 +20,12 @@ package org.apache.hadoop.hive.ql.exec.v
 
 
 import java.sql.Timestamp;
+import java.util.ArrayList;
 import java.util.Random;
 
 import junit.framework.Assert;
 
+import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
@@ -34,11 +36,18 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
@@ -73,6 +82,11 @@ public class TestVectorExpressionWriters
   private Writable getWritableValue(TypeInfo ti, byte[] value) {
     if (ti.equals(TypeInfoFactory.stringTypeInfo)) {
       return new Text(value);
+    } else if (ti.equals(TypeInfoFactory.varcharTypeInfo)) {
+      return new HiveVarcharWritable(
+          new HiveVarchar(new Text(value).toString(), -1));
+    } else if (ti.equals(TypeInfoFactory.binaryTypeInfo)) {
+      return new BytesWritable(value);
     }
     return null;
   }
@@ -112,6 +126,25 @@ public class TestVectorExpressionWriters
       }
     }
   }
+  
+  private void testSetterDouble(TypeInfo type) throws HiveException {
+    DoubleColumnVector dcv = VectorizedRowGroupGenUtil.generateDoubleColumnVector(true, false,
+        this.vectorSize, new Random(10));
+    dcv.isNull[2] = true;
+    Object[] values = new Object[this.vectorSize];
+    
+    VectorExpressionWriter vew = getWriter(type);
+    for (int i = 0; i < vectorSize; i++) {
+      values[i] = vew.initValue(null);
+      values[i] = vew.setValue(values[i], dcv, i);
+      if (values[i] != null) {
+        Writable expected = getWritableValue(type, dcv.vector[i]);
+        Assert.assertEquals(expected, values[i]);
+      } else {
+        Assert.assertTrue(dcv.isNull[i]);
+      }
+    }
+  }  
 
   private void testWriterLong(TypeInfo type) throws HiveException {
     LongColumnVector lcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
@@ -135,8 +168,89 @@ public class TestVectorExpressionWriters
       }
     }
   }
+  
+  private void testSetterLong(TypeInfo type) throws HiveException {
+    LongColumnVector lcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
+        vectorSize, new Random(10));
+    lcv.isNull[3] = true;
 
-  private void testWriterBytes(TypeInfo type) throws HiveException {
+    Object[] values = new Object[this.vectorSize];
+    
+    VectorExpressionWriter vew = getWriter(type);
+    for (int i = 0; i < vectorSize; i++) {
+      values[i] = vew.initValue(null);
+      values[i] = vew.setValue(values[i], lcv, i);
+      if (values[i] != null) {
+        Writable expected = getWritableValue(type, lcv.vector[i]);
+        if (expected instanceof TimestampWritable) {
+          TimestampWritable t1 = (TimestampWritable) expected;
+          TimestampWritable t2 = (TimestampWritable) values[i];
+          Assert.assertTrue(t1.getNanos() == t2.getNanos());
+          Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
+          continue;
+        }
+        Assert.assertEquals(expected, values[i]);
+      } else {
+        Assert.assertTrue(lcv.isNull[i]);
+      }
+    }
+  }
+  
+  private StructObjectInspector genStructOI() {
+    ArrayList<String> fieldNames1 = new ArrayList<String>();
+    fieldNames1.add("theInt");
+    fieldNames1.add("theBool");
+    ArrayList<ObjectInspector> fieldObjectInspectors1 = new ArrayList<ObjectInspector>();
+    fieldObjectInspectors1
+        .add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
+    fieldObjectInspectors1
+        .add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
+    return ObjectInspectorFactory
+        .getStandardStructObjectInspector(fieldNames1, fieldObjectInspectors1);
+  }
+  
+  private void testStructLong(TypeInfo type) throws HiveException {
+    LongColumnVector icv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
+        vectorSize, new Random(10));
+    icv.isNull[3] = true;
+
+    LongColumnVector bcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
+        vectorSize, new Random(10));
+    bcv.isNull[2] = true;
+    
+    ArrayList<Object>[] values = (ArrayList<Object>[]) new ArrayList[this.vectorSize];
+    
+    StructObjectInspector soi = genStructOI();
+    
+    VectorExpressionWriter[] vew = VectorExpressionWriterFactory.getExpressionWriters(soi);
+    
+    for (int i = 0; i < vectorSize; i++) {
+      values[i] = new ArrayList<Object>(2);
+      values[i].add(null);
+      values[i].add(null);
+
+      vew[0].setValue(values[i], icv, i);
+      vew[1].setValue(values[i], bcv, i);
+      
+      Object theInt = values[i].get(0);
+      if (theInt == null) {
+        Assert.assertTrue(icv.isNull[i]);
+      } else {
+        IntWritable w = (IntWritable) theInt;
+        Assert.assertEquals((int) icv.vector[i], w.get());
+      }
+
+      Object theBool = values[i].get(1);
+      if (theBool == null) {
+        Assert.assertTrue(bcv.isNull[i]);
+      } else {
+        BooleanWritable w = (BooleanWritable) theBool;
+        Assert.assertEquals(bcv.vector[i] == 0 ? false : true, w.get());
+      }
+    }
+  }
+
+  private void testWriterText(TypeInfo type) throws HiveException {
     Text t1 = new Text("alpha");
     Text t2 = new Text("beta");
     BytesColumnVector bcv = new BytesColumnVector(vectorSize);
@@ -160,36 +274,100 @@ public class TestVectorExpressionWriters
       }
     }
   }
-
+  
+  private void testSetterText(TypeInfo type) throws HiveException {
+    Text t1 = new Text("alpha");
+    Text t2 = new Text("beta");
+    BytesColumnVector bcv = new BytesColumnVector(vectorSize);
+    bcv.noNulls = false;
+    bcv.initBuffer();
+    bcv.setVal(0, t1.getBytes(), 0, t1.getLength());
+    bcv.isNull[1] = true;
+    bcv.setVal(2, t2.getBytes(), 0, t2.getLength());
+    bcv.isNull[3] = true;
+    bcv.setVal(4, t1.getBytes(), 0, t1.getLength());
+    
+    Object[] values = new Object[this.vectorSize];
+    VectorExpressionWriter vew = getWriter(type);
+    for (int i = 0; i < vectorSize; i++) {
+      values[i] = vew.initValue(null);
+      Writable w = (Writable) vew.setValue(values[i], bcv, i);
+      if (w != null) {
+        byte [] val = new byte[bcv.length[i]];
+        System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]);
+        Writable expected = getWritableValue(type, val);
+        Assert.assertEquals(expected, w);
+      } else {
+        Assert.assertTrue(bcv.isNull[i]);
+      }
+    }
+  }
+  
   @Test
   public void testVectorExpressionWriterDouble() throws HiveException {
     testWriterDouble(TypeInfoFactory.doubleTypeInfo);
   }
 
   @Test
+  public void testVectorExpressionSetterDouble() throws HiveException {
+    testSetterDouble(TypeInfoFactory.doubleTypeInfo);
+  }  
+
+  @Test
   public void testVectorExpressionWriterFloat() throws HiveException {
     testWriterDouble(TypeInfoFactory.floatTypeInfo);
   }
 
   @Test
+  public void testVectorExpressionSetterFloat() throws HiveException {
+    testSetterDouble(TypeInfoFactory.floatTypeInfo);
+  }
+  
+  @Test
   public void testVectorExpressionWriterLong() throws HiveException {
     testWriterLong(TypeInfoFactory.longTypeInfo);
   }
-
+  
+  @Test
+  public void testVectorExpressionSetterLong() throws HiveException {
+    testSetterLong(TypeInfoFactory.longTypeInfo);
+  }
+  
+  @Test
+  public void testVectorExpressionStructLong() throws HiveException {
+    testStructLong(TypeInfoFactory.longTypeInfo);
+  }
+  
   @Test
   public void testVectorExpressionWriterInt() throws HiveException {
     testWriterLong(TypeInfoFactory.intTypeInfo);
   }
 
   @Test
+  public void testVectorExpressionSetterInt() throws HiveException {
+    testSetterLong(TypeInfoFactory.intTypeInfo);
+  }
+
+  @Test
   public void testVectorExpressionWriterShort() throws HiveException {
     testWriterLong(TypeInfoFactory.shortTypeInfo);
   }
 
   @Test
+  public void testVectorExpressionSetterShort() throws HiveException {
+    testSetterLong(TypeInfoFactory.shortTypeInfo);
+  }
+
+  
+  @Test
   public void testVectorExpressionWriterBoolean() throws HiveException {
     testWriterLong(TypeInfoFactory.booleanTypeInfo);
   }
+  
+  @Test
+  public void testVectorExpressionSetterBoolean() throws HiveException {
+    testSetterLong(TypeInfoFactory.booleanTypeInfo);
+  }
 
   @Test
   public void testVectorExpressionWriterTimestamp() throws HiveException {
@@ -197,12 +375,47 @@ public class TestVectorExpressionWriters
   }
 
   @Test
-  public void testVectorExpressionWriterBye() throws HiveException {
+  public void testVectorExpressionSetterTimestamp() throws HiveException {
+    testSetterLong(TypeInfoFactory.timestampTypeInfo);
+  }
+  
+  @Test
+  public void testVectorExpressionWriterByte() throws HiveException {
     testWriterLong(TypeInfoFactory.byteTypeInfo);
   }
+  
+  @Test
+  public void testVectorExpressionSetterByte() throws HiveException {
+    testSetterLong(TypeInfoFactory.byteTypeInfo);
+  }
 
   @Test
-  public void testVectorExpressionWriterBytes() throws HiveException {
-    testWriterBytes(TypeInfoFactory.stringTypeInfo);
+  public void testVectorExpressionWriterString() throws HiveException {
+    testWriterText(TypeInfoFactory.stringTypeInfo);
+  }
+  
+  @Test
+  public void testVectorExpressionSetterString() throws HiveException {
+    testSetterText(TypeInfoFactory.stringTypeInfo);
+  }
+  
+  @Test
+  public void testVectorExpressionWriterVarchar() throws HiveException {
+    testWriterText(TypeInfoFactory.varcharTypeInfo);
+  }
+  
+  @Test
+  public void testVectorExpressionSetterVarchar() throws HiveException {
+    testSetterText(TypeInfoFactory.varcharTypeInfo);
+  }    
+
+  @Test
+  public void testVectorExpressionWriterBinary() throws HiveException {
+    testWriterText(TypeInfoFactory.binaryTypeInfo);
+  }
+  
+  @Test
+  public void testVectorExpressionSetterBinary() throws HiveException {
+    testSetterText(TypeInfoFactory.binaryTypeInfo);
   }
 }



Mime
View raw message