hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmccl...@apache.org
Subject [06/20] hive git commit: HIVE-9862 Vectorized execution corrupts timestamp values (Matt McCline, reviewed by Jason Dere) HIVE-13111: Fix timestamp / interval_day_time wrong results with HIVE-9862 (Matt McCline, reviewed by Jason Dere)
Date Tue, 19 Apr 2016 10:13:00 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
index 419254b..d4f1f6f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
@@ -32,10 +32,12 @@ import java.util.Random;
 import junit.framework.Assert;
 
 import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.TestVectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.udf.UDFDayOfMonth;
 import org.apache.hadoop.hive.ql.udf.UDFHour;
@@ -56,51 +58,42 @@ import org.junit.Test;
 public class TestVectorTimestampExpressions {
   private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
-  /* copied over from VectorUDFTimestampFieldLong */
-  private TimestampWritable toTimestampWritable(long nanos) {
-    long ms = (nanos / (1000 * 1000 * 1000)) * 1000;
-    /* the milliseconds should be kept in nanos */
-    long ns = nanos % (1000*1000*1000);
-    if (ns < 0) {
-      /*
-       * The nano seconds are always positive,
-       * but the milliseconds can be negative
-       */
-      ms -= 1000;
-      ns += 1000*1000*1000;
-    }
-    Timestamp ts = new Timestamp(ms);
-    ts.setNanos((int) ns);
-    return new TimestampWritable(ts);
-  }
-
-  private long[] getAllBoundaries() {
-    List<Long> boundaries = new ArrayList<Long>(1);
+  private Timestamp[] getAllBoundaries(int minYear, int maxYear) {
+     ArrayList<Timestamp> boundaries = new ArrayList<Timestamp>(1);
     Calendar c = Calendar.getInstance();
     c.setTimeInMillis(0); // c.set doesn't reset millis
-    for (int year = 1902; year <= 2038; year++) {
+    for (int year = minYear; year <= maxYear; year++) {
       c.set(year, Calendar.JANUARY, 1, 0, 0, 0);
-      long exactly = c.getTimeInMillis() * 1000 * 1000;
+      if (c.get(Calendar.YEAR) < 0 || c.get(Calendar.YEAR) >= 10000) {
+        continue;
+      }
+      long exactly = c.getTimeInMillis();
       /* one second before and after */
-      long before = exactly - 1000 * 1000 * 1000;
-      long after = exactly + 1000 * 1000 * 1000;
-      boundaries.add(Long.valueOf(before));
-      boundaries.add(Long.valueOf(exactly));
-      boundaries.add(Long.valueOf(after));
+      long before = exactly - 1000;
+      long after = exactly + 1000;
+      if (minYear != 0) {
+        boundaries.add(new Timestamp(before));
+      }
+      boundaries.add(new Timestamp(exactly));
+      if (year != maxYear) {
+        boundaries.add(new Timestamp(after));
+      }
     }
-    Long[] indices = boundaries.toArray(new Long[1]);
-    return ArrayUtils.toPrimitive(indices);
+    return boundaries.toArray(new Timestamp[0]);
+  }
+
+  private Timestamp[] getAllBoundaries() {
+    return getAllBoundaries(RandomTypeUtil.MIN_YEAR, RandomTypeUtil.MAX_YEAR);
   }
 
-  private VectorizedRowBatch getVectorizedRandomRowBatchLong2(int seed, int size) {
+  private VectorizedRowBatch getVectorizedRandomRowBatchTimestampLong(int seed, int size) {
     VectorizedRowBatch batch = new VectorizedRowBatch(2, size);
-    LongColumnVector lcv = new LongColumnVector(size);
+    TimestampColumnVector tcv = new TimestampColumnVector(size);
     Random rand = new Random(seed);
     for (int i = 0; i < size; i++) {
-      /* all 32 bit numbers qualify & multiply up to get nano-seconds */
-      lcv.vector[i] = (long)(1000*1000*1000*rand.nextInt());
+      tcv.set(i, RandomTypeUtil.getRandTimestamp(rand));
     }
-    batch.cols[0] = lcv;
+    batch.cols[0] = tcv;
     batch.cols[1] = new LongColumnVector(size);
     batch.size = size;
     return batch;
@@ -112,7 +105,7 @@ public class TestVectorTimestampExpressions {
     Random rand = new Random(seed);
     for (int i = 0; i < size; i++) {
       /* all 32 bit numbers qualify & multiply up to get nano-seconds */
-      byte[] encoded = encodeTime(1000 * 1000 * 1000 * rand.nextInt());
+      byte[] encoded = encodeTime(RandomTypeUtil.getRandTimestamp(rand));
       bcv.vector[i] = encoded;
       bcv.start[i] = 0;
       bcv.length[i] = encoded.length;
@@ -125,8 +118,8 @@ public class TestVectorTimestampExpressions {
 
   private VectorizedRowBatch getVectorizedRandomRowBatch(int seed, int size, TestType testType) {
     switch (testType) {
-      case LONG2:
-        return getVectorizedRandomRowBatchLong2(seed, size);
+      case TIMESTAMP_LONG:
+        return getVectorizedRandomRowBatchTimestampLong(seed, size);
       case STRING_LONG:
         return getVectorizedRandomRowBatchStringLong(seed, size);
       default:
@@ -137,13 +130,13 @@ public class TestVectorTimestampExpressions {
   /*
    * Input array is used to fill the entire size of the vector row batch
    */
-  private VectorizedRowBatch getVectorizedRowBatchLong2(long[] inputs, int size) {
+  private VectorizedRowBatch getVectorizedRowBatchTimestampLong(Timestamp[] inputs, int size) {
     VectorizedRowBatch batch = new VectorizedRowBatch(2, size);
-    LongColumnVector lcv = new LongColumnVector(size);
+    TimestampColumnVector tcv = new TimestampColumnVector(size);
     for (int i = 0; i < size; i++) {
-      lcv.vector[i] = inputs[i % inputs.length];
+      tcv.set(i, inputs[i % inputs.length]);
     }
-    batch.cols[0] = lcv;
+    batch.cols[0] = tcv;
     batch.cols[1] = new LongColumnVector(size);
     batch.size = size;
     return batch;
@@ -152,7 +145,7 @@ public class TestVectorTimestampExpressions {
   /*
    * Input array is used to fill the entire size of the vector row batch
    */
-  private VectorizedRowBatch getVectorizedRowBatchStringLong(long[] inputs, int size) {
+  private VectorizedRowBatch getVectorizedRowBatchStringLong(Timestamp[] inputs, int size) {
     VectorizedRowBatch batch = new VectorizedRowBatch(2, size);
     BytesColumnVector bcv = new BytesColumnVector(size);
     for (int i = 0; i < size; i++) {
@@ -181,10 +174,10 @@ public class TestVectorTimestampExpressions {
     return batch;
   }
 
-  private VectorizedRowBatch getVectorizedRowBatch(long[] inputs, int size, TestType testType) {
+  private VectorizedRowBatch getVectorizedRowBatch(Timestamp[] inputs, int size, TestType testType) {
     switch (testType) {
-      case LONG2:
-        return getVectorizedRowBatchLong2(inputs, size);
+      case TIMESTAMP_LONG:
+        return getVectorizedRowBatchTimestampLong(inputs, size);
       case STRING_LONG:
         return getVectorizedRowBatchStringLong(inputs, size);
       default:
@@ -192,10 +185,11 @@ public class TestVectorTimestampExpressions {
     }
   }
 
-  private byte[] encodeTime(long time) {
+  private byte[] encodeTime(Timestamp timestamp) {
     ByteBuffer encoded;
+    long time = timestamp.getTime();
     try {
-      String formatted = dateFormat.format(new Date(time / (1000 * 1000)));
+      String formatted = dateFormat.format(new Date(time));
       encoded = Text.encode(formatted);
     } catch (CharacterCodingException e) {
       throw new RuntimeException(e);
@@ -203,17 +197,17 @@ public class TestVectorTimestampExpressions {
     return Arrays.copyOf(encoded.array(), encoded.limit());
   }
 
-  private long decodeTime(byte[] time) {
+  private Timestamp decodeTime(byte[] time) {
     try {
-      return dateFormat.parse(Text.decode(time)).getTime() * 1000 * 1000;
+      return new Timestamp(dateFormat.parse(Text.decode(time)).getTime());
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }
 
-  private long readVectorElementAt(ColumnVector col, int i) {
-    if (col instanceof LongColumnVector) {
-      return ((LongColumnVector) col).vector[i];
+  private Timestamp readVectorElementAt(ColumnVector col, int i) {
+    if (col instanceof TimestampColumnVector) {
+      return ((TimestampColumnVector) col).asScratchTimestamp(i);
     }
     if (col instanceof BytesColumnVector) {
       byte[] timeBytes = ((BytesColumnVector) col).vector[i];
@@ -223,20 +217,24 @@ public class TestVectorTimestampExpressions {
   }
 
   private enum TestType {
-    LONG2, STRING_LONG
+    TIMESTAMP_LONG, STRING_LONG
   }
 
-  private void compareToUDFYearLong(long t, int y) {
+  private void compareToUDFYearLong(Timestamp t, int y) {
     UDFYear udf = new UDFYear();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
+    if (res.get() != y) {
+      System.out.printf("%d vs %d for %s, %d\n", res.get(), y, t.toString(),
+          tsw.getTimestamp().getTime()/1000);
+    }
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFYear(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf = null;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFYearLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFYearTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFYearString(0, 1);
@@ -251,7 +249,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[in].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFYearLong(t, (int) y);
       } else {
@@ -261,7 +259,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFYear(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -269,7 +267,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFYear(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFYear(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -277,8 +275,17 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFYear(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    verifyUDFYear(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFYear(batch, testType);
+
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
     verifyUDFYear(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
@@ -293,8 +300,8 @@ public class TestVectorTimestampExpressions {
   }
 
   @Test
-  public void testVectorUDFYearLong() {
-    testVectorUDFYear(TestType.LONG2);
+  public void testVectorUDFYearTimestamp() {
+    testVectorUDFYear(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -309,17 +316,17 @@ public class TestVectorTimestampExpressions {
     Assert.assertEquals(true, lcv.isNull[0]);
   }
 
-  private void compareToUDFDayOfMonthLong(long t, int y) {
+  private void compareToUDFDayOfMonthLong(Timestamp t, int y) {
     UDFDayOfMonth udf = new UDFDayOfMonth();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFDayOfMonth(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf = null;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFDayOfMonthLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFDayOfMonthTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFDayOfMonthString(0, 1);
@@ -334,7 +341,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[in].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFDayOfMonthLong(t, (int) y);
       } else {
@@ -344,7 +351,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFDayOfMonth(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -352,7 +359,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFDayOfMonth(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFDayOfMonth(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -360,13 +367,22 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFDayOfMonth(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
     verifyUDFDayOfMonth(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
     verifyUDFDayOfMonth(batch, testType);
 
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
+    verifyUDFDayOfMonth(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFDayOfMonth(batch, testType);
+
     batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType);
     verifyUDFDayOfMonth(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -376,8 +392,8 @@ public class TestVectorTimestampExpressions {
   }
 
   @Test
-  public void testVectorUDFDayOfMonthLong() {
-    testVectorUDFDayOfMonth(TestType.LONG2);
+  public void testVectorUDFDayOfMonthTimestamp() {
+    testVectorUDFDayOfMonth(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -385,17 +401,17 @@ public class TestVectorTimestampExpressions {
     testVectorUDFDayOfMonth(TestType.STRING_LONG);
   }
 
-  private void compareToUDFHourLong(long t, int y) {
+  private void compareToUDFHourLong(Timestamp t, int y) {
     UDFHour udf = new UDFHour();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFHour(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf = null;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFHourLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFHourTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFHourString(0, 1);
@@ -410,7 +426,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[in].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFHourLong(t, (int) y);
       } else {
@@ -420,7 +436,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFHour(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -428,7 +444,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFHour(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFHour(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -436,13 +452,22 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFHour(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
     verifyUDFHour(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
     verifyUDFHour(batch, testType);
 
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
+    verifyUDFHour(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFHour(batch, testType);
+
     batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType);
     verifyUDFHour(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -452,8 +477,8 @@ public class TestVectorTimestampExpressions {
   }
 
   @Test
-  public void testVectorUDFHourLong() {
-    testVectorUDFHour(TestType.LONG2);
+  public void testVectorUDFHourTimestamp() {
+    testVectorUDFHour(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -461,17 +486,17 @@ public class TestVectorTimestampExpressions {
     testVectorUDFHour(TestType.STRING_LONG);
   }
 
-  private void compareToUDFMinuteLong(long t, int y) {
+  private void compareToUDFMinuteLong(Timestamp t, int y) {
     UDFMinute udf = new UDFMinute();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFMinute(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf = null;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFMinuteLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFMinuteTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFMinuteString(0, 1);
@@ -486,7 +511,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[in].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFMinuteLong(t, (int) y);
       } else {
@@ -496,7 +521,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFMinute(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -504,7 +529,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFMinute(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFMinute(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -512,13 +537,22 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFMinute(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
     verifyUDFMinute(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
     verifyUDFMinute(batch, testType);
 
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
+    verifyUDFMinute(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFMinute(batch, testType);
+
     batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType);
     verifyUDFMinute(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -529,7 +563,7 @@ public class TestVectorTimestampExpressions {
 
   @Test
   public void testVectorUDFMinuteLong() {
-    testVectorUDFMinute(TestType.LONG2);
+    testVectorUDFMinute(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -537,17 +571,17 @@ public class TestVectorTimestampExpressions {
     testVectorUDFMinute(TestType.STRING_LONG);
   }
 
-  private void compareToUDFMonthLong(long t, int y) {
+  private void compareToUDFMonthLong(Timestamp t, int y) {
     UDFMonth udf = new UDFMonth();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFMonth(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFMonthLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFMonthTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFMonthString(0, 1);
@@ -562,7 +596,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[in].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFMonthLong(t, (int) y);
       } else {
@@ -572,7 +606,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFMonth(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -580,7 +614,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFMonth(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFMonth(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -588,13 +622,22 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFMonth(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
     verifyUDFMonth(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
     verifyUDFMonth(batch, testType);
 
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
+    verifyUDFMonth(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFMonth(batch, testType);
+
     batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType);
     verifyUDFMonth(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -604,8 +647,8 @@ public class TestVectorTimestampExpressions {
   }
 
   @Test
-  public void testVectorUDFMonthLong() {
-    testVectorUDFMonth(TestType.LONG2);
+  public void testVectorUDFMonthTimestamp() {
+    testVectorUDFMonth(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -613,17 +656,17 @@ public class TestVectorTimestampExpressions {
     testVectorUDFMonth(TestType.STRING_LONG);
   }
 
-  private void compareToUDFSecondLong(long t, int y) {
+  private void compareToUDFSecondLong(Timestamp t, int y) {
     UDFSecond udf = new UDFSecond();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFSecond(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFSecondLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFSecondTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFSecondString(0, 1);
@@ -638,7 +681,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[in].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFSecondLong(t, (int) y);
       } else {
@@ -648,7 +691,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFSecond(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -656,7 +699,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFSecond(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFSecond(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -664,13 +707,22 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFSecond(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
     verifyUDFSecond(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
     verifyUDFSecond(batch, testType);
 
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
+    verifyUDFSecond(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFSecond(batch, testType);
+
     batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType);
     verifyUDFSecond(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -681,7 +733,7 @@ public class TestVectorTimestampExpressions {
 
   @Test
   public void testVectorUDFSecondLong() {
-    testVectorUDFSecond(TestType.LONG2);
+    testVectorUDFSecond(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -689,31 +741,18 @@ public class TestVectorTimestampExpressions {
     testVectorUDFSecond(TestType.STRING_LONG);
   }
 
-  private LongWritable getLongWritable(TimestampWritable i) {
-    LongWritable result = new LongWritable();
-    if (i == null) {
-      return null;
-    } else {
-      result.set(i.getSeconds());
-      return result;
+  private void compareToUDFUnixTimeStampLong(Timestamp ts, long y) {
+    long seconds = ts.getTime() / 1000;
+    if(seconds != y) {
+      System.out.printf("%d vs %d for %s\n", seconds, y, ts.toString());
+      Assert.assertTrue(false);
     }
   }
 
-  private void compareToUDFUnixTimeStampLong(long t, long y) {
-    TimestampWritable tsw = toTimestampWritable(t);
-    LongWritable res = getLongWritable(tsw);
-    if(res.get() != y) {
-      System.out.printf("%d vs %d for %d, %d\n", res.get(), y, t,
-          tsw.getTimestamp().getTime()/1000);
-    }
-
-    Assert.assertEquals(res.get(), y);
-  }
-
   private void verifyUDFUnixTimeStamp(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFUnixTimeStampLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFUnixTimeStampTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFUnixTimeStampString(0, 1);
@@ -728,7 +767,7 @@ public class TestVectorTimestampExpressions {
         if (!batch.cols[out].noNulls) {
           Assert.assertEquals(batch.cols[out].isNull[i], batch.cols[in].isNull[i]);
         }
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFUnixTimeStampLong(t, y);
       } else {
@@ -738,7 +777,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFUnixTimeStamp(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -746,7 +785,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFUnixTimeStamp(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFUnixTimeStamp(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -754,13 +793,22 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFUnixTimeStamp(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
     verifyUDFUnixTimeStamp(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
     verifyUDFUnixTimeStamp(batch, testType);
 
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
+    verifyUDFUnixTimeStamp(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+
+    verifyUDFUnixTimeStamp(batch, testType);
     batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, testType);
     verifyUDFUnixTimeStamp(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -770,8 +818,8 @@ public class TestVectorTimestampExpressions {
   }
 
   @Test
-  public void testVectorUDFUnixTimeStampLong() {
-    testVectorUDFUnixTimeStamp(TestType.LONG2);
+  public void testVectorUDFUnixTimeStampTimestamp() {
+    testVectorUDFUnixTimeStamp(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -779,17 +827,17 @@ public class TestVectorTimestampExpressions {
     testVectorUDFUnixTimeStamp(TestType.STRING_LONG);
   }
 
-  private void compareToUDFWeekOfYearLong(long t, int y) {
+  private void compareToUDFWeekOfYearLong(Timestamp t, int y) {
     UDFWeekOfYear udf = new UDFWeekOfYear();
-    TimestampWritable tsw = toTimestampWritable(t);
+    TimestampWritable tsw = new TimestampWritable(t);
     IntWritable res = udf.evaluate(tsw);
     Assert.assertEquals(res.get(), y);
   }
 
   private void verifyUDFWeekOfYear(VectorizedRowBatch batch, TestType testType) {
     VectorExpression udf;
-    if (testType == TestType.LONG2) {
-      udf = new VectorUDFWeekOfYearLong(0, 1);
+    if (testType == TestType.TIMESTAMP_LONG) {
+      udf = new VectorUDFWeekOfYearTimestamp(0, 1);
       udf.setInputTypes(VectorExpression.Type.TIMESTAMP);
     } else {
       udf = new VectorUDFWeekOfYearString(0, 1);
@@ -801,7 +849,7 @@ public class TestVectorTimestampExpressions {
 
     for (int i = 0; i < batch.size; i++) {
       if (batch.cols[in].noNulls || !batch.cols[in].isNull[i]) {
-        long t = readVectorElementAt(batch.cols[in], i);
+        Timestamp t = readVectorElementAt(batch.cols[in], i);
         long y = ((LongColumnVector) batch.cols[out]).vector[i];
         compareToUDFWeekOfYearLong(t, (int) y);
       } else {
@@ -811,7 +859,7 @@ public class TestVectorTimestampExpressions {
   }
 
   private void testVectorUDFWeekOfYear(TestType testType) {
-    VectorizedRowBatch batch = getVectorizedRowBatch(new long[] {0},
+    VectorizedRowBatch batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)},
             VectorizedRowBatch.DEFAULT_SIZE, testType);
     Assert.assertTrue(((LongColumnVector) batch.cols[1]).noNulls);
     Assert.assertFalse(((LongColumnVector) batch.cols[1]).isRepeating);
@@ -819,7 +867,7 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
     verifyUDFWeekOfYear(batch, testType);
 
-    long[] boundaries = getAllBoundaries();
+    Timestamp[] boundaries = getAllBoundaries();
     batch = getVectorizedRowBatch(boundaries, boundaries.length, testType);
     verifyUDFWeekOfYear(batch, testType);
     TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
@@ -827,8 +875,17 @@ public class TestVectorTimestampExpressions {
     TestVectorizedRowBatch.addRandomNulls(batch.cols[1]);
     verifyUDFWeekOfYear(batch, testType);
 
-    batch = getVectorizedRowBatch(new long[] {0}, 1, testType);
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
+    batch.cols[0].isRepeating = true;
+    verifyUDFWeekOfYear(batch, testType);
+    batch.cols[0].noNulls = false;
+    batch.cols[0].isNull[0] = true;
+    verifyUDFWeekOfYear(batch, testType);
+
+    batch = getVectorizedRowBatch(new Timestamp[] {new Timestamp(0)}, 1, testType);
     batch.cols[0].isRepeating = true;
+    batch.selectedInUse = true;
+    batch.selected = new int[] {42};
     verifyUDFWeekOfYear(batch, testType);
     batch.cols[0].noNulls = false;
     batch.cols[0].isNull[0] = true;
@@ -843,8 +900,8 @@ public class TestVectorTimestampExpressions {
   }
 
   @Test
-  public void testVectorUDFWeekOfYearLong() {
-    testVectorUDFWeekOfYear(TestType.LONG2);
+  public void testVectorUDFWeekOfYearTimestamp() {
+    testVectorUDFWeekOfYear(TestType.TIMESTAMP_LONG);
   }
 
   @Test
@@ -854,12 +911,13 @@ public class TestVectorTimestampExpressions {
 
   public static void main(String[] args) {
     TestVectorTimestampExpressions self = new TestVectorTimestampExpressions();
-    self.testVectorUDFYearLong();
-    self.testVectorUDFMonthLong();
-    self.testVectorUDFDayOfMonthLong();
-    self.testVectorUDFHourLong();
-    self.testVectorUDFWeekOfYearLong();
-    self.testVectorUDFUnixTimeStampLong();
+    self.testVectorUDFYearTimestamp();
+    self.testVectorUDFMonthTimestamp();
+    self.testVectorUDFDayOfMonthTimestamp();
+    self.testVectorUDFHourTimestamp();
+    self.testVectorUDFWeekOfYearTimestamp();
+    self.testVectorUDFUnixTimeStampTimestamp();
+
     self.testVectorUDFYearString();
     self.testVectorUDFMonthString();
     self.testVectorUDFDayOfMonthString();

http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
index 0e23680..1e41fce 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
@@ -23,19 +23,29 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.math.RoundingMode;
+import java.sql.Timestamp;
 import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
 import junit.framework.Assert;
 
 import org.apache.hadoop.hive.common.type.Decimal128;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
 import org.junit.Test;
 
 /**
@@ -43,9 +53,6 @@ import org.junit.Test;
  */
 public class TestVectorTypeCasts {
 
-  // Number of nanoseconds in one second
-  private static final long NANOS_PER_SECOND = 1000000000;
-
   @Test
   public void testVectorCastLongToDouble() {
     VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInDoubleOut();
@@ -79,13 +86,13 @@ public class TestVectorTypeCasts {
 
   @Test
   public void testCastDoubleToTimestamp() {
-    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchDoubleInLongOut();
-    LongColumnVector resultV = (LongColumnVector) b.cols[1];
+    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchDoubleInTimestampOut();
+    TimestampColumnVector resultV = (TimestampColumnVector) b.cols[1];
     b.cols[0].noNulls = true;
-    VectorExpression expr = new CastDoubleToTimestampViaDoubleToLong(0, 1);
+    VectorExpression expr = new CastDoubleToTimestamp(0, 1);
     expr.evaluate(b);
-    Assert.assertEquals(0, resultV.vector[3]);
-    Assert.assertEquals((long) (0.5d * NANOS_PER_SECOND), resultV.vector[4]);
+    Assert.assertEquals(0.0, TimestampWritable.getDouble(resultV.asScratchTimestamp(3)));
+    Assert.assertEquals(0.5d, TimestampWritable.getDouble(resultV.asScratchTimestamp(4)));
   }
 
   @Test
@@ -103,39 +110,51 @@ public class TestVectorTypeCasts {
 
   @Test
   public void testCastLongToTimestamp() {
-    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInLongOut();
-    LongColumnVector resultV = (LongColumnVector) b.cols[1];
+    long[] longValues = new long[500];
+    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInTimestampOut(longValues);
+    TimestampColumnVector resultV = (TimestampColumnVector) b.cols[1];
     b.cols[0].noNulls = true;
-    VectorExpression expr = new CastLongToTimestampViaLongToLong(0, 1);
+    VectorExpression expr = new CastLongToTimestamp(0, 1);
     expr.evaluate(b);
-    Assert.assertEquals(-2 * NANOS_PER_SECOND, resultV.vector[0]);
-    Assert.assertEquals(2 * NANOS_PER_SECOND, resultV.vector[1]);
+    for (int i = 0; i < longValues.length; i++) {
+      Timestamp timestamp = resultV.asScratchTimestamp(i);
+      long actual = TimestampWritable.getLong(timestamp);
+      assertEquals(actual, longValues[i]);
+    }
   }
 
   @Test
   public void testCastTimestampToLong() {
-    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInLongOut();
-    LongColumnVector inV = (LongColumnVector) b.cols[0];
-    inV.vector[0] = NANOS_PER_SECOND;  // Make one entry produce interesting result
-      // (1 sec after epoch).
-
+    long[] longValues = new long[500];
+    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchTimestampInLongOut(longValues);
+    TimestampColumnVector inV = (TimestampColumnVector) b.cols[0];
     LongColumnVector resultV = (LongColumnVector) b.cols[1];
     b.cols[0].noNulls = true;
-    VectorExpression expr = new CastTimestampToLongViaLongToLong(0, 1);
+    VectorExpression expr = new CastTimestampToLong(0, 1);
     expr.evaluate(b);
-    Assert.assertEquals(1, resultV.vector[0]);
+    for (int i = 0; i < longValues.length; i++) {
+      long actual = resultV.vector[i];
+      long timestampLong = inV.getTimestampAsLong(i);
+      if (actual != timestampLong) {
+        assertTrue(false);
+      }
+    }
   }
 
   @Test
   public void testCastTimestampToDouble() {
-    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInDoubleOut();
-    LongColumnVector inV = (LongColumnVector) b.cols[0];
+    double[] doubleValues = new double[500];
+    VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchTimestampInDoubleOut(doubleValues);
+    TimestampColumnVector inV = (TimestampColumnVector) b.cols[0];
     DoubleColumnVector resultV = (DoubleColumnVector) b.cols[1];
     b.cols[0].noNulls = true;
-    VectorExpression expr = new CastTimestampToDoubleViaLongToDouble(0, 1);
+    VectorExpression expr = new CastTimestampToDouble(0, 1);
     expr.evaluate(b);
-    Assert.assertEquals(-1E-9D , resultV.vector[1]);
-    Assert.assertEquals(1E-9D, resultV.vector[3]);
+    for (int i = 0; i < doubleValues.length; i++) {
+      double actual = resultV.vector[i];
+      double doubleValue = TimestampWritable.getDouble(inV.asScratchTimestamp(i));
+      assertEquals(actual, doubleValue, 0.000000001F);
+    }
   }
 
   public byte[] toBytes(String s) {
@@ -356,13 +375,19 @@ public class TestVectorTypeCasts {
 
   @Test
   public void testCastDecimalToTimestamp() {
-    VectorizedRowBatch b = getBatchDecimalLong2();
+    double[] doubleValues = new double[500];
+    VectorizedRowBatch b = getBatchDecimalTimestamp(doubleValues);
     VectorExpression expr = new CastDecimalToTimestamp(0, 1);
     expr.evaluate(b);
-    LongColumnVector r = (LongColumnVector) b.cols[1];
-    assertEquals(1111111111L, r.vector[0]);
-    assertEquals(-2222222222L, r.vector[1]);
-    assertEquals(31536000999999999L, r.vector[2]);
+    TimestampColumnVector r = (TimestampColumnVector) b.cols[1];
+    for (int i = 0; i < doubleValues.length; i++) {
+      Timestamp timestamp = r.asScratchTimestamp(i);
+      double asDouble = TimestampWritable.getDouble(timestamp);
+      double expectedDouble = doubleValues[i];
+      if (expectedDouble != asDouble) {
+        assertTrue(false);
+      }
+    }
   }
 
   private VectorizedRowBatch getBatchDecimalLong2() {
@@ -381,6 +406,28 @@ public class TestVectorTypeCasts {
     return b;
   }
 
+  private VectorizedRowBatch getBatchDecimalTimestamp(double[] doubleValues) {
+    VectorizedRowBatch b = new VectorizedRowBatch(2);
+    DecimalColumnVector dv;
+    b.cols[0] = dv = new DecimalColumnVector(doubleValues.length, HiveDecimal.SYSTEM_DEFAULT_PRECISION, HiveDecimal.SYSTEM_DEFAULT_SCALE);
+    b.cols[1] = new TimestampColumnVector(doubleValues.length);
+    dv.noNulls = true;
+    Random r = new Random(94830);
+    for (int i = 0; i < doubleValues.length; i++) {
+      long millis = RandomTypeUtil.randomMillis(r);
+      Timestamp ts = new Timestamp(millis);
+      int nanos = RandomTypeUtil.randomNanos(r);
+      ts.setNanos(nanos);
+      TimestampWritable tsw = new TimestampWritable(ts);
+      double asDouble = tsw.getDouble();
+      doubleValues[i] = asDouble;
+      HiveDecimal hiveDecimal = HiveDecimal.create(new BigDecimal(asDouble));
+      dv.set(i, hiveDecimal);
+    }
+    b.size = doubleValues.length;
+    return b;
+  }
+
   @Test
   public void testCastLongToDecimal() {
     VectorizedRowBatch b = getBatchLongDecimal();
@@ -403,6 +450,47 @@ public class TestVectorTypeCasts {
     return b;
   }
 
+
+  public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
+  public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
+  public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1);
+
+  private VectorizedRowBatch getBatchTimestampDecimal(HiveDecimal[] hiveDecimalValues) {
+    Random r = new Random(994);
+    VectorizedRowBatch b = new VectorizedRowBatch(2);
+    TimestampColumnVector tcv;
+    b.cols[0] = tcv = new TimestampColumnVector(hiveDecimalValues.length);
+    b.cols[1] = new DecimalColumnVector(hiveDecimalValues.length, HiveDecimal.SYSTEM_DEFAULT_PRECISION, HiveDecimal.SYSTEM_DEFAULT_SCALE);
+    for (int i = 0; i < hiveDecimalValues.length; i++) {
+      int optionalNanos = 0;
+      switch (r.nextInt(4)) {
+      case 0:
+        // No nanos.
+        break;
+      case 1:
+        optionalNanos = r.nextInt((int) NANOSECONDS_PER_SECOND);
+        break;
+      case 2:
+        // Limit to milliseconds only...
+        optionalNanos = r.nextInt((int) MILLISECONDS_PER_SECOND) * (int) NANOSECONDS_PER_MILLISSECOND;
+        break;
+      case 3:
+        // Limit to below milliseconds only...
+        optionalNanos = r.nextInt((int) NANOSECONDS_PER_MILLISSECOND);
+        break;
+      }
+      long millis = RandomTypeUtil.randomMillis(r);
+      Timestamp ts = new Timestamp(millis);
+      ts.setNanos(optionalNanos);
+      TimestampWritable tsw = new TimestampWritable(ts);
+      hiveDecimalValues[i] = tsw.getHiveDecimal();
+
+      tcv.set(i, ts);
+    }
+    b.size = hiveDecimalValues.length;
+    return b;
+  }
+
   @Test
   public void testCastDoubleToDecimal() {
     VectorizedRowBatch b = getBatchDoubleDecimal();
@@ -410,9 +498,18 @@ public class TestVectorTypeCasts {
     expr.evaluate(b);
     DecimalColumnVector r = (DecimalColumnVector) b.cols[1];
 
-    assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("0.0")));
-    assertTrue(r.vector[1].getHiveDecimal().equals(HiveDecimal.create("-1.0")));
-    assertTrue(r.vector[2].getHiveDecimal().equals(HiveDecimal.create("99999999999999")));
+    HiveDecimal hd0 = HiveDecimal.create("0.0");
+    if (!hd0.equals(r.vector[0].getHiveDecimal())) {
+      assertTrue(false);
+    }
+    HiveDecimal hd1 = HiveDecimal.create("-1.0");
+    if (!hd1.equals(r.vector[1].getHiveDecimal())) {
+      assertTrue(false);
+    }
+    HiveDecimal hd2 = HiveDecimal.create("99999999999999");
+    if (!hd2.equals(r.vector[2].getHiveDecimal())) {
+      assertTrue(false);
+    }
   }
 
   private VectorizedRowBatch getBatchDoubleDecimal() {
@@ -466,25 +563,37 @@ public class TestVectorTypeCasts {
 
     // The input timestamps are stored as long values
     // measured in nanoseconds from the epoch.
-    VectorizedRowBatch b = getBatchLongDecimal();
+    HiveDecimal[] hiveDecimalValues = new HiveDecimal[500];
+    VectorizedRowBatch b = getBatchTimestampDecimal(hiveDecimalValues);
     VectorExpression expr = new CastTimestampToDecimal(0, 1);
-    LongColumnVector inL = (LongColumnVector) b.cols[0];
-    inL.vector[1] = -1990000000L;
+    TimestampColumnVector inT = (TimestampColumnVector) b.cols[0];
     expr.evaluate(b);
     DecimalColumnVector r = (DecimalColumnVector) b.cols[1];
-    assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("0.00")));
-    assertTrue(r.vector[1].getHiveDecimal().equals(HiveDecimal.create("-1.99")));
-    assertTrue(r.vector[2].getHiveDecimal().equals(HiveDecimal.create("100000.00")));
+    for (int i = 0; i < hiveDecimalValues.length; i++) {
+      HiveDecimal hiveDecimal = r.vector[i].getHiveDecimal();
+      HiveDecimal expectedHiveDecimal = hiveDecimalValues[i];
+      if (!hiveDecimal.equals(expectedHiveDecimal)) {
+        assertTrue(false);
+      }
+    }
 
     // Try again with a value that won't fit in 5 digits, to make
     // sure that NULL is produced.
-    b = getBatchLongDecimalPrec5Scale2();
+    b.cols[1] = r = new DecimalColumnVector(hiveDecimalValues.length, 5, 2);
     expr.evaluate(b);
     r = (DecimalColumnVector) b.cols[1];
-    assertFalse(r.noNulls);
-    assertFalse(r.isNull[0]);
-    assertFalse(r.isNull[1]);
-    assertTrue(r.isNull[2]);
+    for (int i = 0; i < hiveDecimalValues.length; i++) {
+      HiveDecimal hiveDecimal = r.vector[i].getHiveDecimal();
+      HiveDecimal expectedHiveDecimal = hiveDecimalValues[i];
+      if (HiveDecimal.enforcePrecisionScale(expectedHiveDecimal, 5, 2) == null) {
+        assertTrue(r.isNull[i]);
+      } else {
+        assertTrue(!r.isNull[i]);
+        if (!hiveDecimal.equals(expectedHiveDecimal)) {
+          assertTrue(false);
+        }
+      }
+    }
   }
 
   /* This batch has output decimal column precision 5 and scale 2.
@@ -503,30 +612,6 @@ public class TestVectorTypeCasts {
     return b;
   }
 
-  /*
-  @Test
-  public void testCastDecimalToDecimal() {
-
-    // test casting from one precision and scale to another.
-    VectorizedRowBatch b = getBatchDecimalDecimal();
-    VectorExpression expr = new CastDecimalToDecimal(0, 1);
-    expr.evaluate(b);
-    DecimalColumnVector r = (DecimalColumnVector) b.cols[1];
-    assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("10.00", (short) 2)));
-    assertFalse(r.noNulls);
-    assertTrue(r.isNull[1]);
-
-    // test an increase in precision/scale
-    b = getBatchDecimalDecimal();
-    expr = new CastDecimalToDecimal(1, 0);
-    expr.evaluate(b);
-    r = (DecimalColumnVector) b.cols[0];
-    assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("100.01", (short) 4)));
-    assertTrue(r.vector[1].getHiveDecimal().equals(HiveDecimal.create("-200.02", (short) 4)));
-    assertTrue(r.noNulls);
-  }
-  */
-
   private VectorizedRowBatch getBatchDecimalDecimal() {
     VectorizedRowBatch b = new VectorizedRowBatch(2);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
index bbda9a3..98849c3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 
@@ -101,16 +101,16 @@ public class FakeVectorRowBatchFromObjectIterables extends FakeVectorRowBatchBas
           }
         };
       } else if (types[i].equalsIgnoreCase("timestamp")) {
-        batch.cols[i] = new LongColumnVector(batchSize);
+        batch.cols[i] = new TimestampColumnVector(batchSize);
         columnAssign[i] = new ColumnVectorAssign() {
           @Override
           public void assign(
               ColumnVector columnVector,
               int row,
               Object value) {
-            LongColumnVector lcv = (LongColumnVector) columnVector;
+            TimestampColumnVector lcv = (TimestampColumnVector) columnVector;
             Timestamp t = (Timestamp) value;
-            lcv.vector[row] = TimestampUtils.getTimeNanoSec(t);
+            lcv.set(row, t);
           }
         };
 

http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
index 5d79f9c..84717b1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
@@ -18,13 +18,15 @@
 
 package org.apache.hadoop.hive.ql.exec.vector.util;
 
+import java.sql.Timestamp;
 import java.util.Random;
 
-import org.apache.hadoop.hive.common.type.Decimal128;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
 import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
@@ -77,6 +79,37 @@ public class VectorizedRowGroupGenUtil {
     return lcv;
   }
 
+  public static TimestampColumnVector generateTimestampColumnVector(
+      boolean nulls, boolean repeating, int size, Random rand, Timestamp[] timestampValues) {
+    TimestampColumnVector tcv = new TimestampColumnVector(size);
+
+    tcv.noNulls = !nulls;
+    tcv.isRepeating = repeating;
+
+    Timestamp repeatingTimestamp = RandomTypeUtil.getRandTimestamp(rand);
+
+    int nullFrequency = generateNullFrequency(rand);
+
+    for(int i = 0; i < size; i++) {
+      if(nulls && (repeating || i % nullFrequency == 0)) {
+        tcv.isNull[i] = true;
+        tcv.setNullValue(i);
+        timestampValues[i] = null;
+      }else {
+        tcv.isNull[i] = false;
+        if (!repeating) {
+          Timestamp randomTimestamp = RandomTypeUtil.getRandTimestamp(rand);
+          tcv.set(i,  randomTimestamp);
+          timestampValues[i] = randomTimestamp;
+        } else {
+          tcv.set(i, repeatingTimestamp);
+          timestampValues[i] = repeatingTimestamp;
+        }
+      }
+    }
+    return tcv;
+  }
+
   public static DoubleColumnVector generateDoubleColumnVector(boolean nulls,
       boolean repeating, int size, Random rand) {
     DoubleColumnVector dcv = new DoubleColumnVector(size);

http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index a345884..c0d912d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
 import org.apache.hadoop.hive.ql.io.AcidInputFormat;
@@ -1779,7 +1780,7 @@ public class TestInputOutputFormat {
     BytesColumnVector stringColumn = (BytesColumnVector) value.cols[7];
     DecimalColumnVector decimalColumn = (DecimalColumnVector) value.cols[8];
     LongColumnVector dateColumn = (LongColumnVector) value.cols[9];
-    LongColumnVector timestampColumn = (LongColumnVector) value.cols[10];
+    TimestampColumnVector timestampColumn = (TimestampColumnVector) value.cols[10];
     for(int i=0; i < 100; i++) {
       assertEquals("checking boolean " + i, i % 2 == 0 ? 1 : 0,
           booleanColumn.vector[i]);
@@ -1800,8 +1801,8 @@ public class TestInputOutputFormat {
       assertEquals("checking date " + i, i, dateColumn.vector[i]);
       long millis = (long) i * MILLIS_IN_DAY;
       millis -= LOCAL_TIMEZONE.getOffset(millis);
-      assertEquals("checking timestamp " + i, millis * 1000000L,
-          timestampColumn.vector[i]);
+      assertEquals("checking timestamp " + i, millis,
+          timestampColumn.getTime(i));
     }
     assertEquals(false, reader.next(key, value));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 137c39f..e526913 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -525,7 +525,10 @@ public class TestOrcFile {
     int idx = 0;
     while (rows.hasNext()) {
       Object row = rows.next(null);
-      assertEquals(tslist.get(idx++).getNanos(), ((TimestampWritable) row).getNanos());
+      Timestamp tlistTimestamp = tslist.get(idx++);
+      if (tlistTimestamp.getNanos() != ((TimestampWritable) row).getNanos()) {
+        assertTrue(false);
+      }
     }
     assertEquals(1, OrcUtils.getFlattenedColumnsCount(inspector));
     boolean[] expected = new boolean[] {false};


Mime
View raw message