hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject svn commit: r1674367 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/type/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/ serde/src/java/org/apache/ha...
Date Fri, 17 Apr 2015 17:50:15 GMT
Author: jdere
Date: Fri Apr 17 17:50:14 2015
New Revision: 1674367

URL: http://svn.apache.org/r1674367
Log:
HIVE-10270: Cannot use Decimal constants less than 0.1BD (Jason Dere, reviewed by Gunther
Hagleitner)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/decimal_precision2.q
    hive/trunk/ql/src/test/results/clientpositive/decimal_precision2.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java Fri Apr
17 17:50:14 2015
@@ -128,8 +128,25 @@ public class HiveDecimal implements Comp
     return bd.scale();
   }
 
+  /**
+   * Returns the number of digits (integer and fractional) in the number, which is equivalent
+   * to SQL decimal precision. Note that this is different from BigDecimal.precision(),
+   * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision()
= 1,
+   * whereas HiveDecimal.create("0.01").precision() = 2).
+   * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision()
+   * @return
+   */
   public int precision() {
-    return bd.precision();
+    int bdPrecision = bd.precision();
+    int bdScale = bd.scale();
+
+    if (bdPrecision < bdScale) {
+      // This can happen for numbers less than 0.1
+      // For 0.001234: bdPrecision=4, bdScale=6
+      // In this case, we'll set the type to have the same precision as the scale.
+      return bdScale;
+    }
+    return bdPrecision;
   }
 
   public int intValue() {

Added: hive/trunk/ql/src/test/queries/clientpositive/decimal_precision2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/decimal_precision2.q?rev=1674367&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/decimal_precision2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/decimal_precision2.q Fri Apr 17 17:50:14
2015
@@ -0,0 +1,15 @@
+
+explain select 100.001BD;
+
+explain select 100.000BD;
+
+explain select 0.000BD;
+
+explain select 0.100BD;
+
+explain select 0.010BD;
+
+explain select cast(0.010 as decimal(6,3));
+
+explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD;
+select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD;

Added: hive/trunk/ql/src/test/results/clientpositive/decimal_precision2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/decimal_precision2.q.out?rev=1674367&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/decimal_precision2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/decimal_precision2.q.out Fri Apr 17 17:50:14
2015
@@ -0,0 +1,163 @@
+PREHOOK: query: explain select 100.001BD
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 100.001BD
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 100.001 (type: decimal(6,3))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: explain select 100.000BD
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 100.000BD
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 100.000 (type: decimal(3,0))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: explain select 0.000BD
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 0.000BD
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 0.000 (type: decimal(1,0))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: explain select 0.100BD
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 0.100BD
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 0.100 (type: decimal(1,1))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: explain select 0.010BD
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 0.010BD
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 0.010 (type: decimal(2,2))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: explain select cast(0.010 as decimal(6,3))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select cast(0.010 as decimal(6,3))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 0.01 (type: decimal(6,3))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: _dummy_table
+          Row Limit Per Split: 1
+          Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+          Select Operator
+            expressions: 69.0212249755859375 (type: decimal(27,20))
+            outputColumnNames: _col0
+            Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+            ListSink
+
+PREHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+69.0212249755859375

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java
Fri Apr 17 17:50:14 2015
@@ -674,8 +674,7 @@ public class BinarySortableSerDe extends
       case SHORT: {
         ShortObjectInspector spoi = (ShortObjectInspector) poi;
         short v = spoi.get(o);
-        writeByte(buffer, (byte) ((v >> 8) ^ 0x80), invert);
-        writeByte(buffer, (byte) v, invert);
+        serializeShort(buffer, v, invert);
         return;
       }
       case INT: {
@@ -692,38 +691,12 @@ public class BinarySortableSerDe extends
       }
       case FLOAT: {
         FloatObjectInspector foi = (FloatObjectInspector) poi;
-        int v = Float.floatToIntBits(foi.get(o));
-        if ((v & (1 << 31)) != 0) {
-          // negative number, flip all bits
-          v = ~v;
-        } else {
-          // positive number, flip the first bit
-          v = v ^ (1 << 31);
-        }
-        writeByte(buffer, (byte) (v >> 24), invert);
-        writeByte(buffer, (byte) (v >> 16), invert);
-        writeByte(buffer, (byte) (v >> 8), invert);
-        writeByte(buffer, (byte) v, invert);
+        serializeFloat(buffer, foi.get(o), invert);
         return;
       }
       case DOUBLE: {
         DoubleObjectInspector doi = (DoubleObjectInspector) poi;
-        long v = Double.doubleToLongBits(doi.get(o));
-        if ((v & (1L << 63)) != 0) {
-          // negative number, flip all bits
-          v = ~v;
-        } else {
-          // positive number, flip the first bit
-          v = v ^ (1L << 63);
-        }
-        writeByte(buffer, (byte) (v >> 56), invert);
-        writeByte(buffer, (byte) (v >> 48), invert);
-        writeByte(buffer, (byte) (v >> 40), invert);
-        writeByte(buffer, (byte) (v >> 32), invert);
-        writeByte(buffer, (byte) (v >> 24), invert);
-        writeByte(buffer, (byte) (v >> 16), invert);
-        writeByte(buffer, (byte) (v >> 8), invert);
-        writeByte(buffer, (byte) v, invert);
+        serializeDouble(buffer, doi.get(o), invert);
         return;
       }
       case STRING: {
@@ -768,26 +741,19 @@ public class BinarySortableSerDe extends
       case TIMESTAMP: {
         TimestampObjectInspector toi = (TimestampObjectInspector) poi;
         TimestampWritable t = toi.getPrimitiveWritableObject(o);
-        byte[] data = t.getBinarySortable();
-        for (int i = 0; i < data.length; i++) {
-          writeByte(buffer, data[i], invert);
-        }
+        serializeTimestampWritable(buffer, t, invert);
         return;
       }
       case INTERVAL_YEAR_MONTH: {
         HiveIntervalYearMonthObjectInspector ioi = (HiveIntervalYearMonthObjectInspector)
poi;
         HiveIntervalYearMonth intervalYearMonth = ioi.getPrimitiveJavaObject(o);
-        int totalMonths = intervalYearMonth.getTotalMonths();
-        serializeInt(buffer, totalMonths, invert);
+        serializeHiveIntervalYearMonth(buffer, intervalYearMonth, invert);
         return;
       }
       case INTERVAL_DAY_TIME: {
         HiveIntervalDayTimeObjectInspector ioi = (HiveIntervalDayTimeObjectInspector) poi;
         HiveIntervalDayTime intervalDayTime = ioi.getPrimitiveJavaObject(o);
-        long totalSecs = intervalDayTime.getTotalSeconds();
-        int nanos = intervalDayTime.getNanos();
-        serializeLong(buffer, totalSecs, invert);
-        serializeInt(buffer, nanos, invert);
+        serializeHiveIntervalDayTime(buffer, intervalDayTime, invert);
         return;
       }
       case DECIMAL: {
@@ -805,29 +771,7 @@ public class BinarySortableSerDe extends
 
         HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi;
         HiveDecimal dec = boi.getPrimitiveJavaObject(o);
-
-        // get the sign of the big decimal
-        int sign = dec.compareTo(HiveDecimal.ZERO);
-
-        // we'll encode the absolute value (sign is separate)
-        dec = dec.abs();
-
-        // get the scale factor to turn big decimal into a decimal < 1
-        int factor = dec.precision() - dec.scale();
-        factor = sign == 1 ? factor : -factor;
-
-        // convert the absolute big decimal to string
-        dec.scaleByPowerOfTen(Math.abs(dec.scale()));
-        String digits = dec.unscaledValue().toString();
-
-        // finally write out the pieces (sign, scale, digits)
-        writeByte(buffer, (byte) ( sign + 1), invert);
-        writeByte(buffer, (byte) ((factor >> 24) ^ 0x80), invert);
-        writeByte(buffer, (byte) ( factor >> 16), invert);
-        writeByte(buffer, (byte) ( factor >> 8), invert);
-        writeByte(buffer, (byte)   factor, invert);
-        serializeBytes(buffer, digits.getBytes(decimalCharSet),
-            digits.length(), sign == -1 ? !invert : invert);
+        serializeHiveDecimal(buffer, dec, invert);
         return;
       }
 
@@ -918,6 +862,11 @@ public class BinarySortableSerDe extends
     writeByte(buffer, (byte) 0, invert);
   }
 
+  public static void serializeShort(ByteStream.Output buffer, short v, boolean invert) {
+    writeByte(buffer, (byte) ((v >> 8) ^ 0x80), invert);
+    writeByte(buffer, (byte) v, invert);
+  }
+
   public static void serializeInt(ByteStream.Output buffer, int v, boolean invert) {
     writeByte(buffer, (byte) ((v >> 24) ^ 0x80), invert);
     writeByte(buffer, (byte) (v >> 16), invert);
@@ -936,6 +885,88 @@ public class BinarySortableSerDe extends
     writeByte(buffer, (byte) v, invert);
   }
 
+  public static void serializeFloat(ByteStream.Output buffer, float vf, boolean invert) {
+    int v = Float.floatToIntBits(vf);
+    if ((v & (1 << 31)) != 0) {
+      // negative number, flip all bits
+      v = ~v;
+    } else {
+      // positive number, flip the first bit
+      v = v ^ (1 << 31);
+    }
+    writeByte(buffer, (byte) (v >> 24), invert);
+    writeByte(buffer, (byte) (v >> 16), invert);
+    writeByte(buffer, (byte) (v >> 8), invert);
+    writeByte(buffer, (byte) v, invert);
+  }
+
+  public static void serializeDouble(ByteStream.Output buffer, double vd, boolean invert)
{
+    long v = Double.doubleToLongBits(vd);
+    if ((v & (1L << 63)) != 0) {
+      // negative number, flip all bits
+      v = ~v;
+    } else {
+      // positive number, flip the first bit
+      v = v ^ (1L << 63);
+    }
+    writeByte(buffer, (byte) (v >> 56), invert);
+    writeByte(buffer, (byte) (v >> 48), invert);
+    writeByte(buffer, (byte) (v >> 40), invert);
+    writeByte(buffer, (byte) (v >> 32), invert);
+    writeByte(buffer, (byte) (v >> 24), invert);
+    writeByte(buffer, (byte) (v >> 16), invert);
+    writeByte(buffer, (byte) (v >> 8), invert);
+    writeByte(buffer, (byte) v, invert);
+  }
+
+  public static void serializeTimestampWritable(ByteStream.Output buffer, TimestampWritable
t, boolean invert) {
+    byte[] data = t.getBinarySortable();
+    for (int i = 0; i < data.length; i++) {
+      writeByte(buffer, data[i], invert);
+    }
+  }
+
+  public static void serializeHiveIntervalYearMonth(ByteStream.Output buffer,
+      HiveIntervalYearMonth intervalYearMonth, boolean invert) {
+    int totalMonths = intervalYearMonth.getTotalMonths();
+    serializeInt(buffer, totalMonths, invert);
+  }
+
+  public static void serializeHiveIntervalDayTime(ByteStream.Output buffer,
+      HiveIntervalDayTime intervalDayTime, boolean invert) {
+    long totalSecs = intervalDayTime.getTotalSeconds();
+    int nanos = intervalDayTime.getNanos();
+    serializeLong(buffer, totalSecs, invert);
+    serializeInt(buffer, nanos, invert);
+  }
+
+  public static void serializeHiveDecimal(ByteStream.Output buffer, HiveDecimal dec, boolean
invert) {
+    // get the sign of the big decimal
+    int sign = dec.compareTo(HiveDecimal.ZERO);
+
+    // we'll encode the absolute value (sign is separate)
+    dec = dec.abs();
+
+    // get the scale factor to turn big decimal into a decimal < 1
+    // This relies on the BigDecimal precision value, which as of HIVE-10270
+    // is now different from HiveDecimal.precision()
+    int factor = dec.bigDecimalValue().precision() - dec.bigDecimalValue().scale();
+    factor = sign == 1 ? factor : -factor;
+
+    // convert the absolute big decimal to string
+    dec.scaleByPowerOfTen(Math.abs(dec.scale()));
+    String digits = dec.unscaledValue().toString();
+
+    // finally write out the pieces (sign, scale, digits)
+    writeByte(buffer, (byte) ( sign + 1), invert);
+    writeByte(buffer, (byte) ((factor >> 24) ^ 0x80), invert);
+    writeByte(buffer, (byte) ( factor >> 16), invert);
+    writeByte(buffer, (byte) ( factor >> 8), invert);
+    writeByte(buffer, (byte)   factor, invert);
+    serializeBytes(buffer, digits.getBytes(decimalCharSet),
+        digits.length(), sign == -1 ? !invert : invert);
+  }
+
   @Override
   public SerDeStats getSerDeStats() {
     // no support for statistics

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java
Fri Apr 17 17:50:14 2015
@@ -147,8 +147,7 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    BinarySortableSerDe.writeByte(output, (byte) ((v >> 8) ^ 0x80), invert);
-    BinarySortableSerDe.writeByte(output, (byte) v, invert);
+    BinarySortableSerDe.serializeShort(output, v, invert);
   }
 
   /*
@@ -174,15 +173,7 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    BinarySortableSerDe.writeByte(output, (byte) ((v >> 56) ^ 0x80), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 48), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 40), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 32), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 24), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 16), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 8), invert);
-    BinarySortableSerDe.writeByte(output, (byte) v, invert);
-
+    BinarySortableSerDe.serializeLong(output, v, invert);
   }
 
   /*
@@ -195,18 +186,7 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    int v = Float.floatToIntBits(vf);
-    if ((v & (1 << 31)) != 0) {
-      // negative number, flip all bits
-      v = ~v;
-    } else {
-      // positive number, flip the first bit
-      v = v ^ (1 << 31);
-    }
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 24), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 16), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 8), invert);
-    BinarySortableSerDe.writeByte(output, (byte) v, invert);
+    BinarySortableSerDe.serializeFloat(output, vf, invert);
   }
 
   /*
@@ -219,22 +199,7 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    long v = Double.doubleToLongBits(vd);
-    if ((v & (1L << 63)) != 0) {
-      // negative number, flip all bits
-      v = ~v;
-    } else {
-      // positive number, flip the first bit
-      v = v ^ (1L << 63);
-    }
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 56), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 48), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 40), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 32), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 24), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 16), invert);
-    BinarySortableSerDe.writeByte(output, (byte) (v >> 8), invert);
-    BinarySortableSerDe.writeByte(output, (byte) v, invert);
+    BinarySortableSerDe.serializeDouble(output, vd, invert);
   }
 
   /*
@@ -341,10 +306,7 @@ public class BinarySortableSerializeWrit
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
     tempTimestampWritable.set(vt);
-    byte[] data = tempTimestampWritable.getBinarySortable();
-    for (int i = 0; i < data.length; i++) {
-      BinarySortableSerDe.writeByte(output, data[i], invert);
-    }
+    BinarySortableSerDe.serializeTimestampWritable(output, tempTimestampWritable, invert);
   }
 
   /*
@@ -357,8 +319,7 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    int totalMonths = viyt.getTotalMonths();
-    BinarySortableSerDe.serializeInt(output, totalMonths, invert);
+    BinarySortableSerDe.serializeHiveIntervalYearMonth(output, viyt, invert);
   }
 
   @Override
@@ -381,10 +342,7 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    long totalSecs = vidt.getTotalSeconds();
-    int nanos = vidt.getNanos();
-    BinarySortableSerDe.serializeLong(output, totalSecs, invert);
-    BinarySortableSerDe.serializeInt(output, nanos, invert);
+    BinarySortableSerDe.serializeHiveIntervalDayTime(output, vidt, invert);
   }
 
   @Override
@@ -410,39 +368,6 @@ public class BinarySortableSerializeWrit
     // This field is not a null.
     BinarySortableSerDe.writeByte(output, (byte) 1, invert);
 
-    // decimals are encoded in three pieces:
-    // sign: 1, 2 or 3 for smaller, equal or larger than 0 respectively
-    // factor: Number that indicates the amount of digits you have to move
-    // the decimal point left or right until the resulting number is smaller
-    // than zero but has something other than 0 as the first digit.
-    // digits: which is a string of all the digits in the decimal. If the number
-    // is negative the binary string will be inverted to get the correct ordering.
-    // Example: 0.00123
-    // Sign is 3 (bigger than 0)
-    // Factor is -2 (move decimal point 2 positions right)
-    // Digits are: 123
-
-    // get the sign of the big decimal
-    int sign = dec.compareTo(HiveDecimal.ZERO);
-
-    // we'll encode the absolute value (sign is separate)
-    dec = dec.abs();
-
-    // get the scale factor to turn big decimal into a decimal < 1
-    int factor = dec.precision() - dec.scale();
-    factor = sign == 1 ? factor : -factor;
-
-    // convert the absolute big decimal to string
-    dec.scaleByPowerOfTen(Math.abs(dec.scale()));
-    String digits = dec.unscaledValue().toString();
-
-    // finally write out the pieces (sign, scale, digits)
-    BinarySortableSerDe.writeByte(output, (byte) ( sign + 1), invert);
-    BinarySortableSerDe.writeByte(output, (byte) ((factor >> 24) ^ 0x80), invert);
-    BinarySortableSerDe.writeByte(output, (byte) ( factor >> 16), invert);
-    BinarySortableSerDe.writeByte(output, (byte) ( factor >> 8), invert);
-    BinarySortableSerDe.writeByte(output, (byte)   factor, invert);
-    BinarySortableSerDe.serializeBytes(output, digits.getBytes(BinarySortableSerDe.decimalCharSet),
-        digits.length(), sign == -1 ? !invert : invert);
+    BinarySortableSerDe.serializeHiveDecimal(output, dec, invert);
   }
 }
\ No newline at end of file

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java
Fri Apr 17 17:50:14 2015
@@ -96,4 +96,109 @@ public class MyTestClass {
       return result;
     }
 
+    public void nonRandomFill(int idx) {
+      myByte = (Byte) getNonRandValue(nrByte, idx);
+      myShort = (Short) getNonRandValue(nrShort, idx);
+      myInt = (Integer) getNonRandValue(nrInt, idx);
+      myLong = (Long) getNonRandValue(nrLong, idx);
+      myFloat = (Float) getNonRandValue(nrFloat, idx);
+      myDouble = (Double) getNonRandValue(nrDouble, idx);
+      myString = (String) getNonRandValue(nrString, idx);
+      myHiveChar = new HiveChar(myString, myString.length());
+      myHiveVarchar = new HiveVarchar(myString, myString.length());
+      myDecimal = (HiveDecimal) getNonRandValue(nrDecimal, idx);
+      myDate = (Date) getNonRandValue(nrDate, idx);
+      myIntervalYearMonth = (HiveIntervalYearMonth) getNonRandValue(nrIntervalYearMonth,
idx);
+      myIntervalDayTime = (HiveIntervalDayTime) getNonRandValue(nrIntervalDayTime, idx);
+      myStruct = null;
+      myList = null;
+    }
+
+    public static Object getNonRandValue(Object[] nrArray, int index) {
+      return nrArray[index % nrArray.length];
+    }
+
+    static Object[] nrByte = {
+        Byte.valueOf((byte) 1)
+    };
+
+    static Object[] nrShort = {
+        Short.valueOf((short) 1)
+    };
+
+    static Object[] nrInt = {
+        Integer.valueOf(1)
+    };
+
+    static Object[] nrLong = {
+        Long.valueOf(1)
+    };
+
+    static Object[] nrFloat = {
+        Float.valueOf(1.0f)
+    };
+
+    static Object[] nrDouble = {
+        Double.valueOf(1.0)
+    };
+
+    static Object[] nrDecimal = {
+        HiveDecimal.create("100"),
+        HiveDecimal.create("10"),
+        HiveDecimal.create("1"),
+        HiveDecimal.create("0"),
+        HiveDecimal.create("0.1"),
+        HiveDecimal.create("0.01"),
+        HiveDecimal.create("0.001"),
+        HiveDecimal.create("-100"),
+        HiveDecimal.create("-10"),
+        HiveDecimal.create("-1"),
+        HiveDecimal.create("-0.1"),
+        HiveDecimal.create("-0.01"),
+        HiveDecimal.create("-0.001"),
+        HiveDecimal.create("12345678900"),
+        HiveDecimal.create("1234567890"),
+        HiveDecimal.create("123456789"),
+        HiveDecimal.create("12345678.9"),
+        HiveDecimal.create("1234567.89"),
+        HiveDecimal.create("123456.789"),
+        HiveDecimal.create("12345.6789"),
+        HiveDecimal.create("1234.56789"),
+        HiveDecimal.create("123.456789"),
+        HiveDecimal.create("1.23456789"),
+        HiveDecimal.create("0.123456789"),
+        HiveDecimal.create("0.0123456789"),
+        HiveDecimal.create("0.00123456789"),
+        HiveDecimal.create("0.000123456789"),
+        HiveDecimal.create("-12345678900"),
+        HiveDecimal.create("-1234567890"),
+        HiveDecimal.create("-123456789"),
+        HiveDecimal.create("-12345678.9"),
+        HiveDecimal.create("-1234567.89"),
+        HiveDecimal.create("-123456.789"),
+        HiveDecimal.create("-12345.6789"),
+        HiveDecimal.create("-1234.56789"),
+        HiveDecimal.create("-123.456789"),
+        HiveDecimal.create("-1.23456789"),
+        HiveDecimal.create("-0.123456789"),
+        HiveDecimal.create("-0.0123456789"),
+        HiveDecimal.create("-0.00123456789"),
+        HiveDecimal.create("-0.000123456789"),
+    };
+
+    static Object[] nrString = {
+        "abcdefg"
+    };
+
+    static Object[] nrDate = {
+        Date.valueOf("2001-01-01")
+    };
+
+    static Object[] nrIntervalYearMonth = {
+      HiveIntervalYearMonth.valueOf("1-0")
+    };
+
+    static Object[] nrIntervalDayTime = {
+      HiveIntervalDayTime.valueOf("1 0:0:0")
+    };
 }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java
Fri Apr 17 17:50:14 2015
@@ -450,4 +450,24 @@ public class MyTestPrimitiveClass {
       StandardStructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames,
primitiveObjectInspectorList);
       return rowOI;
     }
+
+    public void nonRandomFill(int idx, ExtraTypeInfo extraTypeInfo) {
+      myByte = (Byte) MyTestClass.getNonRandValue(MyTestClass.nrByte, idx);
+      myShort = (Short) MyTestClass.getNonRandValue(MyTestClass.nrShort, idx);
+      myInt = (Integer) MyTestClass.getNonRandValue(MyTestClass.nrInt, idx);
+      myLong = (Long) MyTestClass.getNonRandValue(MyTestClass.nrLong, idx);
+      myFloat = (Float) MyTestClass.getNonRandValue(MyTestClass.nrFloat, idx);
+      myDouble = (Double) MyTestClass.getNonRandValue(MyTestClass.nrDouble, idx);
+      myString = (String) MyTestClass.getNonRandValue(MyTestClass.nrString, idx);
+      myHiveChar = new HiveChar(myString, myString.length());
+      extraTypeInfo.hiveCharMaxLength = myString.length();
+      myHiveVarchar = new HiveVarchar(myString, myString.length());
+      extraTypeInfo.hiveVarcharMaxLength = myString.length();
+      myDecimal = (HiveDecimal) MyTestClass.getNonRandValue(MyTestClass.nrDecimal, idx);
+      extraTypeInfo.precision = myDecimal.precision();
+      extraTypeInfo.scale = myDecimal.scale();
+      myDate = (Date) MyTestClass.getNonRandValue(MyTestClass.nrDate, idx);
+      myIntervalYearMonth = (HiveIntervalYearMonth) MyTestClass.getNonRandValue(MyTestClass.nrIntervalYearMonth,
idx);
+      myIntervalDayTime = (HiveIntervalDayTime) MyTestClass.getNonRandValue(MyTestClass.nrIntervalDayTime,
idx);
+    }
 }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
Fri Apr 17 17:50:14 2015
@@ -196,7 +196,18 @@ public class TestBinarySortableFast exte
       // Need a map because we sort.
       Map<Object, PrimitiveTypeInfo[]> primitiveTypeInfoMap = new HashMap<Object,
PrimitiveTypeInfo[]>();
 
-      for (int i = 0; i < num; i++) {
+      int i;
+      // First try non-random values
+      for (i = 0; i < MyTestClass.nrDecimal.length; i++) {
+        MyTestPrimitiveClass t = new MyTestPrimitiveClass();
+        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+        t.nonRandomFill(i, extraTypeInfo);
+        myTestPrimitiveClasses[i] = t;
+        PrimitiveTypeInfo[] primitiveTypeInfos = MyTestPrimitiveClass.getPrimitiveTypeInfos(extraTypeInfo);
+        primitiveTypeInfoMap.put(t, primitiveTypeInfos);
+      }
+
+      for ( ; i < num; i++) {
         int randField = r.nextInt(MyTestPrimitiveClass.primitiveCount);
         MyTestPrimitiveClass t = new MyTestPrimitiveClass();
         int field = 0;

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java?rev=1674367&r1=1674366&r2=1674367&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java
Fri Apr 17 17:50:14 2015
@@ -147,7 +147,15 @@ public class TestBinarySortableSerDe ext
       Random r = new Random(1234);
       MyTestClass rows[] = new MyTestClass[num];
 
-      for (int i = 0; i < num; i++) {
+      int i;
+      // First try non-random values
+      for (i = 0; i < MyTestClass.nrDecimal.length; i++) {
+        MyTestClass t = new MyTestClass();
+        t.nonRandomFill(i);
+        rows[i] = t;
+      }
+
+      for ( ; i < num; i++) {
         MyTestClass t = new MyTestClass();
         ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
         t.randomFill(r, extraTypeInfo);



Mime
View raw message