hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [31/58] [abbrv] hive git commit: HIVE-11484 : Fix ObjectInspector for Char and VarChar for null and string objects (Deepak Barr, reviewed by Amareshwari)
Date Tue, 19 Apr 2016 22:25:23 GMT
HIVE-11484 : Fix ObjectInspector for Char and VarChar for null and string objects (Deepak Barr,
reviewed by Amareshwari)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddab69c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddab69c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddab69c4

Branch: refs/heads/llap
Commit: ddab69c495ce6c325d7e06cfb0c704546e0d56b9
Parents: 94c1974
Author: Deepak Barr <deepak.barr@gmail.com>
Authored: Wed Apr 13 09:50:24 2016 +0530
Committer: Amareshwari Sriramadasu <amareshwari@apache.org>
Committed: Wed Apr 13 09:50:24 2016 +0530

----------------------------------------------------------------------
 .../hadoop/hive/common/type/HiveBaseChar.java   |  6 ++
 .../hadoop/hive/common/type/HiveVarchar.java    |  4 +-
 .../hive/common/type/TestHiveBaseChar.java      |  2 +
 .../primitive/JavaHiveCharObjectInspector.java  | 15 +++-
 .../JavaHiveVarcharObjectInspector.java         | 15 +++-
 .../PrimitiveObjectInspectorConverter.java      |  8 +-
 .../TestObjectInspectorConverters.java          | 95 ++++++++++++++++++++
 .../TestStandardObjectInspectors.java           | 14 ++-
 8 files changed, 148 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
index 3514726..53684e7 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
@@ -38,6 +38,9 @@ public abstract class HiveBaseChar {
   }
 
   public static String enforceMaxLength(String val, int maxLength) {
+    if (val == null) {
+      return null;
+    }
     String value = val;
 
     if (maxLength > 0) {
@@ -52,6 +55,9 @@ public abstract class HiveBaseChar {
   }
 
   public static String getPaddedValue(String val, int maxLength) {
+    if (val == null) {
+      return null;
+    }
     if (maxLength < 0) {
       return val;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
index 969d474..09009eb 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
@@ -58,10 +58,10 @@ public class HiveVarchar extends HiveBaseChar
     return this.getValue().compareTo(rhs.getValue());
   }
 
-  public boolean equals(HiveVarchar rhs) {
+  public boolean equals(Object rhs) {
     if (rhs == this) {
       return true;
     }
-    return this.getValue().equals(rhs.getValue());
+    return this.getValue().equals(((HiveVarchar)rhs).getValue());
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
index 2848465..98ad74a 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
@@ -80,6 +80,7 @@ public class TestHiveBaseChar extends TestCase {
         assertEquals(strLen, enforcedString.codePointCount(0, enforcedString.length()));
       }
     }
+    assertNull(HiveBaseChar.enforceMaxLength(null, 0));
   }
 
   public void testGetPaddedValue() {
@@ -96,5 +97,6 @@ public class TestHiveBaseChar extends TestCase {
 
     assertEquals("abc       ", HiveBaseChar.getPaddedValue("abc", 10));
     assertEquals("abc       ", HiveBaseChar.getPaddedValue("abc ", 10));
+    assertNull(HiveBaseChar.getPaddedValue(null, 0));
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
index f429709..3d89c92 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
@@ -37,7 +37,12 @@ public class JavaHiveCharObjectInspector extends AbstractPrimitiveJavaObjectInsp
     if (o == null) {
       return null;
     }
-    HiveChar value = (HiveChar) o;
+    HiveChar value;
+    if (o instanceof String) {
+      value = new HiveChar((String) o, getMaxLength());
+    } else {
+      value = (HiveChar) o;
+    }
     if (BaseCharUtils.doesPrimitiveMatchTypeParams(value, (CharTypeInfo) typeInfo)) {
       return value;
     }
@@ -49,7 +54,13 @@ public class JavaHiveCharObjectInspector extends AbstractPrimitiveJavaObjectInsp
     if (o == null) {
       return null;
     }
-    return getWritableWithParams((HiveChar) o);
+    HiveChar var;
+    if (o instanceof String) {
+      var = new HiveChar((String) o, getMaxLength());
+    } else {
+      var = (HiveChar) o;
+    }
+    return getWritableWithParams(var);
   }
 
   private HiveChar getPrimitiveWithParams(HiveChar val) {

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
index a8e34ff..2320d2d 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
@@ -38,7 +38,12 @@ public class JavaHiveVarcharObjectInspector extends AbstractPrimitiveJavaObjectI
     if (o == null) {
       return null;
     }
-    HiveVarchar value = (HiveVarchar) o;
+    HiveVarchar value;
+    if (o instanceof String) {
+      value= new HiveVarchar((String)o, getMaxLength());
+    } else {
+      value = (HiveVarchar)o;
+    }
     if (BaseCharUtils.doesPrimitiveMatchTypeParams(value, (VarcharTypeInfo) typeInfo)) {
       return value;
     }
@@ -51,7 +56,13 @@ public class JavaHiveVarcharObjectInspector extends AbstractPrimitiveJavaObjectI
     if (o == null) {
       return null;
     }
-    return getWritableWithParams((HiveVarchar) o);
+    HiveVarchar var;
+    if (o instanceof String) {
+      var= new HiveVarchar((String)o, getMaxLength());
+    } else {
+      var = (HiveVarchar)o;
+    }
+    return getWritableWithParams(var);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
index 5eb41d5..e08ad43 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
@@ -514,7 +514,7 @@ public class PrimitiveObjectInspectorConverter {
 
     PrimitiveObjectInspector inputOI;
     SettableHiveVarcharObjectInspector outputOI;
-    HiveVarcharWritable hc;
+    Object hc;
 
     public HiveVarcharConverter(PrimitiveObjectInspector inputOI,
         SettableHiveVarcharObjectInspector outputOI) {
@@ -528,7 +528,7 @@ public class PrimitiveObjectInspectorConverter {
       //if (typeParams == null) {
       //  throw new RuntimeException("varchar type used without type params");
       //}
-      hc = new HiveVarcharWritable();
+      hc = outputOI.create(new HiveVarchar("",-1));
     }
 
     @Override
@@ -551,13 +551,13 @@ public class PrimitiveObjectInspectorConverter {
   public static class HiveCharConverter implements Converter {
     PrimitiveObjectInspector inputOI;
     SettableHiveCharObjectInspector outputOI;
-    HiveCharWritable hc;
+    Object hc;
 
     public HiveCharConverter(PrimitiveObjectInspector inputOI,
         SettableHiveCharObjectInspector outputOI) {
       this.inputOI = inputOI;
       this.outputOI = outputOI;
-      hc = new HiveCharWritable();
+      hc = outputOI.create(new HiveChar("",-1));
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
index dd18517..2e1bb22 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
@@ -22,10 +22,15 @@ import java.util.List;
 
 import junit.framework.TestCase;
 
+import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -116,6 +121,96 @@ public class TestObjectInspectorConverters extends TestCase {
           .convert(Integer.valueOf(1)));
       assertEquals("DoubleConverter", null, doubleConverter.convert(null));
 
+      // Char
+      Converter charConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+        PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
+      assertEquals("CharConverter", new HiveChar("TRUE", -1), charConverter
+        .convert(Boolean.valueOf(true)));
+      assertEquals("CharConverter", new HiveChar("FALSE", -1), charConverter
+        .convert(Boolean.valueOf(false)));
+
+      charConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+        PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
+      assertEquals("CharConverter", new HiveCharWritable(new HiveChar("TRUE", -1)), charConverter
+        .convert(Boolean.valueOf(true)));
+      assertEquals("CharConverter", new HiveCharWritable(new HiveChar("FALSE", -1)), charConverter
+        .convert(Boolean.valueOf(false)));
+
+      charConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+        PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
+      assertEquals("CharConverter", new HiveChar("0", -1), charConverter
+        .convert(Integer.valueOf(0)));
+      assertEquals("CharConverter", new HiveChar("1", -1), charConverter
+        .convert(Integer.valueOf(1)));
+
+      charConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+        PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
+      assertEquals("CharConverter", new HiveCharWritable(new HiveChar("0", -1)), charConverter
+        .convert(Integer.valueOf(0)));
+      assertEquals("CharConverter", new HiveCharWritable(new HiveChar("1", -1)), charConverter
+        .convert(Integer.valueOf(1)));
+
+      charConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+        PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
+      assertEquals("CharConverter", new HiveChar("hive", -1), charConverter
+        .convert(String.valueOf("hive")));
+
+      charConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+        PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
+      assertEquals("CharConverter", new HiveCharWritable(new HiveChar("hive", -1)), charConverter
+        .convert(String.valueOf("hive")));
+
+      // VarChar
+      Converter varcharConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+        PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
+      assertEquals("VarCharConverter", new HiveVarchar("TRUE", -1), varcharConverter
+        .convert(Boolean.valueOf(true)));
+      assertEquals("VarCharConverter", new HiveVarchar("FALSE", -1), varcharConverter
+        .convert(Boolean.valueOf(false)));
+
+      varcharConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+        PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
+      assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("TRUE", -1)),
varcharConverter
+        .convert(Boolean.valueOf(true)));
+      assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("FALSE", -1)),
varcharConverter
+        .convert(Boolean.valueOf(false)));
+
+      varcharConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+        PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
+      assertEquals("VarCharConverter", new HiveVarchar("0", -1), varcharConverter
+        .convert(Integer.valueOf(0)));
+      assertEquals("VarCharConverter", new HiveVarchar("1", -1), varcharConverter
+        .convert(Integer.valueOf(1)));
+
+      varcharConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+        PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
+      assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("0", -1)),
varcharConverter
+        .convert(Integer.valueOf(0)));
+      assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("1", -1)),
varcharConverter
+        .convert(Integer.valueOf(1)));
+
+      varcharConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+        PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
+      assertEquals("VarCharConverter", new HiveVarchar("hive", -1), varcharConverter
+        .convert(String.valueOf("hive")));
+
+      varcharConverter = ObjectInspectorConverters.getConverter(
+        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+        PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
+      assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("hive", -1)),
varcharConverter
+        .convert(String.valueOf("hive")));
+
       // Text
       Converter textConverter = ObjectInspectorConverters.getConverter(
           PrimitiveObjectInspectorFactory.javaIntObjectInspector,

http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
index 7d87666..29906c9 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
@@ -24,9 +24,13 @@ import java.util.List;
 
 import junit.framework.TestCase;
 
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -118,8 +122,14 @@ public class TestStandardObjectInspectors extends TestCase {
             .getClass());
       }
 
-      assertEquals(PrimitiveObjectInspectorUtils
+      if (javaClass == HiveVarchar.class) {
+        assertEquals("varchar(65535)", oi1.getTypeName());
+      } else if (javaClass == HiveChar.class) {
+        assertEquals("char(255)", oi1.getTypeName());
+      } else {
+        assertEquals(PrimitiveObjectInspectorUtils
           .getTypeNameFromPrimitiveJava(javaClass), oi1.getTypeName());
+      }
     } catch (Throwable e) {
       e.printStackTrace();
       throw e;
@@ -143,6 +153,8 @@ public class TestStandardObjectInspectors extends TestCase {
       doTestJavaPrimitiveObjectInspector(DoubleWritable.class, Double.class,
           (double) 1);
       doTestJavaPrimitiveObjectInspector(Text.class, String.class, "a");
+      doTestJavaPrimitiveObjectInspector(HiveVarcharWritable.class, HiveVarchar.class, "a");
+      doTestJavaPrimitiveObjectInspector(HiveCharWritable.class, HiveChar.class, "a");
       doTestJavaPrimitiveObjectInspector(BytesWritable.class, byte[].class, new byte[]{'3'});
 
     } catch (Throwable e) {


Mime
View raw message