hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1360812 - in /hive/trunk/serde/src: java/org/apache/hadoop/hive/serde2/lazy/ java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/ java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ test/org/apache/hadoop/hive/serde...
Date Thu, 12 Jul 2012 16:52:40 GMT
Author: hashutosh
Date: Thu Jul 12 16:52:40 2012
New Revision: 1360812

URL: http://svn.apache.org/viewvc?rev=1360812&view=rev
Log:
HIVE-3168: LazyBinaryObjectInspector.getPrimitiveJavaObject copies beyond length of underlying
BytesWritable (Thejas Nair via Ashutosh Chauhan)

Modified:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Thu Jul 12
16:52:40 2012
@@ -349,6 +349,20 @@ public final class LazyUtils {
     }
   }
 
+  /**
+   * Creates a ByteArrayRef with data from source BytesWritable
+   * @param sourceBw - source BytesWritable
+   */
+  public static ByteArrayRef createByteArrayRef(BytesWritable sourceBw){
+    //TODO should use BytesWritable.copyData() here once Hive
+    // removes support for the Hadoop 0.20 series.
+    byte[] newData = Arrays.copyOf(sourceBw.getBytes(), sourceBw.getLength());
+
+    ByteArrayRef bar = new ByteArrayRef();
+    bar.setData(newData);
+    return bar;
+  }
+
   private LazyUtils() {
     // prevent instantiation
   }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java
Thu Jul 12 16:52:40 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.la
 
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
 import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
 import org.apache.hadoop.io.BytesWritable;
@@ -42,9 +43,7 @@ public class LazyBinaryObjectInspector e
     if (null == o) {
       return null;
     }
-    ByteArrayRef ba = new ByteArrayRef();
-    ba.setData(((LazyBinary) o).getWritableObject().getBytes());
-    return ba;
+    return LazyUtils.createByteArrayRef(((LazyBinary) o).getWritableObject());
   }
 
   @Override

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java
Thu Jul 12 16:52:40 2012
@@ -1,6 +1,7 @@
 package org.apache.hadoop.hive.serde2.objectinspector.primitive;
 
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
 import org.apache.hadoop.io.BytesWritable;
 
 /**
@@ -63,9 +64,7 @@ public class JavaBinaryObjectInspector e
     if (null == bw){
       return null;
     }
-    ByteArrayRef ba = (ByteArrayRef)o;
-    ba.setData(bw.getBytes());
-    return ba;
+    return LazyUtils.createByteArrayRef(bw);
   }
 
   @Override
@@ -80,9 +79,7 @@ public class JavaBinaryObjectInspector e
     if(null == bw){
       return null;
     }
-    ByteArrayRef ba = new ByteArrayRef();
-    ba.setData(bw.getBytes());
-    return ba;
+    return LazyUtils.createByteArrayRef(bw);
   }
 
 }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java
Thu Jul 12 16:52:40 2012
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.serde2.objectinspector.primitive;
 
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
 import org.apache.hadoop.io.BytesWritable;
 
 public class WritableBinaryObjectInspector extends AbstractPrimitiveWritableObjectInspector
@@ -44,9 +45,7 @@ public class WritableBinaryObjectInspect
     if (null == o){
       return null;
     }
-    ByteArrayRef ba = new ByteArrayRef();
-    ba.setData(((BytesWritable)o).getBytes());
-    return ba;
+    return LazyUtils.createByteArrayRef((BytesWritable)o);
   }
 
   @Override

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
Thu Jul 12 16:52:40 2012
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.serde2.lazybinary;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -35,14 +36,22 @@ import org.apache.hadoop.hive.serde2.bin
 import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
 import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector;
 import org.apache.hadoop.io.BytesWritable;
 
 /**
@@ -53,7 +62,7 @@ public class TestLazyBinarySerDe extends
 
   /**
    * Generate a random struct array.
-   * 
+   *
    * @param r
    *          random number generator
    * @return an struct array
@@ -71,7 +80,7 @@ public class TestLazyBinarySerDe extends
 
   /**
    * Initialize the LazyBinarySerDe.
-   * 
+   *
    * @param fieldNames
    *          table field names
    * @param fieldTypes
@@ -91,7 +100,7 @@ public class TestLazyBinarySerDe extends
 
   /**
    * Test the LazyBinarySerDe.
-   * 
+   *
    * @param rows
    *          array of structs to be serialized
    * @param rowOI
@@ -134,7 +143,7 @@ public class TestLazyBinarySerDe extends
    * Compare two structs that have different number of fields. We just compare
    * the first few common fields, ignoring the fields existing in one struct but
    * not the other.
-   * 
+   *
    * @see ObjectInspectorUtils#compare(Object, ObjectInspector, Object,
    *      ObjectInspector)
    */
@@ -464,7 +473,7 @@ public class TestLazyBinarySerDe extends
 
   /**
    * The test entrance function.
-   * 
+   *
    * @throws Throwable
    */
   public void testLazyBinarySerDe() throws Throwable {
@@ -522,4 +531,78 @@ public class TestLazyBinarySerDe extends
       throw e;
     }
   }
+
+  private final  byte[] inpBArray = {'1','\u0001','3','4'};
+  private BytesWritable getInputBytesWritable() {
+    //create input BytesWritable. This would have capacity greater than length)
+    BytesWritable bW = new BytesWritable();
+    bW.set(inpBArray, 0, inpBArray.length);
+    return bW;
+  }
+
+  /**
+   * Test to see if ByteArrayRef with correct contents is generated by
+   * JavaBinaryObjectInspector from input BytesWritable
+   * @throws Throwable
+   */
+  public void testJavaBinaryObjectInspector() throws Throwable {
+    BytesWritable bW = getInputBytesWritable();
+
+    //create JavaBinaryObjectInspector
+    JavaBinaryObjectInspector binInspector =
+        PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector;
+
+    //convert BytesWritable to ByteArrayRef
+    ByteArrayRef outBARef = binInspector.set(null, bW);
+
+    assertTrue("compare input and output BAs",
+        Arrays.equals(inpBArray, outBARef.getData()));
+  }
+
+
+  /**
+   * Test to see if ByteArrayRef with correct contents is generated by
+   * WritableBinaryObjectInspector from input BytesWritable
+   * @throws Throwable
+   */
+  public void testWritableBinaryObjectInspector() throws Throwable {
+    BytesWritable bW = getInputBytesWritable();
+
+    //test WritableBinaryObjectInspector
+    WritableBinaryObjectInspector writableBinInsp =
+        PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+
+    //convert BytesWritable to ByteArrayRef
+    ByteArrayRef outBARef = writableBinInsp.getPrimitiveJavaObject(bW);
+
+    assertTrue("compare input and output BAs",
+        Arrays.equals(inpBArray, outBARef.getData()));
+  }
+
+  /**
+   * Test to see if ByteArrayRef with correct contents is generated by
+   * LazyBinaryObjectInspector from input BytesWritable
+   * @throws Throwable
+   */
+  public void testLazyBinaryObjectInspector() throws Throwable {
+
+    //create input ByteArrayRef
+    ByteArrayRef inpBARef = new ByteArrayRef();
+    inpBARef.setData(inpBArray);
+
+    AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory
+    .getLazyObjectInspector(PrimitiveCategory.BINARY, false, (byte)0);
+
+    //create LazyBinary initialed with inputBA
+    LazyBinary lazyBin = (LazyBinary) LazyFactory.createLazyObject(binInspector);
+    lazyBin.init(inpBARef, 0, inpBArray.length);
+
+    //use inspector to get a ByteArrayRef out of LazyBinary
+    ByteArrayRef outBARef = (ByteArrayRef) binInspector.getPrimitiveJavaObject(lazyBin);
+
+    assertTrue("compare input and output BAs",
+        Arrays.equals(inpBArray, outBARef.getData()));
+
+  }
+
 }



Mime
View raw message