hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aihu...@apache.org
Subject hive git commit: HIVE-12406: HIVE-9500 introduced incompatible change to LazySimpleSerDe public interface (Aihua Xu, reviewed by Szehon Ho)
Date Mon, 23 Nov 2015 20:47:09 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 f1c1427e6 -> 8d4b600a4


HIVE-12406: HIVE-9500 introduced incompatible change to LazySimpleSerDe public interface (Aihua
Xu, reviewed by Szehon Ho)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d4b600a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d4b600a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d4b600a

Branch: refs/heads/branch-1
Commit: 8d4b600a4f42d54c42e38e54369a747d58222a8e
Parents: f1c1427
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Mon Nov 23 14:41:58 2015 -0500
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Nov 23 15:38:47 2015 -0500

----------------------------------------------------------------------
 .../hive/serde2/lazy/LazySerDeParameters.java   |  6 ++-
 .../hive/serde2/lazy/LazySimpleSerDe.java       | 38 +++++++++----
 .../hive/serde2/lazy/TestLazySimpleSerDe.java   | 56 +++++++++++++++++---
 3 files changed, 82 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
index 60d11a2..1e72c72 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.hive.serde2.lazy;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.SerDeException;
@@ -40,6 +40,8 @@ import org.apache.hive.common.util.HiveStringUtils;
  * SerDeParameters.
  *
  */
+@Public
+@Stable
 public class LazySerDeParameters implements LazyObjectInspectorParameters {
   public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};
   public static final String SERIALIZATION_EXTEND_NESTING_LEVELS

http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
index c373047..5674d15 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
@@ -19,18 +19,16 @@
 package org.apache.hadoop.hive.serde2.lazy;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe;
 import org.apache.hadoop.hive.serde2.ByteStream;
@@ -38,7 +36,6 @@ import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeSpec;
 import org.apache.hadoop.hive.serde2.SerDeStats;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
@@ -50,14 +47,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.io.BinaryComparable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
-import org.apache.hive.common.util.HiveStringUtils;
-
 
 /**
  * LazySimpleSerDe can be used to read the same data format as
@@ -69,6 +62,8 @@ import org.apache.hive.common.util.HiveStringUtils;
  * Also LazySimpleSerDe outputs typed columns instead of treating all columns as
  * String like MetadataTypedColumnsetSerDe.
  */
+@Public
+@Stable
 @SerDeSpec(schemaProps = {
     serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES,
     serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM,
@@ -423,4 +418,29 @@ public class LazySimpleSerDe extends AbstractEncodingAwareSerDe {
     Text text = (Text)blob;
     return SerDeUtils.transformTextToUTF8(text, this.charset);
   }
+
+  /**
+   * This method is deprecated and is only used for backward compatibility.
+   * Replaced by @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#LazySerDeParameters(Configuration,
Properties, String)
+   */
+  @Deprecated
+  public static SerDeParameters initSerdeParams(Configuration job,
+      Properties tbl,
+      String serdeName) throws SerDeException {
+    return new SerDeParameters(job, tbl, serdeName);
+  }
+
+  /**
+   * This class is deprecated and is only used for backward compatibility. Replace by
+   * @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters.
+   */
+  @Deprecated
+  public static class SerDeParameters extends org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters
{
+
+    public SerDeParameters(Configuration job,
+        Properties tbl,
+        String serdeName) throws SerDeException {
+      super(job, tbl, serdeName);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
index b11ce32..391edd4 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.lazy;
 import java.io.IOException;
 import java.util.List;
 import java.util.Properties;
+import java.util.Random;
 
 import junit.framework.TestCase;
 
@@ -29,15 +30,22 @@ import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.ByteStream;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.junit.Test;
 
 /**
  * TestLazySimpleSerDe.
@@ -81,8 +89,6 @@ public class TestLazySimpleSerDe extends TestCase {
     }
   }
 
-
-
   /**
    * Test the LazySimpleSerDe class with LastColumnTakesRest option.
    */
@@ -167,20 +173,56 @@ public class TestLazySimpleSerDe extends TestCase {
       throw e;
     }
   }
-  
-  Object serializeAndDeserialize(List<Integer> o1, StructObjectInspector oi1,
+
+  /**
+   * Tests the deprecated usage of SerDeParameters.
+   *
+   */
+  @Test
+  @SuppressWarnings("deprecation")
+  public void testSerDeParameters() throws SerDeException, IOException {
+    // Setup
+    LazySimpleSerDe serDe = new LazySimpleSerDe();
+    Configuration conf = new Configuration();
+
+    MyTestClass row = new MyTestClass();
+    ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+    row.randomFill(new Random(1234), extraTypeInfo);
+
+    StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(MyTestClass.class,
+            ObjectInspectorOptions.JAVA);
+
+    String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+    Properties schema = new Properties();
+    schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
+    schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
+
+    SerDeUtils.initializeSerDe(serDe, conf, schema, null);
+    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName");
+
+    // Test
+    LazyStruct data = (LazyStruct)serializeAndDeserialize(row, rowOI, serDe, serdeParams);
+    assertEquals((boolean)row.myBool, ((LazyBoolean)data.getField(0)).getWritableObject().get());
+    assertEquals((int)row.myInt, ((LazyInteger)data.getField(3)).getWritableObject().get());
+  }
+
+  private Object serializeAndDeserialize(Object row,
+      StructObjectInspector rowOI,
       LazySimpleSerDe serde,
       LazySerDeParameters serdeParams) throws IOException, SerDeException {
     ByteStream.Output serializeStream = new ByteStream.Output();
-    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+    LazySimpleSerDe.serialize(serializeStream, row, rowOI, serdeParams
         .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
         .isEscaped(), serdeParams.getEscapeChar(), serdeParams
         .getNeedsEscape());
+
     Text t = new Text(serializeStream.toByteArray());
     return serde.deserialize(t);
   }
-  
-  
+
   private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s,
       Object[] expectedFieldsData) throws SerDeException {
     // Get the row structure


Mime
View raw message