hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From heyongqi...@apache.org
Subject svn commit: r1149027 - in /hive/trunk/serde/src: java/org/apache/hadoop/hive/serde2/objectinspector/ test/org/apache/hadoop/hive/serde2/objectinspector/
Date Thu, 21 Jul 2011 03:45:25 GMT
Author: heyongqiang
Date: Thu Jul 21 03:45:23 2011
New Revision: 1149027

URL: http://svn.apache.org/viewvc?rev=1149027&view=rev
Log:
HIVE-2209:add support for map comparision in serde layer (Krishna Kumar via He Yongqiang)

Added:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java
Modified:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java
(added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+/*
+ * The equality is implemented fully, the greater-than/less-than
+ * values do not implement a transitive relation. 
+ */
+
+public class CrossMapEqualComparer implements MapEqualComparer {
+  @Override
+  public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2)
{
+    int mapsize1 = moi1.getMapSize(o1);
+    int mapsize2 = moi2.getMapSize(o2);
+    if (mapsize1 != mapsize2) {
+      return mapsize1 - mapsize2;
+    }
+    ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector();
+    ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector();
+
+    ObjectInspector mvoi1 = moi1.getMapValueObjectInspector();
+    ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
+
+    Map<?, ?> map1 = moi1.getMap(o1);
+    Map<?, ?> map2 = moi2.getMap(o2);
+    for (Object mk1 : map1.keySet()) {
+      boolean notFound = true;
+      for (Object mk2 : map2.keySet()) {
+        int rc = ObjectInspectorUtils.compare(mk1, mkoi1, mk2, mkoi2, this);
+        if (rc != 0) {
+          continue;
+        }
+        notFound = false;
+        Object mv1 = map1.get(mk1);
+        Object mv2 = map2.get(mk2);
+        rc = ObjectInspectorUtils.compare(mv1, mvoi1, mv2, mvoi2, this);
+        if (rc != 0) {
+          return rc;
+        } else {
+          break;
+        }
+      }
+      if (notFound) {
+        return 1;
+      }
+    }
+    return 0;
+  }
+
+}

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java
(added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
+
+/*
+ * The equality is implemented fully, the implementation sorts the maps
+ * by their keys to provide a transitive compare. 
+ */
+
+public class FullMapEqualComparer implements MapEqualComparer {
+
+  private static class MapKeyComparator implements Comparator<Object> {
+
+    private ObjectInspector oi;
+
+    MapKeyComparator(ObjectInspector oi) {
+      this.oi = oi;
+    }
+
+    @Override
+    public int compare(Object o1, Object o2) {
+      return ObjectInspectorUtils.compare(o1, oi, o2, oi);
+    }
+  }
+
+  @Override
+  public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2)
{
+    int mapsize1 = moi1.getMapSize(o1);
+    int mapsize2 = moi2.getMapSize(o2);
+    if (mapsize1 != mapsize2) {
+      return mapsize1 - mapsize2;
+    }
+    ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector();
+    ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector();
+
+    ObjectInspector mvoi1 = moi1.getMapValueObjectInspector();
+    ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
+
+    Map<?, ?> map1 = moi1.getMap(o1);
+    Map<?, ?> map2 = moi2.getMap(o2);
+
+    Object[] sortedMapKeys1 = map1.keySet().toArray();
+    Arrays.sort(sortedMapKeys1, new MapKeyComparator(mkoi1));
+
+    Object[] sortedMapKeys2 = map2.keySet().toArray();
+    Arrays.sort(sortedMapKeys2, new MapKeyComparator(mkoi2));
+
+    for (int i = 0; i < mapsize1; ++i) {
+      Object mk1 = sortedMapKeys1[i];
+      Object mk2 = sortedMapKeys2[i];
+      int rc = ObjectInspectorUtils.compare(mk1, mkoi1, mk2, mkoi2, this);
+      if (rc != 0) {
+        return rc;
+      }
+      Object mv1 = map1.get(mk1);
+      Object mv2 = map2.get(mk2);
+      rc = ObjectInspectorUtils.compare(mv1, mvoi1, mv2, mvoi2, this);
+      if (rc != 0) {
+        return rc;
+      }
+    }
+    return 0;
+  }
+
+}

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java
(added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+public interface MapEqualComparer {
+  /*
+   * Compare the two map objects for equality.
+   */
+  public int compare(Object o1, MapObjectInspector moi1,
+      Object o2, MapObjectInspector moi2);
+}

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1149027&r1=1149026&r2=1149027&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
Thu Jul 21 03:45:23 2011
@@ -515,6 +515,14 @@ public final class ObjectInspectorUtils 
    */
   public static int compare(Object o1, ObjectInspector oi1, Object o2,
       ObjectInspector oi2) {
+    return compare(o1, oi1, o2, oi2, null);
+  }
+  
+  /**
+   * Compare two objects with their respective ObjectInspectors.
+   */
+  public static int compare(Object o1, ObjectInspector oi1, Object o2,
+      ObjectInspector oi2, MapEqualComparer mapEqualComparer) {
     if (oi1.getCategory() != oi2.getCategory()) {
       return oi1.getCategory().compareTo(oi2.getCategory());
     }
@@ -599,7 +607,8 @@ public final class ObjectInspectorUtils 
       for (int i = 0; i < minimum; i++) {
         int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1
             .get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2,
-            fields2.get(i)), fields2.get(i).getFieldObjectInspector());
+            fields2.get(i)), fields2.get(i).getFieldObjectInspector(), 
+            mapEqualComparer);
         if (r != 0) {
           return r;
         }
@@ -613,7 +622,8 @@ public final class ObjectInspectorUtils 
       for (int i = 0; i < minimum; i++) {
         int r = compare(loi1.getListElement(o1, i), loi1
             .getListElementObjectInspector(), loi2.getListElement(o2, i), loi2
-            .getListElementObjectInspector());
+            .getListElementObjectInspector(),
+            mapEqualComparer);
         if (r != 0) {
           return r;
         }
@@ -621,7 +631,11 @@ public final class ObjectInspectorUtils 
       return loi1.getListLength(o1) - loi2.getListLength(o2);
     }
     case MAP: {
-      throw new RuntimeException("Compare on map type not supported!");
+      if (mapEqualComparer == null) {
+        throw new RuntimeException("Compare on map type not supported!");
+      } else {
+        return mapEqualComparer.compare(o1, (MapObjectInspector)oi1, o2, (MapObjectInspector)oi2);
+      }
     }
     case UNION: {
       UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
@@ -633,7 +647,8 @@ public final class ObjectInspectorUtils 
       }
       return compare(uoi1.getField(o1),
           uoi1.getObjectInspectors().get(tag1),
-          uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2));
+          uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2),
+          mapEqualComparer);
     }
     default:
       throw new RuntimeException("Compare on unknown type: "

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java
(added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+
+/*
+ * Assumes that a getMapValueElement on object2 will work with a key from
+ * object1. The equality is implemented fully, the greater-than/less-than
+ * values do not implement a transitive relation. 
+ */
+public class SimpleMapEqualComparer implements MapEqualComparer {
+
+  @Override
+  public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2)
{
+    int mapsize1 = moi1.getMapSize(o1);
+    int mapsize2 = moi2.getMapSize(o2);
+    if (mapsize1 != mapsize2) {
+      return mapsize1 - mapsize2;
+    }
+    ObjectInspector mvoi1 = moi1.getMapValueObjectInspector();
+    ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
+    Map<?, ?> map1 = moi1.getMap(o1);
+    for (Object mk1: map1.keySet()) {
+      int rc = ObjectInspectorUtils.compare(moi1.getMapValueElement(o1, mk1), mvoi1, 
+          moi2.getMapValueElement(o2, mk1), mvoi2, this);
+      if (rc != 0) {
+        return rc;
+      }
+    }
+    return 0;
+  }
+}

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java
(added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class TestCrossMapEqualComparer extends TestCase {
+
+  public static class IntegerStringMapHolder {
+    Map<Integer, String> mMap;
+
+    public IntegerStringMapHolder() {
+      mMap = new TreeMap<Integer, String>();
+    }
+  }
+
+  public void testSameType() {
+    // empty maps
+    IntegerStringMapHolder o1 = new IntegerStringMapHolder();
+    IntegerStringMapHolder o2 = new IntegerStringMapHolder();
+    ObjectInspector oi1 = ObjectInspectorFactory
+        .getReflectionObjectInspector(IntegerStringMapHolder.class, ObjectInspectorOptions.JAVA);
+    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer());
+    assertEquals(0, rc);
+
+    // equal maps
+    o1.mMap.put(42, "The answer to Life, Universe And Everything");
+    o2.mMap.put(42, "The answer to Life, Universe And Everything");
+
+    o1.mMap.put(1729, "A taxi cab number");
+    o2.mMap.put(1729, "A taxi cab number");
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer());
+    assertEquals(0, rc);
+
+    // unequal maps
+    o2.mMap.put(1729, "Hardy-Ramanujan Number");
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer());
+    assertFalse(0 == rc);
+  }
+
+  public static class TextStringMapHolder {
+    Map<Text, String> mMap;
+
+    public TextStringMapHolder() {
+      mMap = new TreeMap<Text, String>();
+    }
+  }
+
+  Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1,
+      LazySimpleSerDe serde,
+      SerDeParameters serdeParams) throws IOException, SerDeException {
+    ByteStream.Output serializeStream = new ByteStream.Output();
+    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+        .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+        .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+        .getNeedsEscape());
+    Text t = new Text(serializeStream.toByteArray());
+    return serde.deserialize(t);
+  }
+
+  public void testCompatibleType() throws SerDeException, IOException {
+    // empty maps
+    TextStringMapHolder o1 = new TextStringMapHolder();
+    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);
+
+    LazySimpleSerDe serde = new LazySimpleSerDe();
+    Configuration conf = new Configuration();
+    Properties tbl = new Properties();
+    tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+    tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+        LazySimpleSerDe.class.getName());
+    serde.initialize(conf, tbl);
+    ObjectInspector oi2 = serde.getObjectInspector();
+
+    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+    assertEquals(0, rc);
+
+    // equal maps
+    o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything");
+    o1.mMap.put(new Text("1729"), "A taxi cab number");
+    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+    assertEquals(0, rc);
+
+    // unequal maps
+    o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number");
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+    assertFalse(0 == rc);
+  }
+
+  public static class StringTextMapHolder {
+    Map<String, Text> mMap;
+
+    public StringTextMapHolder() {
+      mMap = new TreeMap<String, Text>();
+    }
+  }
+
+  Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1,
+      LazySimpleSerDe serde,
+      SerDeParameters serdeParams) throws IOException, SerDeException {
+    ByteStream.Output serializeStream = new ByteStream.Output();
+    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+        .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+        .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+        .getNeedsEscape());
+    Text t = new Text(serializeStream.toByteArray());
+    return serde.deserialize(t);
+  }
+
+  public void testIncompatibleType() throws SerDeException, IOException {
+    // empty maps
+    StringTextMapHolder o1 = new StringTextMapHolder();
+    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);
+
+    LazySimpleSerDe serde = new LazySimpleSerDe();
+    Configuration conf = new Configuration();
+    Properties tbl = new Properties();
+    tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+    tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+        LazySimpleSerDe.class.getName());
+    serde.initialize(conf, tbl);
+    ObjectInspector oi2 = serde.getObjectInspector();
+
+    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+    assertEquals(0, rc);
+
+    // equal maps
+    o1.mMap.put("42", new Text("The answer to Life, Universe And Everything"));
+    o1.mMap.put("1729", new Text("A taxi cab number"));
+    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+    assertEquals(0, rc);
+
+    // unequal maps
+    o1.mMap.put("1729", new Text("Hardy-Ramanujan Number"));
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+    assertFalse(0 == rc);
+
+  }
+
+}

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java
(added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+
+public class TestFullMapEqualComparer extends TestCase {
+
+  public static class IntegerIntegerMapHolder {
+    Map<Integer, Integer> mMap;
+
+    public IntegerIntegerMapHolder() {
+      mMap = new TreeMap<Integer, Integer>();
+    }
+  }
+
+  public void testAntiSymmetry() {
+    IntegerIntegerMapHolder o1 = new IntegerIntegerMapHolder();
+    IntegerIntegerMapHolder o2 = new IntegerIntegerMapHolder();
+
+    ObjectInspector oi = ObjectInspectorFactory
+        .getReflectionObjectInspector(IntegerIntegerMapHolder.class, ObjectInspectorOptions.JAVA);
+
+    o1.mMap.put(1, 1);
+
+    o2.mMap.put(0, 99);
+
+    { // not anti-symmetric
+      int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new SimpleMapEqualComparer());
+      assertTrue(rc12 > 0);
+      int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new SimpleMapEqualComparer());
+      assertTrue(rc21 > 0);
+    }
+    { // not anti-symmetric
+      int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new CrossMapEqualComparer());
+      assertTrue(rc12 > 0);
+      int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new CrossMapEqualComparer());
+      assertTrue(rc21 > 0);
+    }
+    {// anti-symmetric
+      int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new FullMapEqualComparer());
+      assertTrue(rc12 > 0);
+      int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new FullMapEqualComparer());
+      assertTrue(rc21 < 0);
+    }
+
+  }
+
+  public void testTransitivity() {
+    IntegerIntegerMapHolder o1 = new IntegerIntegerMapHolder();
+    IntegerIntegerMapHolder o2 = new IntegerIntegerMapHolder();
+    IntegerIntegerMapHolder o3 = new IntegerIntegerMapHolder();
+
+    ObjectInspector oi = ObjectInspectorFactory
+        .getReflectionObjectInspector(IntegerIntegerMapHolder.class, ObjectInspectorOptions.JAVA);
+
+    o1.mMap.put(1, 1);
+    o1.mMap.put(99, 99);
+
+    o2.mMap.put(0, 99);
+    o2.mMap.put(99, 99);
+
+    o3.mMap.put(0, 1);
+    o3.mMap.put(1, 99);
+
+    { // non-transitive
+      int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new SimpleMapEqualComparer());
+      assertTrue(rc12 > 0);
+      int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new SimpleMapEqualComparer());
+      assertTrue(rc23 > 0);
+      int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new SimpleMapEqualComparer());
+      assertTrue(rc13 < 0);
+    }
+    { // non-transitive
+      int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new CrossMapEqualComparer());
+      assertTrue(rc12 > 0);
+      int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new CrossMapEqualComparer());
+      assertTrue(rc23 > 0);
+      int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new CrossMapEqualComparer());
+      assertTrue(rc13 < 0);
+    }
+    {// transitive
+      int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new FullMapEqualComparer());
+      assertTrue(rc12 > 0);
+      int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new FullMapEqualComparer());
+      assertTrue(rc23 > 0);
+      int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new FullMapEqualComparer());
+      assertTrue(rc13 > 0);
+    }
+  }
+
+}

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java
(added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java
Thu Jul 21 03:45:23 2011
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class TestSimpleMapEqualComparer extends TestCase {
+
+  public static class IntegerStringMapHolder {
+    Map<Integer, String> mMap;
+
+    public IntegerStringMapHolder() {
+      mMap = new TreeMap<Integer, String>();
+    }
+  }
+
+  public void testSameType() {
+    // empty maps
+    IntegerStringMapHolder o1 = new IntegerStringMapHolder();
+    IntegerStringMapHolder o2 = new IntegerStringMapHolder();
+    ObjectInspector oi1 = ObjectInspectorFactory
+        .getReflectionObjectInspector(IntegerStringMapHolder.class, ObjectInspectorOptions.JAVA);
+    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer());
+    assertEquals(0, rc);
+
+    // equal maps
+    o1.mMap.put(42, "The answer to Life, Universe And Everything");
+    o2.mMap.put(42, "The answer to Life, Universe And Everything");
+
+    o1.mMap.put(1729, "A taxi cab number");
+    o2.mMap.put(1729, "A taxi cab number");
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer());
+    assertEquals(0, rc);
+
+    // unequal maps
+    o2.mMap.put(1729, "Hardy-Ramanujan Number");
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer());
+    assertFalse(0 == rc);
+  }
+
+  public static class TextStringMapHolder {
+    Map<Text, String> mMap;
+
+    public TextStringMapHolder() {
+      mMap = new TreeMap<Text, String>();
+    }
+  }
+
+  Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1,
+      LazySimpleSerDe serde,
+      SerDeParameters serdeParams) throws IOException, SerDeException {
+    ByteStream.Output serializeStream = new ByteStream.Output();
+    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+        .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+        .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+        .getNeedsEscape());
+    Text t = new Text(serializeStream.toByteArray());
+    return serde.deserialize(t);
+  }
+
+  public void testCompatibleType() throws SerDeException, IOException {
+    // empty maps
+    TextStringMapHolder o1 = new TextStringMapHolder();
+    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);
+
+    LazySimpleSerDe serde = new LazySimpleSerDe();
+    Configuration conf = new Configuration();
+    Properties tbl = new Properties();
+    tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+    tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+        LazySimpleSerDe.class.getName());
+    serde.initialize(conf, tbl);
+    ObjectInspector oi2 = serde.getObjectInspector();
+
+    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+    assertEquals(0, rc);
+
+    // equal maps
+    o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything");
+    o1.mMap.put(new Text("1729"), "A taxi cab number");
+    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+    assertEquals(0, rc);
+
+    // unequal maps
+    o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number");
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+    assertFalse(0 == rc);
+  }
+
+  public static class StringTextMapHolder {
+    Map<String, Text> mMap;
+
+    public StringTextMapHolder() {
+      mMap = new TreeMap<String, Text>();
+    }
+  }
+
+  Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1,
+      LazySimpleSerDe serde,
+      SerDeParameters serdeParams) throws IOException, SerDeException {
+    ByteStream.Output serializeStream = new ByteStream.Output();
+    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+        .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+        .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+        .getNeedsEscape());
+    Text t = new Text(serializeStream.toByteArray());
+    return serde.deserialize(t);
+  }
+
+  public void testIncompatibleType() throws SerDeException, IOException {
+    // empty maps
+    StringTextMapHolder o1 = new StringTextMapHolder();
+    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);
+
+    LazySimpleSerDe serde = new LazySimpleSerDe();
+    Configuration conf = new Configuration();
+    Properties tbl = new Properties();
+    tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+    tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+        LazySimpleSerDe.class.getName());
+    serde.initialize(conf, tbl);
+    ObjectInspector oi2 = serde.getObjectInspector();
+
+    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+    assertEquals(0, rc);
+
+    // equal maps
+    o1.mMap.put("42", new Text("The answer to Life, Universe And Everything"));
+    o1.mMap.put("1729", new Text("A taxi cab number"));
+    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+    assertFalse(0 == rc);
+  }
+
+}



Mime
View raw message