pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jcove...@apache.org
Subject svn commit: r1366184 - in /pig/trunk: ./ src/org/apache/pig/data/ test/org/apache/pig/data/
Date Thu, 26 Jul 2012 20:24:17 GMT
Author: jcoveney
Date: Thu Jul 26 20:24:17 2012
New Revision: 1366184

URL: http://svn.apache.org/viewvc?rev=1366184&view=rev
Log:
PIG-2840: Fix SchemaTuple bugs (jcoveney)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/data/AppendableSchemaTuple.java
    pig/trunk/src/org/apache/pig/data/DefaultTuple.java
    pig/trunk/src/org/apache/pig/data/SchemaTuple.java
    pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java
    pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java
    pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jul 26 20:24:17 2012
@@ -208,6 +208,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2840: Fix SchemaTuple bugs (jcoveney)
+
 PIG-2842: TestNewPlanOperatorPlan fails when new Configuration() picks up a previous minicluster
conf file (julien)
 
 PIG-2827: Unwrap exception swallowing in TOP (haitao.yao via jcoveney)

Modified: pig/trunk/src/org/apache/pig/data/AppendableSchemaTuple.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/AppendableSchemaTuple.java?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/AppendableSchemaTuple.java (original)
+++ pig/trunk/src/org/apache/pig/data/AppendableSchemaTuple.java Thu Jul 26 20:24:17 2012
@@ -173,7 +173,16 @@ public abstract class AppendableSchemaTu
     }
 
     public int hashCode() {
-        return super.hashCode() + appendedFields.hashCode();
+        int hash = super.hashCode();
+        if (appendedFields == null) {
+            return hash;
+        }
+        for (Object o : appendedFields) {
+            if (o != null) {
+                hash = 31 * hash + o.hashCode();
+            }
+        }
+        return hash;
     }
 
     public void set(int fieldNum, Object val) throws ExecException {

Modified: pig/trunk/src/org/apache/pig/data/DefaultTuple.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/DefaultTuple.java?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/DefaultTuple.java (original)
+++ pig/trunk/src/org/apache/pig/data/DefaultTuple.java Thu Jul 26 20:24:17 2012
@@ -27,6 +27,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -36,7 +37,6 @@ import org.apache.pig.PigException;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.impl.io.NullableTuple;
 import org.apache.pig.impl.util.ObjectSerializer;
-import org.apache.pig.impl.util.TupleFormat;
 
 /**
  * A default implementation of Tuple. This class will be created by the DefaultTupleFactory.
@@ -458,7 +458,7 @@ public class DefaultTuple extends Abstra
 
     @Override
     public int hashCode() {
-        int hash = 1;
+        int hash = 17;
         for (Iterator<Object> it = mFields.iterator(); it.hasNext();) {
             Object o = it.next();
             if (o != null) {

Modified: pig/trunk/src/org/apache/pig/data/SchemaTuple.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/SchemaTuple.java?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/SchemaTuple.java (original)
+++ pig/trunk/src/org/apache/pig/data/SchemaTuple.java Thu Jul 26 20:24:17 2012
@@ -104,13 +104,14 @@ public abstract class SchemaTuple<T exte
     @SuppressWarnings("unchecked") //this is ok because we only cast to T after checking
     protected SchemaTuple<T> set(Tuple t, boolean checkType) throws ExecException {
         if (checkType) {
-            if (t.getClass() == getClass()) {
+            if (isSpecificSchemaTuple(t)) {
                 return setSpecific((T)t);
             }
 
-            if (t instanceof SchemaTuple<?>)
+            if (t instanceof SchemaTuple<?>) {
                 return set((SchemaTuple<?>)t, false);
         }
+        }
 
         return set(t.getAll());
     }
@@ -236,11 +237,13 @@ public abstract class SchemaTuple<T exte
         return l;
     }
 
+    public abstract boolean isSpecificSchemaTuple(Object o);
+
     //TODO also need to implement the raw comparator
     @SuppressWarnings("unchecked")
     @Override
     public int compareTo(Object other) {
-        if (getClass() == other.getClass()) {
+        if (isSpecificSchemaTuple(other)) {
             return compareToSpecific((T)other);
         }
 
@@ -249,7 +252,7 @@ public abstract class SchemaTuple<T exte
         }
 
         if (other instanceof Tuple) {
-            compareTo((Tuple)other, false);
+            return compareTo((Tuple)other, false);
         }
 
         return DataType.compare(this, other);
@@ -262,7 +265,7 @@ public abstract class SchemaTuple<T exte
     @SuppressWarnings("unchecked")
     protected int compareTo(Tuple t, boolean checkType) {
         if (checkType) {
-            if (getClass() == t.getClass()) {
+            if (isSpecificSchemaTuple(t)) {
                 return compareToSpecific((T)t);
             }
 
@@ -305,7 +308,7 @@ public abstract class SchemaTuple<T exte
 
     @SuppressWarnings("unchecked")
     protected int compareTo(SchemaTuple<?> t, boolean checkType) {
-        if (checkType && getClass() == t.getClass()) {
+        if (checkType && isSpecificSchemaTuple(t)) {
             return compareToSpecific((T)t);
         }
 
@@ -436,40 +439,40 @@ public abstract class SchemaTuple<T exte
     }
 
     protected int hashCodePiece(int hash, int v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + v;
+        return isNull ? hash : 31 * hash + v;
     }
 
     protected int hashCodePiece(int hash, long v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + (int)(v^(v>>>32));
+        return isNull ? hash : 31 * hash + (int)(v^(v>>>32));
     }
 
     protected int hashCodePiece(int hash, float v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + Float.floatToIntBits(v);
+        return isNull ? hash : 31 * hash + Float.floatToIntBits(v);
     }
 
     protected int hashCodePiece(int hash, double v, boolean isNull) {
         long v2 = Double.doubleToLongBits(v);
-        return isNull ? 0 : 31 * hash + (int)(v2^(v2>>>32));
+        return isNull ? hash : 31 * hash + (int)(v2^(v2>>>32));
     }
 
     protected int hashCodePiece(int hash, boolean v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + (v ? 1231 : 1237);
+        return isNull ? hash : 31 * hash + (v ? 1231 : 1237);
     }
 
     protected int hashCodePiece(int hash, byte[] v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + DataByteArray.hashCode(v);
+        return isNull ? hash : 31 * hash + DataByteArray.hashCode(v);
     }
 
     protected int hashCodePiece(int hash, String v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + v.hashCode();
+        return isNull ? hash : 31 * hash + v.hashCode();
     }
 
     protected int hashCodePiece(int hash, Tuple v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + v.hashCode();
+        return isNull ? hash : 31 * hash + v.hashCode();
     }
 
     protected int hashCodePiece(int hash, DataBag v, boolean isNull) {
-        return isNull ? 0 : 31 * hash + v.hashCode();
+        return isNull ? hash : 31 * hash + v.hashCode();
     }
 
     @Override

Modified: pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java (original)
+++ pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java Thu Jul 26 20:24:17 2012
@@ -22,19 +22,9 @@ import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
-import java.net.URI;
 import java.util.List;
-import java.util.Locale;
 import java.util.Queue;
 
-import javax.tools.Diagnostic;
-import javax.tools.DiagnosticCollector;
-import javax.tools.JavaCompiler;
-import javax.tools.JavaFileManager;
-import javax.tools.JavaFileObject;
-import javax.tools.SimpleJavaFileObject;
-import javax.tools.ToolProvider;
-
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -581,6 +571,21 @@ public class SchemaTupleClassGenerator {
         }
     }
 
+    static class IsSpecificSchemaTuple extends TypeInFunctionStringOut {
+        private int id;
+
+        public IsSpecificSchemaTuple(int id) {
+            this.id = id;
+        }
+
+        public void prepare() {
+            add("@Override");
+            add("public boolean isSpecificSchemaTuple(Object o) {");
+            add("    return o instanceof SchemaTuple_" + id + ";");
+            add("}");
+        }
+    }
+
     //this has to write the null state of all the fields, not just the null bytes, though
those
     //will have to be reconstructed
     static class WriteNullsString extends TypeInFunctionStringOut {
@@ -1011,6 +1016,7 @@ public class SchemaTupleClassGenerator {
             listOfFutureMethods.add(new CompareToString(id));
             listOfFutureMethods.add(new CompareToSpecificString(id, appendable));
             listOfFutureMethods.add(new SetEqualToSchemaTupleString(id));
+            listOfFutureMethods.add(new IsSpecificSchemaTuple(id));
             listOfFutureMethods.add(new TypeAwareSetString(DataType.INTEGER));
             listOfFutureMethods.add(new TypeAwareSetString(DataType.LONG));
             listOfFutureMethods.add(new TypeAwareSetString(DataType.FLOAT));

Modified: pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java (original)
+++ pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java Thu Jul 26 20:24:17 2012
@@ -66,7 +66,7 @@ public class SchemaTupleFrontend {
             return pr.getFirst();
         }
         if (!SchemaTupleFactory.isGeneratable(udfSchema)) {
-            LOG.warn("Given Schema is not generatable: " + udfSchema);
+            LOG.debug("Given Schema is not generatable: " + udfSchema);
             return -1;
         }
         int id = SchemaTupleClassGenerator.getNextGlobalClassIdentifier();

Modified: pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java?rev=1366184&r1=1366183&r2=1366184&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java (original)
+++ pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java Thu Jul 26 20:24:17 2012
@@ -316,6 +316,8 @@ public class TestSchemaTuple {
 
         assertEquals(udfSchema.size(), st.size());
 
+        testHashCodeEqualAsNormalTuple(tf);
+
         shouldAllBeNull(tf);
 
         copyThenCompare(tf);
@@ -595,4 +597,15 @@ public class TestSchemaTuple {
 
     }
 
+    public void testHashCodeEqualAsNormalTuple(SchemaTupleFactory tf) {
+        SchemaTuple<?> st = tf.newTuple();
+        Tuple t = TupleFactory.getInstance().newTuple();
+        for (Object o : st) {
+            t.append(o);
+        }
+        assertEquals(t.hashCode(), st.hashCode());
+        assertEquals(st, t);
+        assertEquals(t, st);
+    }
+
 }



Mime
View raw message