pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r945254 - in /hadoop/pig/trunk: ./ src/org/apache/pig/builtin/ src/org/apache/pig/impl/util/ test/org/apache/pig/test/
Date Mon, 17 May 2010 17:24:41 GMT
Author: daijy
Date: Mon May 17 17:24:40 2010
New Revision: 945254

URL: http://svn.apache.org/viewvc?rev=945254&view=rev
Log:
PIG-566: Dump and store outputs do not match for PigStorage (azaroth via daijy)

Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
    hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon May 17 17:24:40 2010
@@ -64,6 +64,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-566: Dump and store outputs do not match for PigStorage (azaroth via daijy)
+
 PIG-1414: Problem with parameter substitution (rding)
 
 PIG-1407: Logging starts before being configured (azaroth via daijy)

Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java Mon May 17 17:24:40
2010
@@ -52,8 +52,10 @@ public class Utf8StorageConverter implem
     protected TupleFactory mTupleFactory = TupleFactory.getInstance();
     protected final Log mLog = LogFactory.getLog(getClass());
 
-    private Integer mMaxInt = Integer.valueOf(Integer.MAX_VALUE);
-    private Long mMaxLong = Long.valueOf(Long.MAX_VALUE);
+    private static final Integer mMaxInt = Integer.valueOf(Integer.MAX_VALUE);
+    private static final Integer mMinInt = Integer.valueOf(Integer.MIN_VALUE);
+    private static final Long mMaxLong = Long.valueOf(Long.MAX_VALUE);
+    private static final Long mMinLong = Long.valueOf(Long.MIN_VALUE);
     private static final int BUFFER_SIZE = 1024;
         
     public Utf8StorageConverter() {
@@ -328,14 +330,12 @@ public class Utf8StorageConverter implem
         if(b == null)
             return null;
         String s;
-        if(b.length > 0 && 
-           (b[b.length - 1] == 'F' || b[b.length - 1] == 'f') ){
+        if (b.length > 0 && (b[b.length - 1] == 'F' || b[b.length - 1] == 'f'))
{
             s = new String(b, 0, b.length - 1);
-        } 
-        else {
+        } else {
             s = new String(b);
         }
-        
+
         try {
             return Float.valueOf(s);
         } catch (NumberFormatException nfe) {
@@ -368,9 +368,10 @@ public class Utf8StorageConverter implem
             try {
                 Double d = Double.valueOf(s);
                 // Need to check for an overflow error
-                if (d.doubleValue() > mMaxInt.doubleValue() + 1.0) {
+                if (Double.compare(d.doubleValue(), mMaxInt.doubleValue() + 1) >= 0 ||
+                        Double.compare(d.doubleValue(), mMinInt.doubleValue() - 1) <=
0) {
                     LogUtils.warn(this, "Value " + d + " too large for integer", 
-                                PigWarning.TOO_LARGE_FOR_INT, mLog);
+                            PigWarning.TOO_LARGE_FOR_INT, mLog);
                     return null;
                 }
                 return Integer.valueOf(d.intValue());
@@ -385,18 +386,15 @@ public class Utf8StorageConverter implem
     }
 
     public Long bytesToLong(byte[] b) throws IOException {
-        if(b == null)
+        if (b == null)
             return null;
-
         String s;
-        if(b.length > 0  &&  
-           (b[b.length - 1] == 'L' || b[b.length - 1] == 'l') ){
+        if (b.length > 0 && (b[b.length - 1] == 'L' || b[b.length - 1] == 'l'))
{
             s = new String(b, 0, b.length - 1);
-        } 
-        else {
+        } else {
             s = new String(b);
         }
-
+        
         try {
             return Long.valueOf(s);
         } catch (NumberFormatException nfe) {
@@ -407,8 +405,9 @@ public class Utf8StorageConverter implem
             try {
                 Double d = Double.valueOf(s);
                 // Need to check for an overflow error
-                if (d.doubleValue() > mMaxLong.doubleValue() + 1.0) {
-                	LogUtils.warn(this, "Value " + d + " too large for integer", 
+                if (Double.compare(d.doubleValue(), mMaxLong.doubleValue() + 1) > 0 ||
+                        Double.compare(d.doubleValue(), mMinLong.doubleValue() - 1) <
0) {
+                	LogUtils.warn(this, "Value " + d + " too large for long", 
                 	            PigWarning.TOO_LARGE_FOR_INT, mLog);
                     return null;
                 }

Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java Mon May 17 17:24:40 2010
@@ -62,11 +62,6 @@ public class TupleFormat {
                     }
                     else {
                         sb.append(d.toString());
-                        if (d instanceof Long) {
-                            sb.append("L");
-                        } else if (d instanceof Float) {
-                            sb.append("F");
-                        }
                     }
                 } else {
                     sb.append("");

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java Mon May 17 17:24:40 2010
@@ -127,9 +127,9 @@ public class TestCombiner extends TestCa
         pig.registerQuery("C = foreach B generate flatten(group), COUNT($1);");
         Iterator<Tuple> resultIterator = pig.openIterator("C");
         Tuple tuple = resultIterator.next();
-        assertEquals("(a,b,2L)", tuple.toString());
+        assertEquals("(a,b,2)", tuple.toString());
         tuple = resultIterator.next();
-        assertEquals("(a,c,1L)", tuple.toString());
+        assertEquals("(a,c,1)", tuple.toString());
         
         return inputFileName;
     }
@@ -364,9 +364,9 @@ public class TestCombiner extends TestCa
                 "pig1\t20\t3.1" };
         
         String expected[] = {
-                "(pig1,75L,{(pig1,18,2.1),(pig1,18,2.1),(pig1,19,2.1),(pig1,20,3.1)})",
-                "(pig2,48L,{(pig2,24,3.3),(pig2,24,4.5)})",
-                "(pig5,45L,{(pig5,45,2.4)})"
+                "(pig1,75,{(pig1,18,2.1),(pig1,18,2.1),(pig1,19,2.1),(pig1,20,3.1)})",
+                "(pig2,48,{(pig2,24,3.3),(pig2,24,4.5)})",
+                "(pig5,45,{(pig5,45,2.4)})"
         };
 
         try {

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java Mon May 17 17:24:40 2010
@@ -40,7 +40,6 @@ import org.apache.pig.impl.util.Utils;
 import org.junit.Test;
 
 import junit.framework.TestCase;
-import junit.framework.AssertionFailedError;
 
 /**
  * Test class to test conversions from bytes to types
@@ -80,7 +79,7 @@ public class TestConversions extends Tes
     {
         // valid floats
         String[] a = {"1", "-2.345",  "12.12334567", "1.02e-2",".23344",
-		      "23.1234567897", "12312.33f", "002312.33F", "1.02e-2f", ""};
+		      "23.1234567897", "12312.33", "002312.33", "1.02e-2", ""};
 
         Float[] f = {1f, -2.345f,  12.12334567f, 1.02e-2f,.23344f, 23.1234567f, // 23.1234567f
is a truncation case
 		     12312.33f, 2312.33f, 1.02e-2f }; 
@@ -126,7 +125,7 @@ public class TestConversions extends Tes
     {
         // valid Longs
         String[] a = {"1", "-2345",  "123456789012345678", "1.1", "-23.45",
-		      "21345345l", "3422342L", ""};
+		      "21345345", "3422342", ""};
         Long[] la = {1L, -2345L, 123456789012345678L, 1L, -23L, 
 		     21345345L, 3422342L};
         
@@ -304,6 +303,7 @@ public class TestConversions extends Tes
         }
     }
     
+    @Test
     public void testBytesToComplexTypeMisc() throws IOException, ParseException {
         String s = "(a,b";
         Schema schema = Utils.getSchemaFromString("t:tuple(a:chararray, b:chararray)");
@@ -370,4 +370,48 @@ public class TestConversions extends Tes
         t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
         assertTrue(t==null);
     }
+    
+    @Test
+    public void testOverflow() throws IOException, ParseException {
+    	Schema schema;
+    	ResourceFieldSchema rfs;
+    	Tuple tuple, convertedTuple;
+        tuple = TupleFactory.getInstance().newTuple(1);
+    	
+        schema = Utils.getSchemaFromString("t:tuple(a:int)");
+        rfs = new ResourceSchema(schema).getFields()[0];
+
+        // long bigger than Integer.MAX_VALUE
+    	tuple.set(0, Integer.valueOf(Integer.MAX_VALUE).longValue() + 1);
+    	convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0),
convertedTuple.get(0));
+        
+        // long smaller than Integer.MIN_VALUE
+    	tuple.set(0, Integer.valueOf(Integer.MIN_VALUE).longValue() - 1);
+    	convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0),
convertedTuple.get(0));
+        
+        // double bigger than Integer.MAX_VALUE
+        tuple.set(0, Integer.valueOf(Integer.MAX_VALUE).doubleValue() + 1);
+        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0),
convertedTuple.get(0));
+        
+        // double smaller than Integer.MIN_VALUE
+        tuple.set(0, Integer.valueOf(Integer.MIN_VALUE).doubleValue() - 1);
+        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0),
convertedTuple.get(0));
+ 
+        schema = Utils.getSchemaFromString("t:tuple(a:long)");
+        rfs = new ResourceSchema(schema).getFields()[0];
+        
+        // double bigger than Long.MAX_VALUE
+        tuple.set(0, Long.valueOf(Long.MAX_VALUE).doubleValue() + 10000);
+        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+        assertNull("Invalid cast to long: " + tuple.get(0) + " -> " + convertedTuple.get(0),
convertedTuple.get(0));
+        
+        // double smaller than Long.MIN_VALUE
+        tuple.set(0, Long.valueOf(Long.MIN_VALUE).doubleValue() - 10000);
+        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+        assertNull("Invalid cast to long: " + tuple.get(0) + " -> " + convertedTuple.get(0),
convertedTuple.get(0));
+    }
 }

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java Mon May 17 17:24:40 2010
@@ -266,7 +266,7 @@ public class TestDataModel extends junit
     public void testTupleToString() throws Exception {
         Tuple t = giveMeOneOfEach();
 
-        assertEquals("toString", "((3,3.0F),{(4),(mary had a little lamb)},[hello#world,goodbye#all],42,5000000000L,3.1415927F,2.99792458E8,true,hello,goodbye,)",
t.toString());
+        assertEquals("toString", "((3,3.0),{(4),(mary had a little lamb)},[hello#world,goodbye#all],42,5000000000,3.1415927,2.99792458E8,true,hello,goodbye,)",
t.toString());
     }
 
     @Test

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Mon May 17 17:24:40 2010
@@ -441,9 +441,9 @@ public class TestSecondarySort extends T
         pigServer.registerQuery("D = foreach C { E = limit A 10; F = E.a1; G = DISTINCT F;
generate group, COUNT(G);};");
         Iterator<Tuple> iter = pigServer.openIterator("D");
         assertTrue(iter.hasNext());
-        assertTrue(iter.next().toString().equals("(1,2L)"));
+        assertTrue(iter.next().toString().equals("(1,2)"));
         assertTrue(iter.hasNext());
-        assertTrue(iter.next().toString().equals("(2,1L)"));
+        assertTrue(iter.next().toString().equals("(2,1)"));
         assertFalse(iter.hasNext());
         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
         Util.deleteFile(cluster, tmpFile2.getCanonicalPath());

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Mon May 17 17:24:40
2010
@@ -86,14 +86,14 @@ public class TestTextDataParser extends 
 
     @Test
     public void testLong() throws Exception{
-        String myLong = "1l";
+        String myLong = "1";
         Long l = ps.getLoadCaster().bytesToLong(myLong.getBytes());
         assertTrue(l.equals(1l));
     }
     
     @Test
     public void testFloat() throws Exception{
-        String myFloat = "0.1f";
+        String myFloat = "0.1";
         Float f = ps.getLoadCaster().bytesToFloat(myFloat.getBytes());
         assertTrue(f.equals(0.1f));
     }

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java Mon May 17 17:24:40 2010
@@ -52,7 +52,7 @@ public class TestTupleFormat extends Tes
             tuple.set(6, bag);
 
             assertEquals(
-                    "(12,[pig#scalability],,12L,1.2F,(innerTuple),{(innerTuple)})",
+                    "(12,[pig#scalability],,12,1.2,(innerTuple),{(innerTuple)})",
                     TupleFormat.format(tuple));
         } catch (ExecException e) {
             e.printStackTrace();



Mime
View raw message