pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1605270 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ test/org/apache/pig/builtin/
Date Wed, 25 Jun 2014 05:41:57 GMT
Author: daijy
Date: Wed Jun 25 05:41:57 2014
New Revision: 1605270

URL: http://svn.apache.org/r1605270
Log:
PIG-4023: BigDec/Int sort is broken

Added:
    pig/trunk/test/org/apache/pig/builtin/TestBigTypeSort.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/BigDecimalWritable.java
    pig/trunk/src/org/apache/pig/backend/hadoop/BigIntegerWritable.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1605270&r1=1605269&r2=1605270&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 25 05:41:57 2014
@@ -40,6 +40,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-4023: BigDec/Int sort is broken (ahireanup via daijy)
+
 PIG-4003: Error is thrown by JobStats.getOutputSize() when storing to a Hive table (cheolsoo)
 
 PIG-4035: Fix CollectedGroup e2e tests for tez (daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/BigDecimalWritable.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/BigDecimalWritable.java?rev=1605270&r1=1605269&r2=1605270&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/BigDecimalWritable.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/BigDecimalWritable.java Wed Jun 25 05:41:57
2014
@@ -97,8 +97,8 @@ public class BigDecimalWritable implemen
         public int compare(byte[] b1, int s1, int l1,
                            byte[] b2, int s2, int l2) {
             try {
-                thisValue.readFields(new DataInputStream(new ByteArrayInputStream(b1)));
-                thatValue.readFields(new DataInputStream(new ByteArrayInputStream(b2)));
+                thisValue.readFields(new DataInputStream(new ByteArrayInputStream(b1,s1,l1)));
+                thatValue.readFields(new DataInputStream(new ByteArrayInputStream(b2,s2,l2)));
             } catch (IOException e) {
                 throw new RuntimeException("Unable to read field from byte array: " + e);
             }

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/BigIntegerWritable.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/BigIntegerWritable.java?rev=1605270&r1=1605269&r2=1605270&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/BigIntegerWritable.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/BigIntegerWritable.java Wed Jun 25 05:41:57
2014
@@ -97,8 +97,8 @@ public class BigIntegerWritable implemen
         public int compare(byte[] b1, int s1, int l1,
                            byte[] b2, int s2, int l2) {
             try {
-                thisValue.readFields(new DataInputStream(new ByteArrayInputStream(b1)));
-                thatValue.readFields(new DataInputStream(new ByteArrayInputStream(b2)));
+                thisValue.readFields(new DataInputStream(new ByteArrayInputStream(b1,s1,l1)));
+                thatValue.readFields(new DataInputStream(new ByteArrayInputStream(b2,s2,l2)));
             } catch (IOException e) {
                 throw new RuntimeException("Unable to read field from byte array: " + e);
             }

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java?rev=1605270&r1=1605269&r2=1605270&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigDecimalRawComparator.java
Wed Jun 25 05:41:57 2014
@@ -18,6 +18,7 @@
 package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer;
 
 import java.io.IOException;
+import java.math.BigDecimal;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -90,7 +91,7 @@ public class PigBigDecimalRawComparator 
 
         // If either are null, handle differently.
         if (!ndw1.isNull() && !ndw2.isNull()) {
-            rc = ((Double)ndw1.getValueAsPigType()).compareTo((Double)ndw2.getValueAsPigType());
+            rc = ((BigDecimal)ndw1.getValueAsPigType()).compareTo((BigDecimal)ndw2.getValueAsPigType());
         } else {
             // For sorting purposes two nulls are equal.
             if (ndw1.isNull() && ndw2.isNull()) rc = 0;

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java?rev=1605270&r1=1605269&r2=1605270&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigBigIntegerRawComparator.java
Wed Jun 25 05:41:57 2014
@@ -18,6 +18,7 @@
 package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer;
 
 import java.io.IOException;
+import java.math.BigInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -90,7 +91,7 @@ public class PigBigIntegerRawComparator 
 
         // If either are null, handle differently.
         if (!ndw1.isNull() && !ndw2.isNull()) {
-            rc = ((Double)ndw1.getValueAsPigType()).compareTo((Double)ndw2.getValueAsPigType());
+            rc = ((BigInteger)ndw1.getValueAsPigType()).compareTo((BigInteger)ndw2.getValueAsPigType());
         } else {
             // For sorting purposes two nulls are equal.
             if (ndw1.isNull() && ndw2.isNull()) rc = 0;

Added: pig/trunk/test/org/apache/pig/builtin/TestBigTypeSort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestBigTypeSort.java?rev=1605270&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestBigTypeSort.java (added)
+++ pig/trunk/test/org/apache/pig/builtin/TestBigTypeSort.java Wed Jun 25 05:41:57 2014
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.builtin;
+
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.tuple;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.builtin.mock.Storage.Data;
+import org.apache.pig.data.Tuple;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class TestBigTypeSort {
+	private static PigServer pigServer;
+
+	@Before
+	public void setUp() throws Exception {
+		pigServer = new PigServer(ExecType.LOCAL);
+	}
+
+	@Test
+	public void testBigTypeSort() throws Exception {
+		Data data = resetData(pigServer);
+
+		List<Tuple> bigtypes = Lists.newArrayList();
+		bigtypes.add(tuple("123456789012314", "123000456000789000123000456000789000123000456000789.123"));
+		bigtypes.add(tuple("123456789012334", "1.0E40"));
+		bigtypes.add(tuple("103456789012034", "123000456000780000123000456000789000123000456000789.113"));
+
+		data.set("bigtypes", bigtypes);
+
+		pigServer.registerQuery("A = LOAD 'bigtypes' USING mock.Storage() as (x:biginteger,y:bigdecimal);");
+		pigServer.registerQuery("B = ORDER A BY x ASC;");
+		pigServer.registerQuery("C = ORDER A BY y ASC;");
+
+		Iterator<Tuple> it = pigServer.openIterator("B");
+		assertTrue(it.hasNext());
+		assertEquals(new BigInteger("103456789012034"), it.next().get(0));
+		assertEquals(new BigInteger("123456789012314"), it.next().get(0));
+		assertEquals(new BigInteger("123456789012334"), it.next().get(0));
+		assertFalse(it.hasNext());
+
+		it = pigServer.openIterator("C");
+		assertTrue(it.hasNext());
+		assertEquals(new BigDecimal("1.0E40"), it.next().get(1));
+		assertEquals(new BigDecimal("123000456000780000123000456000789000123000456000789.113"),
it.next().get(1));
+		assertEquals(new BigDecimal("123000456000789000123000456000789000123000456000789.123"),
it.next().get(1));
+		assertFalse(it.hasNext());
+
+	}
+}



Mime
View raw message