pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From y...@apache.org
Subject svn commit: r1044075 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
Date Thu, 09 Dec 2010 18:38:51 GMT
Author: yanz
Date: Thu Dec  9 18:38:50 2010
New Revision: 1044075

URL: http://svn.apache.org/viewvc?rev=1044075&view=rev
Log:
PIG-1757: After split combination, the number of maps may vary slightly (yanz)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1044075&r1=1044074&r2=1044075&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Dec  9 18:38:50 2010
@@ -230,6 +230,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1757: After split combination, the number of maps may vary slightly (yanz)
+
 PIG-1760: Need to report progress in all databags (rding)
 
 PIG-1709: Skewed join use fewer reducer for extreme large key (daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java?rev=1044075&r1=1044074&r2=1044075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java Thu Dec
 9 18:38:50 2010
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Comparator;
 import java.util.Collections;
+import java.util.Arrays;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -375,6 +376,8 @@ public class MapRedUtil {
             } else {
                 ComparableSplit csplit = new ComparableSplit(split, comparableSplitId++);
                 String[] locations = split.getLocations();
+                // sort the locations to stabilize the number of maps: PIG-1757
+                Arrays.sort(locations);
                 HashSet<String> locationSeen = new HashSet<String>();
                 for (String location : locations)
                 {



Mime
View raw message