pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject svn commit: r595880 - in /incubator/pig/trunk: CHANGES.txt src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
Date Sat, 17 Nov 2007 00:21:48 GMT
Author: olga
Date: Fri Nov 16 16:21:46 2007
New Revision: 595880

URL: http://svn.apache.org/viewvc?rev=595880&view=rev
Log:
added binary comparator; PIG-8

Modified:
    incubator/pig/trunk/CHANGES.txt
    incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java

Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=595880&r1=595879&r2=595880&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Fri Nov 16 16:21:46 2007
@@ -18,4 +18,4 @@
 
 	PIG-23 Made pig work with java 1.5. (milindb via gates)
 
-
+	PIG-8 added binary comparator (olgan)

Modified: incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java?rev=595880&r1=595879&r2=595880&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java Fri Nov
16 16:21:46 2007
@@ -40,6 +40,7 @@
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.TaskReport;
 import org.apache.hadoop.mapred.JobClient;
@@ -63,6 +64,16 @@
         numMRJobs = numMRJobsIn;
         mrJobNumber = 0;
     }
+
+    public static class PigWritableComparator extends WritableComparator {
+        public PigWritableComparator() {
+            super(Tuple.class);
+        }
+
+        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){
+            return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+	    }
+    }
         
     static Random rand = new Random();
 
@@ -136,7 +147,11 @@
                 	conf.setCombinerClass(PigCombine.class);
             	if (pom.quantilesFile!=null){
             		conf.set("pig.quantilesFile", pom.quantilesFile);
-            	}
+                }
+                else{
+                    // this is not a sort job - can use byte comparison to speed up processing
+                    conf.setOutputKeyComparatorClass(PigWritableComparator.class);					
+                }
             	if (pom.partitionFunction!=null){
             		conf.setPartitionerClass(SortPartitioner.class);
             	}



Mime
View raw message