Return-Path: Delivered-To: apmail-incubator-pig-commits-archive@locus.apache.org Received: (qmail 68442 invoked from network); 17 Nov 2007 00:22:10 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 17 Nov 2007 00:22:10 -0000 Received: (qmail 3074 invoked by uid 500); 17 Nov 2007 00:21:57 -0000 Delivered-To: apmail-incubator-pig-commits-archive@incubator.apache.org Received: (qmail 3054 invoked by uid 500); 17 Nov 2007 00:21:57 -0000 Mailing-List: contact pig-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: pig-dev@incubator.apache.org Delivered-To: mailing list pig-commits@incubator.apache.org Received: (qmail 3045 invoked by uid 99); 17 Nov 2007 00:21:57 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Nov 2007 16:21:57 -0800 X-ASF-Spam-Status: No, hits=-100.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 Nov 2007 00:21:55 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 46FE61A9832; Fri, 16 Nov 2007 16:21:49 -0800 (PST) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r595880 - in /incubator/pig/trunk: CHANGES.txt src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java Date: Sat, 17 Nov 2007 00:21:48 -0000 To: pig-commits@incubator.apache.org From: olga@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20071117002149.46FE61A9832@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: olga Date: Fri Nov 16 16:21:46 2007 New Revision: 595880 URL: http://svn.apache.org/viewvc?rev=595880&view=rev Log: added binary comparator; PIG-8 Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=595880&r1=595879&r2=595880&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Fri Nov 16 16:21:46 2007 @@ -18,4 +18,4 @@ PIG-23 Made pig work with java 1.5. (milindb via gates) - + PIG-8 added binary comparator (olgan) Modified: incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java?rev=595880&r1=595879&r2=595880&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java Fri Nov 16 16:21:46 2007 @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.TaskReport; import org.apache.hadoop.mapred.JobClient; @@ -63,6 +64,16 @@ numMRJobs = numMRJobsIn; mrJobNumber = 0; } + + public static class PigWritableComparator extends WritableComparator { + public PigWritableComparator() { + super(Tuple.class); + } + + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ + return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2); + } + } static Random rand = new Random(); @@ -136,7 +147,11 @@ conf.setCombinerClass(PigCombine.class); if (pom.quantilesFile!=null){ conf.set("pig.quantilesFile", pom.quantilesFile); - } + } + else{ + // this is not a sort job - can use byte comparison to speed up processing + conf.setOutputKeyComparatorClass(PigWritableComparator.class); + } if (pom.partitionFunction!=null){ conf.setPartitionerClass(SortPartitioner.class); }