Return-Path: Delivered-To: apmail-lucene-hadoop-commits-archive@locus.apache.org Received: (qmail 29179 invoked from network); 31 Oct 2006 19:43:09 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 31 Oct 2006 19:43:09 -0000 Received: (qmail 70434 invoked by uid 500); 31 Oct 2006 19:43:20 -0000 Delivered-To: apmail-lucene-hadoop-commits-archive@lucene.apache.org Received: (qmail 70418 invoked by uid 500); 31 Oct 2006 19:43:20 -0000 Mailing-List: contact hadoop-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hadoop-dev@lucene.apache.org Delivered-To: mailing list hadoop-commits@lucene.apache.org Received: (qmail 70409 invoked by uid 99); 31 Oct 2006 19:43:20 -0000 Received: from herse.apache.org (HELO herse.apache.org) (140.211.11.133) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Oct 2006 11:43:20 -0800 X-ASF-Spam-Status: No, hits=0.6 required=10.0 tests=NO_REAL_NAME X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Oct 2006 11:43:06 -0800 Received: by eris.apache.org (Postfix, from userid 65534) id 4AA571A9846; Tue, 31 Oct 2006 11:42:43 -0800 (PST) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r469622 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/io/ArrayFile.java src/java/org/apache/hadoop/io/MapFile.java src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java Date: Tue, 31 Oct 2006 19:42:42 -0000 To: hadoop-commits@lucene.apache.org From: cutting@apache.org X-Mailer: svnmailer-1.1.0 Message-Id: <20061031194243.4AA571A9846@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cutting Date: Tue Oct 31 11:42:41 2006 New Revision: 469622 URL: http://svn.apache.org/viewvc?view=rev&rev=469622 Log: HADOOP-636. Add MapFile and ArrayFile constructors which accept a Progressable. Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ArrayFile.java lucene/hadoop/trunk/src/java/org/apache/hadoop/io/MapFile.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=469622&r1=469621&r2=469622 ============================================================================== --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Tue Oct 31 11:42:41 2006 @@ -82,6 +82,11 @@ 22. HADOOP-658. Update source file headers per Apache policy. (cutting) +23. HADOOP-636. Add MapFile & ArrayFile constructors which accept a + Progressable, and pass it down to SequenceFile. This permits + reduce tasks which use MapFile to still report progress while + writing blocks to the filesystem. (cutting) + Release 0.7.2 - 2006-10-18 Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ArrayFile.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ArrayFile.java?view=diff&rev=469622&r1=469621&r2=469622 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ArrayFile.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ArrayFile.java Tue Oct 31 11:42:41 2006 @@ -21,6 +21,9 @@ import java.io.*; import org.apache.hadoop.fs.*; import org.apache.hadoop.conf.*; +import org.apache.hadoop.util.*; +import org.apache.hadoop.io.SequenceFile.CompressionType; + /** A dense file-based mapping from integers to values. */ public class ArrayFile extends MapFile { @@ -31,9 +34,20 @@ public static class Writer extends MapFile.Writer { private LongWritable count = new LongWritable(0); - /** Create the named file for values of the named class. */ - public Writer(FileSystem fs, String file, Class valClass) throws IOException { + /** Create the named file for values of the named class. + * @deprecated specify {@link CompressionType} and {@link Progressable} + */ + public Writer(FileSystem fs, String file, Class valClass) + throws IOException { super(fs, file, LongWritable.class, valClass); + } + + /** Create the named file for values of the named class. */ + public Writer(Configuration conf, FileSystem fs, + String file, Class valClass, + CompressionType compress, Progressable progress) + throws IOException { + super(conf, fs, file, LongWritable.class, valClass, compress, progress); } /** Append a value to the file. */ Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/MapFile.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/MapFile.java?view=diff&rev=469622&r1=469621&r2=469622 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/MapFile.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/MapFile.java Tue Oct 31 11:42:41 2006 @@ -21,6 +21,7 @@ import java.io.*; import org.apache.hadoop.fs.*; import org.apache.hadoop.conf.*; +import org.apache.hadoop.util.Progressable; import org.apache.hadoop.io.SequenceFile.CompressionType; /** A file-based map from keys to values. @@ -73,7 +74,7 @@ } /** Create the named map for keys of the named class. - * @deprecated specify a {@link CompressionType} instead + * @deprecated specify {@link CompressionType} and {@link Progressable} */ public Writer(FileSystem fs, String dirName, Class keyClass, Class valClass, boolean compress) @@ -82,19 +83,30 @@ } /** Create the named map for keys of the named class. */ public Writer(Configuration conf, FileSystem fs, String dirName, + Class keyClass, Class valClass, + CompressionType compress, Progressable progress) + throws IOException { + this(conf,fs,dirName,WritableComparator.get(keyClass),valClass, + compress, progress); + } + + /** Create the named map for keys of the named class. */ + public Writer(Configuration conf, FileSystem fs, String dirName, Class keyClass, Class valClass, CompressionType compress) throws IOException { this(conf,fs,dirName,WritableComparator.get(keyClass),valClass,compress); } - /** Create the named map using the named key comparator. */ + /** Create the named map using the named key comparator. + * @deprecated specify {@link CompressionType} and {@link Progressable} + */ public Writer(FileSystem fs, String dirName, WritableComparator comparator, Class valClass) throws IOException { this(fs, dirName, comparator, valClass, false); } /** Create the named map using the named key comparator. - * @deprecated specify a {@link CompressionType} instead + * @deprecated specify {@link CompressionType} and {@link Progressable} */ public Writer(FileSystem fs, String dirName, WritableComparator comparator, Class valClass, @@ -105,11 +117,21 @@ compress ? CompressionType.RECORD : CompressionType.NONE); } - /** Create the named map using the named key comparator. */ + /** Create the named map using the named key comparator. + * @deprecated specify a {@link Progressable} + */ public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator, Class valClass, SequenceFile.CompressionType compress) throws IOException { + this(conf, fs, dirName, comparator, valClass, compress, null); + } + /** Create the named map using the named key comparator. */ + public Writer(Configuration conf, FileSystem fs, String dirName, + WritableComparator comparator, Class valClass, + SequenceFile.CompressionType compress, + Progressable progress) + throws IOException { this.comparator = comparator; this.lastKey = comparator.newKey(); @@ -123,10 +145,11 @@ Class keyClass = comparator.getKeyClass(); this.data = SequenceFile.createWriter - (fs,conf,dataFile,keyClass,valClass,compress); + (fs,conf,dataFile,keyClass,valClass,compress,progress); this.index = SequenceFile.createWriter - (fs,conf,indexFile,keyClass,LongWritable.class,CompressionType.BLOCK); + (fs, conf, indexFile, keyClass, LongWritable.class, + CompressionType.BLOCK, progress); } /** The number of entries that are added before an index entry is added.*/ Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java?view=diff&rev=469622&r1=469621&r2=469622 ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java Tue Oct 31 11:42:41 2006 @@ -46,7 +46,8 @@ new MapFile.Writer(job, fs, file.toString(), job.getMapOutputKeyClass(), job.getMapOutputValueClass(), - SequenceFile.getCompressionType(job)); + SequenceFile.getCompressionType(job), + progress); return new RecordWriter() {