hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r644999 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/io/MapFile.java src/test/org/apache/hadoop/io/TestMapFile.java
Date Sat, 05 Apr 2008 00:10:04 GMT
Author: cutting
Date: Fri Apr  4 17:10:02 2008
New Revision: 644999

URL: http://svn.apache.org/viewvc?rev=644999&view=rev
Log:
HADOOP-3152.  Make index interval configurable for MapFileInputFormat.  Contributed by Rong-En
Fan.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/io/MapFile.java
    hadoop/core/trunk/src/test/org/apache/hadoop/io/TestMapFile.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=644999&r1=644998&r2=644999&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Apr  4 17:10:02 2008
@@ -221,6 +221,9 @@
     HADOOP-2841. Unwrap AccessControlException and FileNotFoundException
     from RemoteException for DFSClient. (shv)
 
+    HADOOP-3152.  Make index interval configuable when using
+    MapFileOutputFormat for map-reduce job.  (Rong-En Fan via cutting)
+
   OPTIMIZATIONS
 
     HADOOP-2790.  Fixed inefficient method hasSpeculativeTask by removing

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/io/MapFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/io/MapFile.java?rev=644999&r1=644998&r2=644999&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/io/MapFile.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/io/MapFile.java Fri Apr  4 17:10:02 2008
@@ -62,6 +62,7 @@
     private SequenceFile.Writer data;
     private SequenceFile.Writer index;
 
+    final private static String INDEX_INTERVAL = "io.map.index.interval";
     private int indexInterval = 128;
 
     private long size;
@@ -139,6 +140,8 @@
                   Progressable progress)
       throws IOException {
 
+      this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval);
+
       this.comparator = comparator;
       this.lastKey = comparator.newKey();
 
@@ -166,6 +169,13 @@
      * @see #getIndexInterval()
      */
     public void setIndexInterval(int interval) { indexInterval = interval; }
+
+    /** Sets the index interval and stores it in conf
+     * @see #getIndexInterval()
+     */
+    public static void setIndexInterval(Configuration conf, int interval) {
+      conf.setInt(INDEX_INTERVAL, interval);
+    }
 
     /** Close the map. */
     public synchronized void close() throws IOException {

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/io/TestMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/io/TestMapFile.java?rev=644999&r1=644998&r2=644999&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/io/TestMapFile.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/io/TestMapFile.java Fri Apr  4 17:10:02 2008
@@ -38,10 +38,12 @@
       getName() + ".mapfile"); 
     FileSystem fs = FileSystem.getLocal(conf);
     Path qualifiedDirName = fs.makeQualified(dirName);
+    // Make an index entry for each insertion.
+    MapFile.Writer.setIndexInterval(conf, 1);
     MapFile.Writer writer = new MapFile.Writer(conf, fs,
       qualifiedDirName.toString(), Text.class, Text.class);
-    // Make an index entry for each insertion.
-    writer.setIndexInterval(1);
+    // Assert that the index interval is 1
+    assertEquals(1, writer.getIndexInterval());
     // Add entries up to 100 in intervals of ten.
     final int FIRST_KEY = 10;
     for (int i = FIRST_KEY; i < 100; i += 10) {
@@ -81,4 +83,4 @@
     closest = (Text)reader.getClosest(key, value, true);
     assertNull(closest);
   }
-}
\ No newline at end of file
+}



Mime
View raw message