hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r751053 - in /hadoop/hbase/branches/0.19: CHANGES.txt src/java/org/apache/hadoop/hbase/io/MapFile.java
Date Fri, 06 Mar 2009 19:57:34 GMT
Author: stack
Date: Fri Mar  6 19:57:33 2009
New Revision: 751053

URL: http://svn.apache.org/viewvc?rev=751053&view=rev
Log:
HBASE-1229 Apply HADOOP-5369 to HBase MapFile

Modified:
    hadoop/hbase/branches/0.19/CHANGES.txt
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java

Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=751053&r1=751052&r2=751053&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Fri Mar  6 19:57:33 2009
@@ -38,6 +38,7 @@
                (Erik Holstad via Stack)
    HBASE-1240  Would be nice if RowResult could be comparable
                (Erik Holstad via Stack)
+   HBASE-1229  Apply HADOOP-5369 to HBase MapFile (Ben Maurer via Stack)
 
 Release 0.19.0
   INCOMPATIBLE CHANGES

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java?rev=751053&r1=751052&r2=751053&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java Fri Mar  6
19:57:33 2009
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hbase.io;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.io.*;
 
 import org.apache.commons.logging.Log;
@@ -82,6 +84,12 @@
     private DataOutputBuffer outBuf = new DataOutputBuffer();
     private WritableComparable lastKey;
 
+    /** What's the position (in bytes) we wrote when we got the last index */
+    private long lastIndexPos = 0;
+
+    /** What was size when we last wrote an index */
+    private long lastIndexKeyCount = 0;
+
 
     /** Create the named map for keys of the named class. */
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -198,10 +206,15 @@
       throws IOException {
 
       checkKey(key);
-      
-      if (size % indexInterval == 0) {            // add an index entry
-        position.set(data.getLength());           // point to current eof
+
+      long pos = data.getLength();      
+      // Only write an index if we've changed positions. In a block compressed
+      // file, this means we write an entry at the start of each block      
+      if (size >= lastIndexKeyCount + indexInterval && pos > lastIndexPos)
{
+        position.set(pos);                        // point to current eof
         index.append(key, position);
+        lastIndexPos = pos;
+        lastIndexKeyCount = size;
       }
 
       data.append(key, val);                      // append key/value to data
@@ -315,12 +328,14 @@
       if (this.keys != null)
         return;
       this.count = 0;
-      this.keys = new WritableComparable[1024];
       this.positions = new long[1024];
+
       try {
         int skip = INDEX_SKIP;
         LongWritable position = new LongWritable();
         WritableComparable lastKey = null;
+        long lastIndex = -1;
+        ArrayList<WritableComparable> keyBuilder = new ArrayList<WritableComparable>(1024);
         while (true) {
           WritableComparable k = comparator.newKey();
 
@@ -331,7 +346,6 @@
           if (lastKey != null && comparator.compare(lastKey, k) > 0)
             throw new IOException("key out of order: "+k+" after "+lastKey);
           lastKey = k;
-          
           if (skip > 0) {
             skip--;
             continue;                             // skip this entry
@@ -339,20 +353,23 @@
             skip = INDEX_SKIP;                    // reset skip
           }
 
-          if (count == keys.length) {                // time to grow arrays
-            int newLength = (keys.length*3)/2;
-            WritableComparable[] newKeys = new WritableComparable[newLength];
-            long[] newPositions = new long[newLength];
-            System.arraycopy(keys, 0, newKeys, 0, count);
-            System.arraycopy(positions, 0, newPositions, 0, count);
-            keys = newKeys;
-            positions = newPositions;
+	  // don't read an index that is the same as the previous one. Block
+	  // compressed map files used to do this (multiple entries would point
+	  // at the same block)
+	  if (position.get() == lastIndex)
+	    continue;
+
+          if (count == positions.length) {
+	    positions = Arrays.copyOf(positions, positions.length * 2);
           }
 
-          keys[count] = k;
+          keyBuilder.add(k);
           positions[count] = position.get();
           count++;
         }
+
+        this.keys = keyBuilder.toArray(new WritableComparable[count]);
+        positions = Arrays.copyOf(positions, count);
       } catch (EOFException e) {
         LOG.warn("Unexpected EOF reading " + index +
                               " at entry #" + count + ".  Ignoring.");



Mime
View raw message