hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1459466 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/io/hfile/ main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/metrics/ main/java/org/...
Date Thu, 21 Mar 2013 18:53:35 GMT
Author: liyin
Date: Thu Mar 21 18:53:35 2013
New Revision: 1459466

URL: http://svn.apache.org/r1459466
Log:
[HBASE-5032] Add Delete Column into the delete bloom filter

Author: adela

Summary:
https://issues.apache.org/jira/browse/HBASE-5032

Previously, the delete family bloom filter only contains the row key
which has the delete family. It helps us to avoid the top-row seek
operation.

This jira attempts to add the delete column into this delete bloom
filter as well (rename the delete family bloom filter as delete bloom
filter).

The motivation is to save seek ops for scan time-range queries if we
know there is no delete column for this row/column.
We can seek directly to the exact timestamp we are interested in,
instead of seeking to the latest timestamp and keep skipping to find
out whether there is any delete column before the interested timestamp.

Test Plan: write unit tests

Reviewers: kannan, liyintang, kranganathan

Reviewed By: liyintang

CC: erling, hbase-eng@

Differential Revision: https://phabricator.fb.com/D622271

Task ID: 849012, 1980066

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu Mar 21 18:53:35 2013
@@ -442,11 +442,6 @@ public final class HConstants {
    */
   public static final int WEEK_IN_SECONDS = 7 * 24 * 3600;
 
-  //TODO: HBASE_CLIENT_RETRIES_NUMBER_KEY is only used by TestMigrate. Move it
-  //      there.
-  public static final String HBASE_CLIENT_RETRIES_NUMBER_KEY =
-    "hbase.client.retries.number";
-
   //TODO: although the following are referenced widely to format strings for
   //      the shell. They really aren't a part of the public API. It would be
   //      nice if we could put them somewhere where they did not need to be
@@ -651,7 +646,17 @@ public final class HConstants {
    */
   public static  String COMPACTION_HOOK_JAR = "compaction_hook_jar";
 
+  public static final String GENERAL_BLOOM_FILTER = "general_bloom_filter";
+
+  public static final String DELETE_FAMILY_BLOOM_FILTER = "delete_family_bloom_filter";
+
+  public static final String DELETE_COLUMN_BLOOM_FILTER = "delete_column_bloom_filter";
+
+  public static final boolean ENABLE_DELETE_COLUMN_BLOOM_FILTER = false;
+
+  public static final String ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING = "enable_delete_column_bloom_filter";
+
   private HConstants() {
-    // Can't be instantiated with this ctor.
+    // Can't be instantiated with this constructor.
   }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java Thu Mar 21 18:53:35 2013
@@ -989,6 +989,13 @@ public class KeyValue implements Writabl
   }
 
   /**
+   * @return True if this KV is a delete column type.
+   */
+  public boolean isDeleteColumn() {
+    return getType() == Type.DeleteColumn.getCode();
+  }
+
+  /**
    *
    * @return True if this KV is a delete family or column type.
    */

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java Thu Mar 21 18:53:35 2013
@@ -75,6 +75,9 @@ public enum BlockType {
   /** Delete Family Bloom filter metadata, version 2 */
   DELETE_FAMILY_BLOOM_META("DFBLMET2", BlockCategory.BLOOM),
 
+  /** Delete Column Bloom filter metadata, version 2 */
+  DELETE_COLUMN_BLOOM_META("DCBLMET2", BlockCategory.BLOOM),
+
   // Trailer
 
   /** Fixed file trailer, both versions (always just a magic string) */

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Thu Mar 21 18:53:35 2013
@@ -252,6 +252,13 @@ public class HFile {
      * HFile V2.
      */
     void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
+
+    /**
+     * Store delete column Bloom filter in the file, which is only supported in
+     * HFile V2.
+     * @throws IOException
+     */
+    void addDeleteColumnBloomFilter(BloomFilterWriter bfw) throws IOException;
   }
 
   /**
@@ -447,6 +454,14 @@ public class HFile {
      */
     DataInput getDeleteBloomFilterMetadata() throws IOException;
 
+    /**
+     * Retrieves delete column Bloom filter metadata as appropriate for each
+     * {@link HFile} version.
+     * Knows nothing about how metadata is structured.
+     * @throws IOException
+     */
+    DataInput getDeleteColumnBloomFilterMetadata() throws IOException;
+
     Path getPath();
 
     /** Close method with optional evictOnClose */

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java Thu Mar 21 18:53:35 2013
@@ -38,9 +38,11 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.BloomFilter;
@@ -370,34 +372,36 @@ public class HFilePrettyPrinter {
     }
     System.out.println("Mid-key: " + Bytes.toStringBinary(reader.midkey()));
 
-    // Printing general bloom information
-    DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
-    BloomFilter bloomFilter = null;
-    if (bloomMeta != null)
-      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
-
-    System.out.println("Bloom filter:");
-    if (bloomFilter != null) {
-      System.out.println(FOUR_SPACES
-          + bloomFilter.toString().replaceAll(ByteBloomFilter.STATS_RECORD_SEP,
-              "\n" + FOUR_SPACES));
-    } else {
-      System.out.println(FOUR_SPACES + "Not present");
-    }
-
-    // Printing delete bloom information
-    bloomMeta = reader.getDeleteBloomFilterMetadata();
-    bloomFilter = null;
-    if (bloomMeta != null)
-      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
-
-    System.out.println("Delete Family Bloom filter:");
-    if (bloomFilter != null) {
-      System.out.println(FOUR_SPACES
-          + bloomFilter.toString().replaceAll(ByteBloomFilter.STATS_RECORD_SEP,
-              "\n" + FOUR_SPACES));
-    } else {
-      System.out.println(FOUR_SPACES + "Not present");
+    // Printing bloom filter information
+    String[] bloomFilters = new String[] {HConstants.GENERAL_BLOOM_FILTER,
+        HConstants.DELETE_FAMILY_BLOOM_FILTER, HConstants.DELETE_COLUMN_BLOOM_FILTER};
+
+    for (String blf : bloomFilters) {
+      DataInput bloomMeta = getCustomBloomFilterMetadata(blf, reader);
+      BloomFilter bloomFilter = null;
+      if (bloomMeta != null)
+        bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
+
+      System.out.println("Bloom filter: " + blf);
+      if (bloomFilter != null) {
+        System.out.println(FOUR_SPACES
+            + bloomFilter.toString().replaceAll(
+                ByteBloomFilter.STATS_RECORD_SEP, "\n" + FOUR_SPACES));
+      } else {
+        System.out.println(FOUR_SPACES + "Not present");
+      }
+    }
+  }
+
+  private DataInput getCustomBloomFilterMetadata(String bloomFilter,
+      Reader reader) throws IOException {
+    if (bloomFilter.equals(HConstants.GENERAL_BLOOM_FILTER)) {
+      return reader.getGeneralBloomFilterMetadata();
+    } else if (bloomFilter.equals(HConstants.DELETE_FAMILY_BLOOM_FILTER)) {
+      return reader.getDeleteBloomFilterMetadata();
+    } else if (bloomFilter.equals(HConstants.DELETE_COLUMN_BLOOM_FILTER)) {
+      return reader.getDeleteColumnBloomFilterMetadata();
     }
+    return null;
   }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Thu Mar 21 18:53:35 2013
@@ -701,4 +701,9 @@ public class HFileReaderV1 extends Abstr
     return fileInfoLoaded;
   }
 
+  @Override
+  public DataInput getDeleteColumnBloomFilterMetadata() {
+    return null;
+  }
+
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Thu Mar 21 18:53:35 2013
@@ -236,7 +236,7 @@ public class HFileReaderV2 extends Abstr
       return metaBlock.getBufferWithoutHeader();
     }
   }
-  
+
   /**
    * Read in a file block.
    * @param dataBlockOffset offset to read.
@@ -273,8 +273,8 @@ public class HFileReaderV2 extends Abstr
         new BlockCacheKey(name, dataBlockOffset,
             dataBlockEncoder.getEffectiveEncodingInCache(isCompaction),
             expectedBlockType);
-    
-    // Checking the block cache. 
+
+    // Checking the block cache.
     HFileBlock cachedBlock = this.getCachedBlock(cacheKey, cacheBlock, isCompaction,
         expectedBlockType, true);
     if (cachedBlock != null) {
@@ -355,7 +355,7 @@ public class HFileReaderV2 extends Abstr
       if (cachedBlock != null) {
         // Validate the block type first
         validateBlockType(cachedBlock, expectedBlockType);
-        
+
         // Validate encoding type for encoded blocks. We include encoding
         // type in the cache key, and we expect it to match on a cache hit.
         if (cachedBlock.getBlockType() == BlockType.ENCODED_DATA &&
@@ -365,7 +365,7 @@ public class HFileReaderV2 extends Abstr
               "has wrong encoding: " + cachedBlock.getDataBlockEncoding() +
               " (expected: " + dataBlockEncoder.getEncodingInCache() + ")");
         }
-        
+
         // Update the metrics if enabled
         if (updateMetrics) {
           BlockCategory blockCategory =
@@ -455,16 +455,16 @@ public class HFileReaderV2 extends Abstr
   protected abstract static class AbstractScannerV2
       extends AbstractHFileReader.Scanner {
     protected HFileBlock block;
-    
+
     /**
      * The next indexed key is to keep track of the indexed key of the next data block.
-     * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the 
+     * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
      * current data block is the last data block.
-     * 
+     *
      * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
      */
     protected byte[] nextIndexedKey;
-    
+
     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
         final boolean isCompaction) {
       super(r, cacheBlocks, isCompaction);
@@ -526,12 +526,12 @@ public class HFileReaderV2 extends Abstr
         } else {
           if (this.nextIndexedKey != null &&
               (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
-               reader.getComparator().compare(key, offset, length, 
+               reader.getComparator().compare(key, offset, length,
                    nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
-            // The reader shall continue to scan the current data block instead of querying the 
-            // block index as long as it knows the target key is strictly smaller than 
+            // The reader shall continue to scan the current data block instead of querying the
+            // block index as long as it knows the target key is strictly smaller than
             // the next indexed key or the current data block is the last data block.
-            return loadBlockAndSeekToKey(this.block, this.nextIndexedKey, 
+            return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
                 false, key, offset, length, false);
           }
         }
@@ -551,7 +551,7 @@ public class HFileReaderV2 extends Abstr
         return false;
       }
       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
-      
+
       if (reader.getComparator().compare(firstKey.array(),
           firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0)
       {
@@ -748,7 +748,7 @@ public class HFileReaderV2 extends Abstr
     }
 
     @Override
-    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey, 
+    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
         boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
         throws IOException {
       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
@@ -756,12 +756,12 @@ public class HFileReaderV2 extends Abstr
       } else if (rewind) {
         blockBuffer.rewind();
       }
-      
+
       // Update the nextIndexedKey
       this.nextIndexedKey = nextIndexedKey;
       return blockSeek(key, offset, length, seekBefore);
     }
-    
+
     /**
      * Updates the current block to be the given {@link HFileBlock}. Seeks to
      * the the first key/value pair.
@@ -783,7 +783,7 @@ public class HFileReaderV2 extends Abstr
       blockBuffer = block.getBufferWithoutHeader();
       readKeyValueLen();
       blockFetches++;
-      
+
       // Reset the next indexed key
       this.nextIndexedKey = null;
     }
@@ -1098,10 +1098,16 @@ public class HFileReaderV2 extends Abstr
     return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
   }
 
+  @Override
+  public DataInput getDeleteColumnBloomFilterMetadata() throws IOException {
+    return this.getBloomFilterMetadata(BlockType.DELETE_COLUMN_BLOOM_META);
+  }
+
   private DataInput getBloomFilterMetadata(BlockType blockType)
   throws IOException {
     if (blockType != BlockType.GENERAL_BLOOM_META &&
-        blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
+        blockType != BlockType.DELETE_FAMILY_BLOOM_META &&
+        blockType != BlockType.DELETE_COLUMN_BLOOM_META) {
       throw new RuntimeException("Block Type: " + blockType.toString() +
           " is not supported") ;
     }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java Thu Mar 21 18:53:35 2013
@@ -394,6 +394,12 @@ public class HFileWriterV1 extends Abstr
     throw new IOException("Delete Bloom filter is not supported in HFile V1");
   }
 
+  @Override
+  public void addDeleteColumnBloomFilter(BloomFilterWriter bfw)
+      throws IOException {
+    throw new IOException("Delete Bloom filter is not supported in HFile V1");
+  }
+
   /**
    * Write out the index in the version 1 format. This conforms to the legacy
    * version 1 format, but can still be read by
@@ -426,4 +432,4 @@ public class HFileWriterV1 extends Abstr
     return pos;
   }
 
-}
\ No newline at end of file
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Thu Mar 21 18:53:35 2013
@@ -404,13 +404,19 @@ public class HFileWriterV2 extends Abstr
     this.addBloomFilter(bfw, BlockType.DELETE_FAMILY_BLOOM_META);
   }
 
+  @Override
+  public void addDeleteColumnBloomFilter(final BloomFilterWriter bfw) {
+    this.addBloomFilter(bfw, BlockType.DELETE_COLUMN_BLOOM_META);
+  }
+
   private void addBloomFilter(final BloomFilterWriter bfw,
       final BlockType blockType) {
     if (bfw.getKeyCount() <= 0)
       return;
 
     if (blockType != BlockType.GENERAL_BLOOM_META &&
-        blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
+        blockType != BlockType.DELETE_FAMILY_BLOOM_META &&
+        blockType != BlockType.DELETE_COLUMN_BLOOM_META) {
       throw new RuntimeException("Block Type: " + blockType.toString() +
           "is not supported");
     }
@@ -430,4 +436,4 @@ public class HFileWriterV2 extends Abstr
     });
   }
 
-}
\ No newline at end of file
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Mar 21 18:53:35 2013
@@ -350,7 +350,7 @@ public class HRegionServer implements HR
   // large GC issues.
   private static long responseSizeLimit;
   public static boolean enableServerSideProfilingForAllCalls;
-  
+  public static AtomicInteger numOptimizedSeeks = new AtomicInteger(0);
   private int numRowRequests = 0;
 
   public static boolean runMetrics = true;
@@ -410,9 +410,9 @@ public class HRegionServer implements HR
         HConstants.HBASE_RPC_TIMEOUT_KEY,
         HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 
-    this.responseSizeLimit = conf.getLong("hbase.regionserver.results.size.max",
+    responseSizeLimit = conf.getLong("hbase.regionserver.results.size.max",
         (long)Integer.MAX_VALUE); // set the max to 2G
-    this.enableServerSideProfilingForAllCalls = conf.getBoolean(
+    enableServerSideProfilingForAllCalls = conf.getBoolean(
         "hbase.regionserver.enable.serverside.profiling", false);
 
     reinitialize();
@@ -1346,6 +1346,7 @@ public class HRegionServer implements HR
 
   protected void metrics() {
     this.metrics.regions.set(this.onlineRegions.size());
+    this.metrics.numOptimizedSeeks.set(numOptimizedSeeks.intValue());
     // Is this too expensive every three seconds getting a lock on onlineRegions
     // and then per store carried?  Can I make metrics be sloppier and avoid
     // the synchronizations?

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java Thu Mar 21 18:53:35 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.PriorityQueue;
@@ -402,4 +403,20 @@ public class KeyValueHeap extends NonLaz
   KeyValueScanner getCurrentForTesting() {
     return current;
   }
+
+  /**
+   * @return all the scanners from the heap and the current KeyValueScanner,
+   * since it is not included in the heap
+   */
+  List<KeyValueScanner> getActiveScanners() {
+    List<KeyValueScanner> allScanners = new ArrayList<KeyValueScanner>();
+    allScanners.addAll(this.heap);
+    allScanners.add(current);
+    return allScanners;
+  }
+
+  @Override
+  public boolean passesDeleteColumnCheck(KeyValue kv) {
+    return true;
+  }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java Thu Mar 21 18:53:35 2013
@@ -116,4 +116,12 @@ public interface KeyValueScanner {
    */
   public void enforceSeek() throws IOException;
 
+  /**
+   * MemStoreScanner and StoreFileScanner should implement this method to check
+   * if there are any deletes in the bloom filter or memstore.
+   *
+   * @return true if there are any deletes.
+   */
+  boolean passesDeleteColumnCheck(KeyValue kv);
+
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java Thu Mar 21 18:53:35 2013
@@ -78,6 +78,9 @@ public class MemStore implements HeapSiz
   // Used to track own heapSize
   final AtomicLong size;
 
+  private AtomicLong numDeletesInKvSet;
+  private AtomicLong numDeletesInSnapshot;
+
   TimeRangeTracker timeRangeTracker;
   TimeRangeTracker snapshotTimeRangeTracker;
 
@@ -102,6 +105,8 @@ public class MemStore implements HeapSiz
     timeRangeTracker = new TimeRangeTracker();
     snapshotTimeRangeTracker = new TimeRangeTracker();
     this.size = new AtomicLong(DEEP_OVERHEAD);
+    this.numDeletesInKvSet = new AtomicLong(0);
+    this.numDeletesInSnapshot = new AtomicLong(0);
   }
 
   void dump() {
@@ -134,6 +139,8 @@ public class MemStore implements HeapSiz
           this.timeRangeTracker = new TimeRangeTracker();
           // Reset heap to not include any keys
           this.size.set(DEEP_OVERHEAD);
+          this.numDeletesInSnapshot = numDeletesInKvSet;
+          this.numDeletesInKvSet.set(0);
         }
       }
     } finally {
@@ -190,6 +197,9 @@ public class MemStore implements HeapSiz
       s = heapSizeChange(kv, this.kvset.add(kv));
       timeRangeTracker.includeTimestamp(kv);
       this.size.addAndGet(s);
+      if (kv.isDelete()) {
+        this.numDeletesInKvSet.incrementAndGet();
+      }
     } finally {
       this.lock.readLock().unlock();
     }
@@ -210,6 +220,9 @@ public class MemStore implements HeapSiz
     } finally {
       this.lock.readLock().unlock();
     }
+    if (delete.isDelete()) {
+      this.numDeletesInKvSet.incrementAndGet();
+    }
     this.size.addAndGet(s);
     return s;
   }
@@ -667,10 +680,17 @@ public class MemStore implements HeapSiz
         long oldestUnexpiredTS) {
       return shouldSeek(scan, oldestUnexpiredTS);
     }
+
+    @Override
+    public boolean passesDeleteColumnCheck(KeyValue kv) {
+      if (numDeletesInKvSet.get() > 0 || numDeletesInSnapshot.get() > 0)
+        return true;
+      return false;
+    }
   }
 
   public final static long FIXED_OVERHEAD = ClassSize.align(
-      ClassSize.OBJECT + (9 * ClassSize.REFERENCE));
+      ClassSize.OBJECT + (11 * ClassSize.REFERENCE));
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java Thu Mar 21 18:53:35 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.util.Bytes;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.NavigableSet;
 
 /**
@@ -248,7 +249,6 @@ public class ScanQueryMatcher {
     if (!this.deletes.isEmpty()) {
       DeleteResult deleteResult = deletes.isDeleted(bytes, offset, qualLength,
           timestamp);
-
       switch (deleteResult) {
         case FAMILY_DELETED:
         case COLUMN_DELETED:
@@ -373,7 +373,23 @@ public class ScanQueryMatcher {
     }
   }
 
-  public KeyValue getKeyForNextColumn(KeyValue kv) {
+  /**
+   * If there is no next column, we return a KV which is the last possible key
+   * for the current row. Otherwise, we proceed to the next kv (the first kv on
+   * the next column). if there is no delete column blooom filter enabled, we
+   * return the next kv on the column. If the delete column bloom filter is
+   * enabled, we first check whether the scanners contain delete information
+   * about the next kv. If there is no delete, we proceed to the searched kv
+   * with highest timestamp.
+   *
+   * @param kv - current keyvalue
+   * @param allScanners - all scanners including the scanners from memstore and
+   * storefiles
+   * @param deleteColBloomFilterEnabled - is the delete column bloom filter
+   * ebnabled
+   * @return - kv in the next column
+   */
+  public KeyValue getKeyForNextColumn(KeyValue kv, List<KeyValueScanner> allScanners, boolean deleteColBloomFilterEnabled) {
     ColumnCount nextColumn = columns.getColumnHint();
     if (nextColumn == null) {
       return KeyValue.createLastOnRow(
@@ -381,10 +397,22 @@ public class ScanQueryMatcher {
           kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
           kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
     } else {
-      return KeyValue.createFirstOnRow(
-          kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
-          kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
-          nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
+      KeyValue nextKV = KeyValue.createFirstOnRow(kv.getBuffer(),
+          kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
+          kv.getFamilyOffset(), kv.getFamilyLength(), nextColumn.getBuffer(),
+          nextColumn.getOffset(), nextColumn.getLength());
+      if (!deleteColBloomFilterEnabled) {
+        return nextKV;
+      }
+      for (KeyValueScanner kvScanner : allScanners) {
+        if (kvScanner.passesDeleteColumnCheck(nextKV))
+          return nextKV;
+      }
+      HRegionServer.numOptimizedSeeks.incrementAndGet();
+      return KeyValue.createFirstOnRow(nextKV.getBuffer(), nextKV.getRowOffset(), nextKV.getRowLength(),
+          nextKV.getBuffer(), nextKV.getFamilyOffset(), nextKV.getFamilyLength(),
+          nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength(),
+          Math.max(tr.getMax() -1, tr.getMin()));
     }
   }
 

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Mar 21 18:53:35 2013
@@ -285,6 +285,7 @@ public class Store extends SchemaConfigu
    */
   void setCompactionPolicy(String managerClassName) {
     try {
+      @SuppressWarnings("unchecked")
       Class<? extends CompactionManager> managerClass =
         (Class<? extends CompactionManager>) Class.forName(managerClassName);
       compactionManager = managerClass.getDeclaredConstructor(
@@ -693,7 +694,6 @@ public class Store extends SchemaConfigu
         writer = createWriterInTmp(snapshot.size());
         writer.setTimeRangeTracker(snapshotTimeRangeTracker);
         fileName = writer.getPath().getName();
-        int entries = 0;
         try {
           final List<KeyValue> kvs = new ArrayList<KeyValue>();
           boolean hasMore;
@@ -710,7 +710,6 @@ public class Store extends SchemaConfigu
                   kv.setMemstoreTS(0);
                 }
                 writer.append(kv);
-                entries++;
                 flushed += this.memstore.heapSizeChange(kv, true);
               }
               kvs.clear();
@@ -1020,7 +1019,7 @@ public class Store extends SchemaConfigu
       // Ready to go. Have list of files to compact.
       StoreFile.Writer writer = compactStores(filesToCompact, isMajor, maxId);
       // Move the compaction into place.
-      StoreFile sf = completeCompaction(filesToCompact, writer);
+      completeCompaction(filesToCompact, writer);
     } finally {
       synchronized (filesCompacting) {
         filesCompacting.removeAll(filesToCompact);
@@ -1920,4 +1919,5 @@ public class Store extends SchemaConfigu
     }
    }
   }
+
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Thu Mar 21 18:53:35 2013
@@ -124,6 +124,10 @@ public class StoreFile extends SchemaCon
   public static final byte[] DELETE_FAMILY_COUNT =
       Bytes.toBytes("DELETE_FAMILY_COUNT");
 
+  /** Delete Column Count in FileInfo */
+  public static final byte[] DELETE_COLUMN_COUNT =
+      Bytes.toBytes("DELETE_COLUMN_COUNT");
+
   /** Last Bloom filter key in FileInfo */
   private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY");
 
@@ -397,7 +401,7 @@ public class StoreFile extends SchemaCon
    * the given list. Store files that were created by a mapreduce
    * bulk load are ignored, as they do not correspond to any edit
    * log items.
-   * @param sfs 
+   * @param sfs
    * @param includeBulkLoadedFiles
    * @return 0 if no non-bulk-load files are provided or, this is Store that
    * does not yet have any store files.
@@ -467,8 +471,8 @@ public class StoreFile extends SchemaCon
           this.sequenceid += 1;
         }
       }
-    } 
-    
+    }
+
     if (isBulkLoadResult()){
       // generate the sequenceId from the fileName
       // fileName is of the form <randomName>_SeqId_<id-when-loaded>_
@@ -523,6 +527,9 @@ public class StoreFile extends SchemaCon
     // load delete family bloom filter
     reader.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
 
+    // load delete column bloom filter
+    reader.loadBloomfilter(BlockType.DELETE_COLUMN_BLOOM_META);
+
     try {
       byte [] timerangeBytes = metadataMap.get(TIMERANGE_KEY);
       if (timerangeBytes != null) {
@@ -848,13 +855,16 @@ public class StoreFile extends SchemaCon
   public static class Writer {
     private final BloomFilterWriter generalBloomFilterWriter;
     private final BloomFilterWriter deleteFamilyBloomFilterWriter;
+    private final BloomFilterWriter deleteColumnBloomFilterWriter;
     private final BloomType bloomType;
     private byte[] lastBloomKey;
     private int lastBloomKeyOffset, lastBloomKeyLen;
     private final KVComparator kvComparator;
     private KeyValue lastKv = null;
     private KeyValue lastDeleteFamilyKV = null;
+    private KeyValue lastDeleteColumnKV = null;
     private long deleteFamilyCnt = 0;
+    private long deleteColumnCnt = 0;
 
     protected final HFileDataBlockEncoder dataBlockEncoder;
 
@@ -904,7 +914,7 @@ public class StoreFile extends SchemaCon
         this.deleteFamilyBloomFilterWriter = BloomFilterFactory
             .createDeleteBloomAtWrite(wb.conf, wb.cacheConf,
                 (int) Math.min(wb.maxKeyCount, Integer.MAX_VALUE), writer,
-                wb.bloomErrorRate);
+                wb.bloomErrorRate, HConstants.DELETE_FAMILY_BLOOM_FILTER);
       } else {
         deleteFamilyBloomFilterWriter = null;
       }
@@ -912,6 +922,16 @@ public class StoreFile extends SchemaCon
         LOG.info("Delete Family Bloom filter type for " + wb.filePath + ": "
             + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
       }
+      // initialize DeleteColumn bloom filter
+      this.deleteColumnBloomFilterWriter = BloomFilterFactory
+          .createDeleteBloomAtWrite(wb.conf, wb.cacheConf,
+              (int) Math.min(wb.maxKeyCount, Integer.MAX_VALUE), writer,
+              wb.bloomErrorRate, HConstants.DELETE_COLUMN_BLOOM_FILTER);
+
+      if (deleteColumnBloomFilterWriter != null) {
+        LOG.info("Delete Column Family filter type for " + wb.filePath + ": "
+            + deleteColumnBloomFilterWriter.getClass().getSimpleName());
+      }
     }
 
     /**
@@ -1073,9 +1093,33 @@ public class StoreFile extends SchemaCon
       }
     }
 
+    private void appendDeleteColumnBloomFilter(final KeyValue kv)
+        throws IOException {
+      if (!kv.isDeleteColumn()) {
+        return;
+      }
+      deleteColumnCnt++;
+      if (this.deleteColumnBloomFilterWriter != null) {
+        boolean newKey = true;
+        if (lastDeleteColumnKV != null) {
+          boolean compared = kvComparator.compare(kv, lastDeleteColumnKV) == 0 ? true
+              : false;
+          newKey = !compared;
+        }
+        if (newKey) {
+          byte[] bloomKey = deleteColumnBloomFilterWriter.createBloomKey(
+              kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
+              kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
+          this.deleteColumnBloomFilterWriter.add(bloomKey, 0, bloomKey.length);
+          this.lastDeleteColumnKV = kv;
+        }
+      }
+    }
+
     public void append(final KeyValue kv) throws IOException {
       appendGeneralBloomfilter(kv);
       appendDeleteFamilyBloomFilter(kv);
+      appendDeleteColumnBloomFilter(kv);
       writer.append(kv);
       includeInTimeRangeTracker(kv);
     }
@@ -1139,19 +1183,36 @@ public class StoreFile extends SchemaCon
       return hasDeleteFamilyBloom;
     }
 
+    private boolean closeDeleteColumnBloomFilter() throws IOException {
+      boolean hasDeleteColumnBloom = closeBloomFilter(deleteColumnBloomFilterWriter);
+
+      //add the delete column Bloom filter writer
+      if (hasDeleteColumnBloom) {
+        writer.addDeleteColumnBloomFilter(deleteColumnBloomFilterWriter);
+      }
+
+      // append file info about the number of delete column kvs
+      // even if there is no delete column Bloom.
+      writer.appendFileInfo(DELETE_COLUMN_COUNT,
+          Bytes.toBytes(this.deleteColumnCnt));
+      return hasDeleteColumnBloom;
+    }
+
     public void close() throws IOException {
       // Save data block encoder metadata in the file info.
       dataBlockEncoder.saveMetadata(this);
 
       boolean hasGeneralBloom = this.closeGeneralBloomFilter();
       boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
+      boolean hasDeleteColumnBloom = this.closeDeleteColumnBloomFilter();
 
       writer.close();
 
       // Log final Bloom filter statistics. This needs to be done after close()
       // because compound Bloom filters might be finalized as part of closing.
       StoreFile.LOG.info((hasGeneralBloom ? "" : "NO ") + "General Bloom and "
-          + (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily"
+          + (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily and "
+          + (hasDeleteColumnBloom ? "" : "NO ") + "DeleteColumn"
           + " was added to HFile (" + getPath() + ") ");
 
     }
@@ -1174,12 +1235,14 @@ public class StoreFile extends SchemaCon
 
     protected BloomFilter generalBloomFilter = null;
     protected BloomFilter deleteFamilyBloomFilter = null;
+    protected BloomFilter deleteColumnBloomFilter = null;
     protected BloomType bloomFilterType;
     private final HFile.Reader reader;
     protected TimeRangeTracker timeRangeTracker = null;
     protected long sequenceID = -1;
     private byte[] lastBloomKey;
     private long deleteFamilyCnt = -1;
+    private long deleteColumnCnt = -1;
 
     public Reader(FileSystem fs, Path path, CacheConfig cacheConf,
         DataBlockEncoding preferredEncodingInCache) throws IOException {
@@ -1345,6 +1408,31 @@ public class StoreFile extends SchemaCon
       return true;
     }
 
+    public boolean passesDeleteColumnBloomFilter (KeyValue kv) {
+      // Empty file or there is no delete column at all
+      if (reader.getTrailer().getEntryCount() == 0 || deleteColumnCnt == 0) {
+        return false;
+      }
+      BloomFilter bloomFilter = this.deleteColumnBloomFilter;
+      if (bloomFilter == null) {
+        return true;
+      }
+      try {
+        if (!bloomFilter.supportsAutoLoading()) {
+          return true;
+        }
+        byte[] bloomKey = deleteColumnBloomFilter.createBloomKey(kv.getBuffer(),
+            kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
+            kv.getQualifierOffset(), kv.getQualifierLength());
+        return bloomFilter.contains(bloomKey, 0, bloomKey.length, null);
+      } catch (IllegalArgumentException e) {
+        LOG.error("Bad Delete Column bloom filter data -- proceeding without",
+            e);
+        setDeleteColumnBloomFilterFaulty();
+      }
+      return true;
+    }
+
     /**
      * A method for checking Bloom filters. Called directly from
      * {@link StoreFileScanner} in case of a multi-column query.
@@ -1469,6 +1557,10 @@ public class StoreFile extends SchemaCon
       if (cnt != null) {
         deleteFamilyCnt = Bytes.toLong(cnt);
       }
+      cnt = fi.get(DELETE_COLUMN_COUNT);
+      if (cnt != null) {
+        deleteColumnCnt = Bytes.toLong(cnt);
+      }
 
       return fi;
     }
@@ -1476,6 +1568,7 @@ public class StoreFile extends SchemaCon
     public void loadBloomfilter() {
       this.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
       this.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
+      this.loadBloomfilter(BlockType.DELETE_COLUMN_BLOOM_META);
     }
 
     private void loadBloomfilter(BlockType blockType) {
@@ -1510,6 +1603,17 @@ public class StoreFile extends SchemaCon
                 + deleteFamilyBloomFilter.getClass().getSimpleName()
                 + ") metadata for " + reader.getName());
           }
+        } else if (blockType == BlockType.DELETE_COLUMN_BLOOM_META) {
+          if (this.deleteColumnBloomFilter != null) {
+            return;
+          }
+          DataInput bloomMeta = reader.getDeleteColumnBloomFilterMetadata();
+          if (bloomMeta != null) {
+            deleteColumnBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
+            LOG.info("Loaded Delete Column Bloom ( "
+                + deleteColumnBloomFilter.getClass().getSimpleName()
+                + ") metadata for " + reader.getName());
+          }
         } else {
           throw new RuntimeException("Block Type: " + blockType.toString()
               + "is not supported for Bloom filter");
@@ -1530,6 +1634,8 @@ public class StoreFile extends SchemaCon
         setGeneralBloomFilterFaulty();
       } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
         setDeleteFamilyBloomFilterFaulty();
+      } else if (blockType == BlockType.DELETE_COLUMN_BLOOM_META) {
+        setDeleteColumnBloomFilterFaulty();
       }
     }
 
@@ -1553,6 +1659,10 @@ public class StoreFile extends SchemaCon
       this.deleteFamilyBloomFilter = null;
     }
 
+    public void setDeleteColumnBloomFilterFaulty() {
+      this.deleteColumnBloomFilter = null;
+    }
+
     public byte[] getLastKey() {
       return reader.getLastKey();
     }
@@ -1573,6 +1683,10 @@ public class StoreFile extends SchemaCon
       return deleteFamilyCnt;
     }
 
+    public long getDeleteColumnCnt() {
+      return deleteColumnCnt;
+    }
+
     public byte[] getFirstKey() {
       return reader.getFirstKey();
     }
@@ -1654,7 +1768,7 @@ public class StoreFile extends SchemaCon
         return sf.getMaxSequenceId();
       }
     }
-    
+
     private static class GetBulkTime implements Function<StoreFile, Long> {
       @Override
       public Long apply(StoreFile sf) {
@@ -1662,7 +1776,7 @@ public class StoreFile extends SchemaCon
         return sf.getBulkLoadTimestamp();
       }
     }
-    
+
     private static class GetPathName implements Function<StoreFile, String> {
       @Override
       public String apply(StoreFile sf) {

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java Thu Mar 21 18:53:35 2013
@@ -367,4 +367,9 @@ class StoreFileScanner implements KeyVal
     return reader.passesTimerangeFilter(scan, oldestUnexpiredTS) &&
         reader.passesBloomFilter(scan, columns);
   }
+
+  @Override
+  public boolean passesDeleteColumnCheck(KeyValue kv) {
+    return this.reader.passesDeleteColumnBloomFilter(kv);
+  }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Thu Mar 21 18:53:35 2013
@@ -64,6 +64,7 @@ public class StoreScanner extends NonLaz
   private final KeyValueAggregator keyValueAggregator;
   private final NavigableSet<byte[]> columns;
   private final long oldestUnexpiredTS;
+  private boolean deleteColBloomEnabled = false;
 
   /** We don't ever expect to change this, the constant is just for clarity. */
   static final boolean LAZY_SEEK_ENABLED_BY_DEFAULT = true;
@@ -103,6 +104,11 @@ public class StoreScanner extends NonLaz
     // for multi-row (non-"get") scans because this is not done in
     // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
     useRowColBloom = numCol > 1 || (!isGet && numCol == 1);
+    if (store != null) {
+      this.deleteColBloomEnabled = store.conf.getBoolean(
+          HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING,
+          HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER);
+    }
   }
 
   /**
@@ -267,7 +273,6 @@ public class StoreScanner extends NonLaz
       final List<? extends KeyValueScanner> allScanners) {
     List<KeyValueScanner> scanners =
       new ArrayList<KeyValueScanner>(allScanners.size());
-
     // include only those scan files which pass all filters
     for (KeyValueScanner kvs : allScanners) {
       if (kvs.shouldUseScanner(scan, columns, oldestUnexpiredTS)) {
@@ -385,7 +390,6 @@ public class StoreScanner extends NonLaz
         }
         prevKV = kv;
         ScanQueryMatcher.MatchCode qcode = matcher.match(copyKv);
-
         if ((qcode == MatchCode.INCLUDE) ||
           (qcode == MatchCode.INCLUDE_AND_SEEK_NEXT_COL) ||
           (qcode == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
@@ -397,9 +401,7 @@ public class StoreScanner extends NonLaz
             qcode = keyValueAggregator.nextAction(qcode);
           }
         }
-
         switch(qcode) {
-
           case SEEK_TO_EFFECTIVE_TS:
             reseek(matcher.getKeyForEffectiveTSOnRow(kv));
             break;
@@ -439,7 +441,13 @@ public class StoreScanner extends NonLaz
               }
               reseek(matcher.getKeyForNextRow(kv));
             } else if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) {
-              reseek(matcher.getKeyForNextColumn(kv));
+            // we check this since some tests have store = null
+              if (this.store == null) {
+                reseek(matcher.getKeyForNextColumn(kv, null, false));
+              } else {
+                reseek(matcher.getKeyForNextColumn(kv,
+                    this.heap.getActiveScanners(), deleteColBloomEnabled));
+              }
             } else {
               this.heap.next();
             }
@@ -476,7 +484,12 @@ public class StoreScanner extends NonLaz
             break;
 
           case SEEK_NEXT_COL:
-            reseek(matcher.getKeyForNextColumn(kv));
+          // we check this since some tests have store = null
+            if (this.store == null) {
+              reseek(matcher.getKeyForNextColumn(kv, null, false));
+            } else {
+              reseek(matcher.getKeyForNextColumn(kv, this.heap.getActiveScanners(),deleteColBloomEnabled));
+            }
             break;
 
           case SKIP:
@@ -661,4 +674,8 @@ public class StoreScanner extends NonLaz
     lazySeekEnabledGlobally = enable;
   }
 
+  @Override
+  public boolean passesDeleteColumnCheck(KeyValue kv) {
+    return true;
+  }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Thu Mar 21 18:53:35 2013
@@ -53,7 +53,6 @@ import java.util.List;
  * these variables (objects) have methods to update their values.
  */
 public class RegionServerMetrics implements Updater {
-  @SuppressWarnings({"FieldCanBeLocal"})
   private final Log LOG = LogFactory.getLog(this.getClass());
   private final MetricsRecord metricsRecord;
   private long lastUpdate = System.currentTimeMillis();
@@ -216,6 +215,9 @@ public class RegionServerMetrics impleme
   public final MetricsTimeVaryingRate mvccWaitTime =
     new MetricsTimeVaryingRate("mvccWait", registry);
 
+  public final MetricsRate numOptimizedSeeks =
+    new MetricsRate("numOptimizedSeeks", registry);
+
   /**
    * time each scheduled compaction takes
    */
@@ -332,6 +334,7 @@ public class RegionServerMetrics impleme
       this.blockCacheHitRatio.pushMetric(this.metricsRecord);
       this.rowReadCnt.pushMetric(this.metricsRecord);
       this.rowUpdatedCnt.pushMetric(this.metricsRecord);
+      this.numOptimizedSeeks.pushMetric(this.metricsRecord);
 
       // Be careful. Here is code for MTVR from up in hadoop:
       // public synchronized void inc(final int numOps, final long time) {
@@ -500,6 +503,8 @@ public class RegionServerMetrics impleme
       Float.valueOf(this.rowUpdatedCnt.getPreviousIntervalValue()));
     sb = Strings.appendKeyValue(sb, "numReads",
       Float.valueOf(this.rowReadCnt.getPreviousIntervalValue()));
+    sb = Strings.appendKeyValue(sb, "numOptimizedSeeks",
+      Float.valueOf(this.numOptimizedSeeks.getPreviousIntervalValue()));
 
     // Duplicate from jvmmetrics because metrics are private there so
     // inaccessible.

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java Thu Mar 21 18:53:35 2013
@@ -26,6 +26,7 @@ import java.io.IOException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -73,6 +74,10 @@ public final class BloomFilterFactory {
   public static final String IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED =
       "io.storefile.delete.family.bloom.enabled";
 
+  /** Master switch to enable Delete Column Family filters */
+  public static final String IO_STOREFILE_DELETECOLUMN_BLOOM_ENABLED =
+      "io.storefile.delete.column.bloom.enabled";
+
   /**
    * Target Bloom block size. Bloom filter blocks of approximately this size
    * are interleaved with data blocks.
@@ -131,6 +136,13 @@ public final class BloomFilterFactory {
   }
 
   /**
+   * @return true if Delete Column Bloom filters are enabled in the given configuration
+   */
+  public static boolean isDeleteColumnBloomEnabled(Configuration conf) {
+    return conf.getBoolean(IO_STOREFILE_DELETECOLUMN_BLOOM_ENABLED, true);
+  }
+
+  /**
    * @return the Bloom filter error rate in the given configuration
    */
   public static float getErrorRate(Configuration conf) {
@@ -239,7 +251,7 @@ public final class BloomFilterFactory {
   }
 
   /**
-   * Creates a new Delete Family Bloom filter at the time of
+   * Creates a new Delete Family/Column Bloom filter at the time of
    * {@link org.apache.hadoop.hbase.regionserver.StoreFile} writing.
    * @param conf
    * @param maxKeys an estimate of the number of keys we expect to insert.
@@ -251,9 +263,14 @@ public final class BloomFilterFactory {
    */
   public static BloomFilterWriter createDeleteBloomAtWrite(
       Configuration conf, CacheConfig cacheConf, int maxKeys,
-      HFile.Writer writer, float bloomErrorRate) {
-    if (!isDeleteFamilyBloomEnabled(conf)) {
-      LOG.info("Delete Bloom filters are disabled by configuration for "
+      HFile.Writer writer, float bloomErrorRate, String deleteBloomType) {
+    if (deleteBloomType.equals(HConstants.DELETE_FAMILY_BLOOM_FILTER) && !isDeleteFamilyBloomEnabled(conf)) {
+      LOG.info("Delete Family Bloom filters are disabled by configuration for "
+          + writer.getPath()
+          + (conf == null ? " (configuration is null)" : ""));
+      return null;
+    } else if (deleteBloomType.equals(HConstants.DELETE_COLUMN_BLOOM_FILTER) && !isDeleteColumnBloomEnabled(conf)) {
+      LOG.info("Delete Column Bloom filters are disabled by configuration for "
           + writer.getPath()
           + (conf == null ? " (configuration is null)" : ""));
       return null;
@@ -269,7 +286,7 @@ public final class BloomFilterFactory {
       writer.addInlineBlockWriter(bloomWriter);
       return bloomWriter;
     } else {
-      LOG.info("Delete Family Bloom filter is not supported in HFile V1");
+      LOG.info("Delete Bloom filter is not supported in HFile V1");
       return null;
     }
   }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java Thu Mar 21 18:53:35 2013
@@ -126,4 +126,9 @@ public class CollectionBackedScanner ext
     closed = true;
   }
 
+  @Override
+  public boolean passesDeleteColumnCheck(KeyValue kv) {
+    return true;
+  }
+
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Thu Mar 21 18:53:35 2013
@@ -257,11 +257,11 @@ public class TestCacheOnWrite {
     // so number of blocks depends on this parameter as well.
     if (encoder.getEncodingOnDisk() == DataBlockEncoding.PREFIX) {
       assertEquals("{" + cachedDataBlockType
-          + "=965, LEAF_INDEX=121, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=17}",
+          + "=965, LEAF_INDEX=121, BLOOM_CHUNK=10, INTERMEDIATE_INDEX=17}",
           countByType);
     } else {
       assertEquals("{" + cachedDataBlockType
-          + "=1379, LEAF_INDEX=173, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=24}",
+          + "=1379, LEAF_INDEX=173, BLOOM_CHUNK=10, INTERMEDIATE_INDEX=24}",
           countByType);
     }
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java Thu Mar 21 18:53:35 2013
@@ -12,6 +12,7 @@ import org.apache.hadoop.hbase.HBaseConf
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
@@ -27,10 +28,11 @@ import org.apache.hadoop.hbase.util.Byte
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
 import org.junit.Test;
 
+@SuppressWarnings("deprecation")
 public class TestBlocksRead extends HBaseTestCase {
   static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
-  static final BloomType[] BLOOM_TYPE = new BloomType[] { BloomType.ROWCOL,
-      BloomType.ROW, BloomType.NONE };
+  static final BloomType[] BLOOM_TYPE = new BloomType[] {BloomType.ROWCOL,
+      BloomType.ROW, BloomType.NONE};
 
   private static BlockCache blockCache;
 
@@ -50,13 +52,11 @@ public class TestBlocksRead extends HBas
   /**
    * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
    */
-  @SuppressWarnings("deprecation")
   @Override
   protected void setUp() throws Exception {
     super.setUp();
   }
 
-  @SuppressWarnings("deprecation")
   @Override
   protected void tearDown() throws Exception {
     super.tearDown();
@@ -64,7 +64,7 @@ public class TestBlocksRead extends HBas
   }
 
   private void initHRegion(byte[] tableName, String callingMethod,
-      HBaseConfiguration conf, String family) throws IOException {
+      HBaseConfiguration conf, String family, boolean versions) throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     HColumnDescriptor familyDesc;
     for (int i = 0; i < BLOOM_TYPE.length; i++) {
@@ -72,6 +72,8 @@ public class TestBlocksRead extends HBas
       familyDesc = new HColumnDescriptor(family + "_" + bloomType)
           .setBlocksize(1)
           .setBloomFilterType(BLOOM_TYPE[i]);
+      if (versions)
+        familyDesc.setMaxVersions(Integer.MAX_VALUE);
       htd.addFamily(familyDesc);
     }
 
@@ -81,6 +83,8 @@ public class TestBlocksRead extends HBas
     blockCache = new CacheConfig(conf).getBlockCache();
   }
 
+
+
   private void putData(String family, String row, String col, long version)
       throws IOException {
     for (int i = 0; i < BLOOM_TYPE.length; i++) {
@@ -111,6 +115,40 @@ public class TestBlocksRead extends HBas
   }
 
   private KeyValue[] getData(String family, String row, List<String> columns,
+      long timestamp, int expBlocks) throws IOException {
+    return getData(family, row, columns, timestamp, expBlocks, expBlocks, expBlocks);
+  }
+
+  private KeyValue[] getData(String family, String row, List<String> columns,
+      long timestamp, int expBlocksRowCol, int expBlocksRow, int expBlocksNone)
+      throws IOException {
+    int[] expBlocks = new int[] { expBlocksRowCol, expBlocksRow, expBlocksNone };
+    KeyValue[] kvs = null;
+
+    for (int i = 0; i < BLOOM_TYPE.length; i++) {
+      BloomType bloomType = BLOOM_TYPE[i];
+      byte[] cf = Bytes.toBytes(family + "_" + bloomType);
+      long blocksStart = getBlkAccessCount(cf);
+      Get get = new Get(Bytes.toBytes(row));
+
+      for (String column : columns) {
+        get.addColumn(cf, Bytes.toBytes(column));
+        get.setTimeStamp(timestamp);
+      }
+
+      kvs = region.get(get, null).raw();
+      long blocksEnd = getBlkAccessCount(cf);
+      if (expBlocks[i] != -1) {
+        assertEquals("Blocks Read Check for Bloom: " + bloomType, expBlocks[i],
+            blocksEnd - blocksStart);
+      }
+      System.out.println("Blocks Read for Bloom: " + bloomType + " = "
+          + (blocksEnd - blocksStart) + "Expected = " + expBlocks[i]);
+    }
+    return kvs;
+  }
+
+  private KeyValue[] getData(String family, String row, List<String> columns,
       int expBlocksRowCol, int expBlocksRow, int expBlocksNone)
       throws IOException {
     int[] expBlocks = new int[] { expBlocksRowCol, expBlocksRow, expBlocksNone };
@@ -160,6 +198,26 @@ public class TestBlocksRead extends HBas
     region.delete(del, null, true);
   }
 
+  public void deleteColumn(String family, String qualifier, String row,
+      long version) throws IOException {
+    Delete del = new Delete(Bytes.toBytes(row));
+    for (int i=0; i<BLOOM_TYPE.length; i++) {
+      del.deleteColumn(Bytes.toBytes(family + BLOOM_TYPE[i]),
+          Bytes.toBytes(qualifier), version);
+    }
+    region.delete(del, null, true);
+  }
+
+  public void deleteColumn(String family, String qualifier, String row)
+      throws IOException {
+    Delete del = new Delete(Bytes.toBytes(row));
+    for (int i=0; i<BLOOM_TYPE.length; i++) {
+      del.deleteColumns(Bytes.toBytes(family + "_" + BLOOM_TYPE[i]),
+          Bytes.toBytes(qualifier));
+    }
+    region.delete(del, null, true);
+  }
+
   private static void verifyData(KeyValue kv, String expectedRow,
       String expectedCol, long expectedVersion) {
     assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow()));
@@ -191,7 +249,7 @@ public class TestBlocksRead extends HBas
     String FAMILY = "cf1";
     KeyValue kvs[];
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
+    initHRegion(TABLE, getName(), conf, FAMILY, false);
 
     putData(FAMILY, "row", "col1", 1);
     putData(FAMILY, "row", "col2", 2);
@@ -200,6 +258,7 @@ public class TestBlocksRead extends HBas
     putData(FAMILY, "row", "col5", 5);
     putData(FAMILY, "row", "col6", 6);
     putData(FAMILY, "row", "col7", 7);
+
     region.flushcache();
 
     // Expected block reads: 1
@@ -243,7 +302,7 @@ public class TestBlocksRead extends HBas
     String FAMILY = "cf1";
     KeyValue kvs[];
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
+    initHRegion(TABLE, getName(), conf, FAMILY, false);
 
     // File 1
     putData(FAMILY, "row", "col1", 1);
@@ -340,7 +399,7 @@ public class TestBlocksRead extends HBas
     String FAMILY = "cf1";
     KeyValue kvs[];
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
+    initHRegion(TABLE, getName(), conf, FAMILY, true);
 
     deleteFamily(FAMILY, "row", 200);
     for (int i = 0; i < 100; i++) {
@@ -357,6 +416,231 @@ public class TestBlocksRead extends HBas
     verifyData(kvs[0], "row", "col99", 201);
   }
 
+  @Test
+  public void testDeleteColBloomFilterWithoutDeletesWithFlushCache() throws IOException{
+    HRegionServer.numOptimizedSeeks.set(0);
+    byte[] TABLE = Bytes.toBytes("testDeleteColBloomFilterWithoutDeletes");
+    String FAMILY = "cf1";
+    KeyValue kvs[];
+    HBaseConfiguration conf = getConf();
+    conf.setBoolean("io.storefile.delete.column.bloom.enabled", true);
+    initHRegion(TABLE, getName(), conf, FAMILY, true);
+    if (!conf.getBoolean(HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING, HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER)) {
+      System.out.println("ignoring this test since the delete bloom filter is not enabled...");
+      return;
+    }
+    for (int i = 1; i < 8; i++) {
+      for (int j = 1; j < 6; j++) {
+        putData(FAMILY, "row", "col"+i, j);
+      }
+    }
+    region.flushcache();
+
+    Scan scan = new Scan();
+    scan.setMaxVersions(5);
+    InternalScanner s = region.getScanner(scan);
+    List<KeyValue> results = new ArrayList<KeyValue>(10);
+    while (s.next(results))
+      ;
+    s.close();
+    for (KeyValue kv : results) {
+      System.out.println(kv.toString());
+    }
+    System.out.println("======");
+    for (int i = 1; i < 8; i++) {
+      for (int j = 1; j < 6; j++) {
+        if (i == 1 && j == 5) {
+          /**
+           * Since this is the top KV (KV with smallest column and highest
+           * timestamp, we just need one seek
+           */
+          kvs = getData(FAMILY, "row", Arrays.asList("col" + i), j, 1);
+          verifyData(kvs[0], "row", "col" + i, j);
+          assertEquals(1, kvs.length);
+          System.out.println("=====");
+        } else {
+          /**
+           * We first go on the KV with max timestamp, then land on the actual
+           * KV, which is 2 blocks read
+           */
+          kvs = getData(FAMILY, "row", Arrays.asList("col" + i), j, 2);
+          verifyData(kvs[0], "row", "col" + i, j);
+          assertEquals(1, kvs.length);
+        }
+      }
+    }
+    int optimizedSeeks = HRegionServer.numOptimizedSeeks.get();
+    /**
+     * Since we have 7 columns, 3 col families and 5 versions in total, and no
+     * deletes, then we should get 7 * 3 * 5 optimized reads (all of them should
+     * be optimized)
+     **/
+    assertEquals(7 * 3 * 5, optimizedSeeks);
+  }
+
+
+  @Test
+  public void testDeleteColBloomFilterWithDeletesWithoutFlushCache() throws IOException{
+    HRegionServer.numOptimizedSeeks.set(0);
+    byte[] TABLE = Bytes.toBytes("testDeleteColBloomFilterWithDeletes");
+    String FAMILY = "cf1";
+    KeyValue kvs[];
+    HBaseConfiguration conf = getConf();
+    conf.setBoolean("io.storefile.delete.column.bloom.enabled", true);
+    initHRegion(TABLE, getName(), conf, FAMILY, true);
+    if (!conf.getBoolean(HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING, HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER)) {
+      System.out.println("ignoring this test since the delete bloom filter is not enabled...");
+      return;
+    }
+    for (int i = 1; i < 8; i++) {
+      for (int j = 1; j < 6; j++) {
+        putData(FAMILY, "row", "col"+i, j);
+      }
+    }
+
+    deleteColumn(FAMILY, "col2", "row");
+    deleteColumn(FAMILY, "col5", "row");
+    deleteColumn(FAMILY, "col7", "row");
+
+    kvs = getData(FAMILY, "row",  Arrays.asList("col2"), 5, 0);
+    assertTrue(kvs.length == 0);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col3"), 3, 0);
+    verifyData(kvs[0], "row", "col3", 3);
+    assertEquals(1, kvs.length);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col6"), 4, 0 );
+    verifyData(kvs[0], "row", "col6", 4);
+    assertEquals(1, kvs.length);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col7"), 5, 0);
+    assertTrue(kvs.length == 0);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col5"), 5, 0);
+    assertTrue(kvs.length == 0);
+
+    /**
+     * Since we don't do flush we read just from the memstore, we are not
+     * supposed to update the number of optimized seeks
+     **/
+    int optimizedSeeks = HRegionServer.numOptimizedSeeks.get();
+    assertEquals(0, optimizedSeeks);
+  }
+
+  @Test
+  public void testDeleteColBloomFilterWithDeletesWithFlushCache() throws IOException{
+    HRegionServer.numOptimizedSeeks.set(0);
+    byte[] TABLE = Bytes.toBytes("testDeleteColBloomFilterWithDeletes");
+    String FAMILY = "cf1";
+    KeyValue kvs[];
+    HBaseConfiguration conf = getConf();
+    conf.setBoolean("io.storefile.delete.column.bloom.enabled", true);
+    initHRegion(TABLE, getName(), conf, FAMILY, true);
+    if (!conf.getBoolean(HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING, HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER)) {
+      System.out.println("ignoring this test since the delete bloom filter is not enabled...");
+      return;
+    }
+    for (int i = 1; i < 8; i++) {
+      for (int j = 1; j < 6; j++) {
+        putData(FAMILY, "row", "col"+i, j);
+      }
+    }
+
+    deleteColumn(FAMILY, "col2", "row");
+    deleteColumn(FAMILY, "col5", "row");
+    deleteColumn(FAMILY, "col7", "row");
+    region.flushcache();
+
+    /**
+     * only the seeks for the KVs for which we don't have any deletes should be
+     * optimized, and since we have 3 col families we will have number of seeks
+     *
+     **/
+    kvs = getData(FAMILY, "row",  Arrays.asList("col2"), 5, 3);
+    assertTrue(kvs.length == 0);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col3"), 3, 2);
+    verifyData(kvs[0], "row", "col3", 3);
+    assertEquals(1, kvs.length);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col6"), 4, 2 );
+    verifyData(kvs[0], "row", "col6", 4);
+    assertEquals(1, kvs.length);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col7"), 5, 2);
+    assertTrue(kvs.length == 0);
+    kvs = getData(FAMILY, "row",  Arrays.asList("col5"), 5, 3);
+    assertTrue(kvs.length == 0);
+    int numSeeks = HRegionServer.numOptimizedSeeks.get();
+    assertEquals(6, numSeeks);
+  }
+
+  /**
+   * This test will make a number of puts, and then do flush, then do another
+   * series of puts. Then we will test the number of blocks read while doing get
+   * @throws IOException
+   */
+  @Test
+  public void testDeleteColBloomFilterWithoutDeletes() throws IOException {
+    HRegionServer.numOptimizedSeeks.set(0);
+    byte[] TABLE = Bytes.toBytes("testDeleteColBloomFilterWithDeletes");
+    String FAMILY = "cf1";
+    KeyValue kvs[];
+    HBaseConfiguration conf = getConf();
+    conf.setBoolean("io.storefile.delete.column.bloom.enabled", true);
+    initHRegion(TABLE, getName(), conf, FAMILY, true);
+    if (!conf.getBoolean(HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING, HConstants.ENABLE_DELETE_COLUMN_BLOOM_FILTER)) {
+      System.out.println("ignoring this test since the delete bloom filter is not enabled...");
+      return;
+    }
+    int i, j;
+    for (i = 1; i < 8; i++) {
+      for (j = 1; j < 6; j++) {
+        putData(FAMILY, "row", "col" + i, j);
+      }
+    }
+    region.flushcache();
+
+    /** Do some more puts and don't flush them **/
+    for (i = 5; i < 10; i++) {
+      for (j = 6; j < 10; j++) {
+        putData(FAMILY, "row", "col" + i, j);
+      }
+    }
+
+    /** Check the number of blocks read for the puts which are flushed **/
+    for (i = 1; i < 8; i++) {
+      for (j = 1; j < 6; j++) {
+        if (i == 1 && j == 5) {
+          /**
+           * Since this is the top KV (KV with smallest column and highest
+           * timestamp, we just need one seek
+           */
+          kvs = getData(FAMILY, "row", Arrays.asList("col" + i), j, 1);
+          verifyData(kvs[0], "row", "col" + i, j);
+          assertEquals(1, kvs.length);
+          System.out.println("=====");
+        } else {
+          /**
+           * We first go on the KV with max timestamp, then land on the actual
+           * KV, which is 2 blocks read
+           */
+          System.out.println("i: "+ i + "j: "+ j);
+          kvs = getData(FAMILY, "row", Arrays.asList("col" + i), j, 2);
+          verifyData(kvs[0], "row", "col" + i, j);
+          assertEquals(1, kvs.length);
+        }
+      }
+    }
+    /** Check the number of blocks read for the puts which were not flushed **/
+    for (i = 6; i < 10; i++) {
+      for (j = 6; j < 10; j++) {
+        kvs = getData(FAMILY, "row", Arrays.asList("col" + i), j, 0);
+        verifyData(kvs[0], "row", "col" + i, j);
+      }
+    }
+
+    int optimizedSeeks = HRegionServer.numOptimizedSeeks.get();
+    /**
+     * since there are no deletes, the additional gets for puts which are not
+     * flushed will be counted as optimized too
+     **/
+    assertEquals((7 * 5 * 3 + 4 *3 * 4), optimizedSeeks);
+  }
+
   /**
    * Test # of blocks read to ensure disabling cache-fill on Scan works.
    * @throws Exception
@@ -367,7 +651,7 @@ public class TestBlocksRead extends HBas
     String FAMILY = "cf1";
 
     HBaseConfiguration conf = getConf();
-    initHRegion(TABLE, getName(), conf, FAMILY);
+    initHRegion(TABLE, getName(), conf, FAMILY, false);
 
     putData(FAMILY, "row", "col1", 1);
     putData(FAMILY, "row", "col2", 2);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1459466&r1=1459465&r2=1459466&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Thu Mar 21 18:53:35 2013
@@ -64,6 +64,7 @@ import com.google.common.collect.Lists;
 /**
  * Test HStoreFile
  */
+@SuppressWarnings("deprecation")
 public class TestStoreFile extends HBaseTestCase {
   static final Log LOG = LogFactory.getLog(TestStoreFile.class);
   private MiniDFSCluster cluster;



Mime
View raw message