hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1569877 - in /hbase/branches/0.89-fb: ./ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/test/java/org/apache/hadoop/hbase/regionserver/
Date Wed, 19 Feb 2014 19:18:31 GMT
Author: liyin
Date: Wed Feb 19 19:18:30 2014
New Revision: 1569877

URL: http://svn.apache.org/r1569877
Log:
[HBASE-10568] Adding MemstoreBloomFilter which is an in memory bloom filter

Author: manukranthk

Summary: This diff adds MemstoreBloomFilter which is an in memory bloom filter against the
Memstore. This is introduced to add RowKeyPrefixBloomFilter in Memstore and store the bloom
filter for Rowkey Prefixes. The underlying bloom filter used is ByteBloomFilter. Since this
bloom filter needs to be accessed from different threads, the class needs to be thread safe.
This diff also incorporates the changes related to making the ByteBloomFilter class thread
safe.

Test Plan: Unit test. Currently I have a basic unit test which tests the common case. Will
add more unit tests in the next revision.

Reviewers: adela, liyintang, aaiyer

Reviewed By: liyintang

CC: hbase-eng@, gauravm, arice

Differential Revision: https://phabricator.fb.com/D1152434

Task ID: 3624202

Added:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilter.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterBase.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterContainer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreRowPrefixBloomFilter.java
Modified:
    hbase/branches/0.89-fb/pom.xml
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java

Modified: hbase/branches/0.89-fb/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/pom.xml?rev=1569877&r1=1569876&r2=1569877&view=diff
==============================================================================
--- hbase/branches/0.89-fb/pom.xml (original)
+++ hbase/branches/0.89-fb/pom.xml Wed Feb 19 19:18:30 2014
@@ -905,7 +905,7 @@
     <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-core-asl</artifactId>
-      <version>1.8.3</version>
+      <version>1.9.4</version>
     </dependency>
     <dependency>
       <groupId>org.codehaus.jackson</groupId>

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1569877&r1=1569876&r2=1569877&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Wed Feb 19
19:18:30 2014
@@ -916,6 +916,10 @@ public final class HConstants {
   public static final String USE_HFILEHISTOGRAM = "hbase.client.hfilehistogram.enabled";
   public static final boolean DEFAULT_USE_HFILEHISTOGRAM = true;
 
+  public static final String IN_MEMORY_BLOOM_ENABLED =
+      "hbase.hregion.memstore.bloom.filter.enabled";
+  public static final boolean DEFAULT_IN_MEMORY_BLOOM_ENABLED = false;
+
   private HConstants() {
     // Can't be instantiated with this constructor.
   }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java?rev=1569877&r1=1569876&r2=1569877&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
Wed Feb 19 19:18:30 2014
@@ -46,6 +46,13 @@ import java.util.concurrent.ConcurrentSk
  */
 class KeyValueSkipListSet implements NavigableSet<KeyValue> {
   private final ConcurrentNavigableMap<KeyValue, KeyValue> delegatee;
+  private volatile MemstoreBloomFilterContainer bloomFilterContainer = null;
+
+  KeyValueSkipListSet(final KeyValue.KVComparator c,
+      final MemstoreBloomFilterContainer bloomFilterContainer) {
+    this(c);
+    this.bloomFilterContainer = bloomFilterContainer;
+  }
 
   KeyValueSkipListSet(final KeyValue.KVComparator c) {
     this.delegatee = new ConcurrentSkipListMap<KeyValue, KeyValue>(c);
@@ -155,6 +162,7 @@ class KeyValueSkipListSet implements Nav
   }
 
   public boolean add(KeyValue e) {
+    this.bloomFilterContainer.add(e);
     return this.delegatee.put(e, e) == null;
   }
 
@@ -202,4 +210,8 @@ class KeyValueSkipListSet implements Nav
   public <T> T[] toArray(T[] a) {
     throw new UnsupportedOperationException("Not implemented");
   }
-}
\ No newline at end of file
+
+  public boolean containsRowPrefixForKeyValue(KeyValue kv) {
+    return this.bloomFilterContainer.containsRowPrefixForKeyValue(kv);
+  }
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1569877&r1=1569876&r2=1569877&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
Wed Feb 19 19:18:30 2014
@@ -35,6 +35,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
@@ -99,6 +100,7 @@ public class MemStore implements HeapSiz
   // Keep track of the total size of KVs not just the
   // bytes stored in the MSLAB
   private final AtomicLong successfullyAllocatedKvBytes;
+  private final HColumnDescriptor familyDesc;
 
   /**
    * Default constructor. Used for tests.
@@ -108,18 +110,30 @@ public class MemStore implements HeapSiz
   }
 
   /**
+   * Constructor which defaults the familyDescriptor to null and pass on
+   * the other arguments as is.
+   * @param conf
+   * @param c
+   * @param familyDesc
+   */
+  public MemStore(final Configuration conf, final KeyValue.KVComparator c) {
+    this(conf, c, null);
+  }
+
+  /**
    * Constructor.
    * @param c Comparator
    */
   public MemStore(final Configuration conf,
-      final KeyValue.KVComparator c) {
+      final KeyValue.KVComparator c, HColumnDescriptor familyDesc) {
     this.conf = conf;
     this.comparator = c;
     this.comparatorIgnoreTimestamp =
       this.comparator.getComparatorIgnoringTimestamps();
     this.comparatorIgnoreType = this.comparator.getComparatorIgnoringType();
-    this.kvset = new KeyValueSkipListSet(c);
-    this.snapshot = new KeyValueSkipListSet(c);
+    this.familyDesc = familyDesc;
+    this.kvset = this.createNewKVSet();
+    this.snapshot = this.createNewKVSet();
     timeRangeTracker = new TimeRangeTracker();
     snapshotTimeRangeTracker = new TimeRangeTracker();
     this.size = new AtomicLong(DEEP_OVERHEAD);
@@ -174,6 +188,16 @@ public class MemStore implements HeapSiz
              Math.min(smallestSeqNumberVal, seqNum)));
   }
 
+  private KeyValueSkipListSet createNewKVSet() {
+    int rowPrefixLength = -1;
+    if (familyDesc != null) {
+      rowPrefixLength = familyDesc.getRowPrefixLengthForBloom();
+    }
+    return new KeyValueSkipListSet(this.comparator,
+        new MemstoreBloomFilterContainer.Builder(conf)
+          .withRowPrefixFilter(rowPrefixLength).create());
+  }
+
   /**
    * Creates a snapshot of the current memstore.
    * Snapshot must be cleared by call to {@link #clearSnapshot(SortedSet<KeyValue>)}
@@ -190,7 +214,8 @@ public class MemStore implements HeapSiz
       } else {
         if (!this.kvset.isEmpty()) {
           this.snapshot = this.kvset;
-          this.kvset = new KeyValueSkipListSet(this.comparator);
+          this.kvset = createNewKVSet();
+
           // Reset the smallest sequence number
           this.smallestSeqNumber.set(Long.MAX_VALUE);
           this.snapshotTimeRangeTracker = this.timeRangeTracker;
@@ -244,7 +269,7 @@ public class MemStore implements HeapSiz
       // OK. Passed in snapshot is same as current snapshot.  If not-empty,
       // create a new snapshot and let the old one go.
       if (!ss.isEmpty()) {
-        this.snapshot = new KeyValueSkipListSet(this.comparator);
+        this.snapshot = createNewKVSet();
         this.snapshotTimeRangeTracker = new TimeRangeTracker();
       }
       if (this.snapshotAllocator != null) {
@@ -841,14 +866,13 @@ public class MemStore implements HeapSiz
 
     @Override
     public boolean passesRowKeyPrefixBloomFilter(KeyValue kv) {
-      // Need to fix this behavior and actually return a result
-      // from a bloom filter backed by the memstore.
-      return true;
+      return this.kvsetAtCreation.containsRowPrefixForKeyValue(kv)
+          || this.snapshotAtCreation.containsRowPrefixForKeyValue(kv);
     }
   }
 
   public final static long FIXED_OVERHEAD = ClassSize.align(
-      ClassSize.OBJECT + (18 * ClassSize.REFERENCE));
+      ClassSize.OBJECT + (19 * ClassSize.REFERENCE));
 
   public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilter.java?rev=1569877&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilter.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilter.java
Wed Feb 19 19:18:30 2014
@@ -0,0 +1,48 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * An interface which provides the interface to be followed by the set of
+ * in memory Bloom filters backed by the memstore.
+ *
+ * In general the in memory bloom filters for the memstore would be containing
+ * two bloom filters where one is backed by the current working memstore and the
+ * other is backed by the snapshot memstore.
+ */
+public interface MemstoreBloomFilter {
+
+  /**
+   * Adds the key value to the bloom filter. Specific implementations can
+   * override the way the key value is inserted into the bloom filter.
+   * @param kv
+   */
+  public void add(KeyValue kv);
+
+  /**
+   * Returns whether the given kv is present in the current active bloom filter.
+   * @param kv
+   * @return
+   */
+  public boolean contains(KeyValue kv);
+}

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterBase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterBase.java?rev=1569877&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterBase.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterBase.java
Wed Feb 19 19:18:30 2014
@@ -0,0 +1,62 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Hash;
+
+public class MemstoreBloomFilterBase implements MemstoreBloomFilter{
+  protected final ConcurrentHashMap<Integer, Boolean> map;
+  final Hash hash;
+
+  MemstoreBloomFilterBase(Configuration conf) {
+    map = new ConcurrentHashMap<Integer, Boolean>();
+    hash = Hash.getInstance(conf);
+  }
+
+  private int hashKey(byte[] buffer, int offset, int length) {
+    int h = hash.hash(buffer, offset, length, 0);
+    h = hash.hash(buffer, offset, length, h);
+    return h;
+  }
+
+  protected void addToMap(byte[] buffer, int offset, int length) {
+    map.put(hashKey(buffer, offset, length), true);
+  }
+
+  protected boolean containsInMap(byte[] buffer, int offset, int length) {
+    return map.containsKey(hashKey(buffer, offset, length));
+  }
+
+  @Override
+  public void add(KeyValue kv) {
+    // Empty add method. Sub classes should implement the respective add method.
+  }
+
+  @Override
+  public boolean contains(KeyValue kv) {
+    // Sub classes should implement the respective add method.
+    return true;
+  }
+}

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterContainer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterContainer.java?rev=1569877&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterContainer.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreBloomFilterContainer.java
Wed Feb 19 19:18:30 2014
@@ -0,0 +1,100 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.BloomFilterFactory;
+
+/**
+ *
+ * Container class which holds several MemstoreBloomFilters and applies the
+ * add, snapshot and clearSnapshot functions on all the members.
+ *
+ * Adding a new BloomFilter to this container will require us to
+ * add the corresponding builder method and respective contains method.
+ */
+public class MemstoreBloomFilterContainer {
+  private final MemstoreRowPrefixBloomFilter rowPrefixBloomFilter;
+  private final boolean enabled;
+  private static final Log LOG =
+    LogFactory.getLog(MemstoreBloomFilterContainer.class);
+  private MemstoreBloomFilterContainer(
+      MemstoreRowPrefixBloomFilter rowPrefixBloomFilter,
+      Configuration conf) {
+    this.rowPrefixBloomFilter = rowPrefixBloomFilter;
+    this.enabled = conf.getBoolean(HConstants.IN_MEMORY_BLOOM_ENABLED,
+        HConstants.DEFAULT_IN_MEMORY_BLOOM_ENABLED);
+  }
+
+  public void add(KeyValue kv) {
+    if (!enabled) return;
+    if (rowPrefixBloomFilter != null) rowPrefixBloomFilter.add(kv);
+  }
+
+  public boolean containsRowPrefixForKeyValue(KeyValue kv) {
+    if (!enabled) return true;
+    if (rowPrefixBloomFilter != null) {
+      return rowPrefixBloomFilter.contains(kv);
+    }
+    return true;
+  }
+
+  public static class Builder {
+    private MemstoreRowPrefixBloomFilter rowPrefixBloomFilter = null;
+    private Configuration conf;
+    public Builder(Configuration conf) {
+      this.conf = conf;
+    }
+
+    /**
+     * Adds a MemstoreRowPrefixBloomFilter to the MemstoreBloomFilterContainer.
+     * @param rowPrefix
+     * @return
+     */
+    public Builder withRowPrefixFilter(int rowPrefix) {
+      MemstoreRowPrefixBloomFilter rpbf = null;
+      if (conf.getBoolean(
+          BloomFilterFactory.IO_STOREFILE_ROWKEYPREFIX_BLOOM_ENABLED, false) &&
+          rowPrefix > 0) {
+        rpbf = new MemstoreRowPrefixBloomFilter(rowPrefix, this.conf);
+      }
+      if (rpbf == null) {
+        LOG.info("Row Key Prefix bloom filter disabled in table schema");
+      }
+      rowPrefixBloomFilter = rpbf;
+      return this;
+    }
+
+    /**
+     * Creates the actual MemstoreBloomFilterContainer
+     * @return
+     */
+    public MemstoreBloomFilterContainer create() {
+      return new MemstoreBloomFilterContainer(
+          this.rowPrefixBloomFilter,
+          this.conf);
+    }
+  }
+}

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreRowPrefixBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreRowPrefixBloomFilter.java?rev=1569877&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreRowPrefixBloomFilter.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreRowPrefixBloomFilter.java
Wed Feb 19 19:18:30 2014
@@ -0,0 +1,52 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * In memory temporary bloom filter present in the memstore.
+ * The bloom filter will report elements irrespective of the read point.
+ * TODO : Address the above point.
+ *
+ */
+public class MemstoreRowPrefixBloomFilter extends MemstoreBloomFilterBase {
+  private final int prefixLength;
+
+  MemstoreRowPrefixBloomFilter(int prefixLength, Configuration conf) {
+    super(conf);
+    this.prefixLength = prefixLength;
+  }
+
+  @Override
+  public void add(KeyValue kv) {
+    if (kv.getRowLength() < this.prefixLength) return;
+    this.addToMap(kv.getBuffer(), kv.getRowOffset(), prefixLength);
+  }
+
+  @Override
+  public boolean contains(KeyValue kv) {
+    if (kv.getRowLength() < this.prefixLength) return false;
+    return this.containsInMap(kv.getBuffer(), kv.getRowOffset(),
+        this.prefixLength);
+  }
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1569877&r1=1569876&r2=1569877&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Wed
Feb 19 19:18:30 2014
@@ -262,7 +262,7 @@ public class Store extends SchemaConfigu
     LOG.info("time to purge deletes set to " + timeToPurgeDeletes +
         "ms in store " + this);
 
-    this.memstore = new MemStore(conf, this.comparator);
+    this.memstore = new MemStore(conf, this.comparator, this.family);
     this.storeNameStr = getColumnFamilyName();
 
     // Setting up cache configuration for this family

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1569877&r1=1569876&r2=1569877&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Wed Feb 19 19:18:30 2014
@@ -29,9 +29,17 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.CompoundRowPrefixFilter;
+import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.io.Reference.Range;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.*;
@@ -408,6 +416,80 @@ public class TestStoreFile extends HBase
     bloomWriteRead(writer, fs);
   }
 
+  public void testMemstoreRowPrefixBloomFilter() throws Exception {
+    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_ROWKEYPREFIX_BLOOM_ENABLED,
+        true);
+    conf.setBoolean(HConstants.IN_MEMORY_BLOOM_ENABLED, true);
+
+    byte[] FAMILY = Bytes.toBytes("testMemstoreRowPrefixBloomFiltercf");
+    byte[] TABLE = Bytes.toBytes("testMemstoreRowPrefixBloomFilter");
+    int ROW_PREFIX_LENGTH = 4;
+    //Create table
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
+        .setMaxVersions(Integer.MAX_VALUE);
+    hcd.setRowKeyPrefixLengthForBloom(ROW_PREFIX_LENGTH);
+
+    HTableDescriptor htd = new HTableDescriptor(TABLE);
+    htd.addFamily(hcd);
+    HRegionInfo info = new HRegionInfo(htd, null, null, false);
+    Path tableDir = new Path(ROOT_DIR, "testMemstoreRowPrefixBloomFilter");
+    Path rootDir = new Path(ROOT_DIR);
+    this.fs.mkdirs(tableDir);
+    HRegion region = HRegion.createHRegion(info, rootDir, conf);
+
+    byte[][] prefixes = new byte[][] {
+        Bytes.toBytes("abcd"),
+        Bytes.toBytes("mnop"),
+        Bytes.toBytes("xyzd")};
+    KeyValue[] kvs = new KeyValue[] {
+        KeyValue.createFirstOnRow(prefixes[0]),
+        KeyValue.createFirstOnRow(prefixes[1]),
+        KeyValue.createFirstOnRow(prefixes[2])};
+    Put p = new Put(prefixes[0]);
+    p.add(FAMILY, null, prefixes[0]);
+
+    region.put(p);
+    KeyValueScanner kvScannerBeforeFlush =
+        region.getStore(FAMILY).memstore.getScanners().get(0);
+    assertTrue(kvScannerBeforeFlush.passesRowKeyPrefixBloomFilter(kvs[0]));
+    assertFalse(kvScannerBeforeFlush.passesRowKeyPrefixBloomFilter(kvs[1]));
+
+    //Flush
+    region.flushcache();
+    KeyValueScanner kvScannerAfterFlush =
+        region.getStore(FAMILY).memstore.getScanners().get(0);
+    assertFalse(kvScannerAfterFlush.passesRowKeyPrefixBloomFilter(kvs[0]));
+    assertFalse(kvScannerAfterFlush.passesRowKeyPrefixBloomFilter(kvs[1]));
+
+    p = new Put(prefixes[1]);
+    p.add(FAMILY, null, prefixes[1]);
+    region.put(p);
+
+    p = new Put(prefixes[2]);
+    p.add(FAMILY, null, prefixes[2]);
+    region.put(p);
+
+    InternalScanner scanner = region.getScanner(new Scan().setFilter(
+        new CompoundRowPrefixFilter.Builder()
+          .addRowPrefix(prefixes[0])
+          .addRowPrefix(prefixes[1])
+          .create()
+    ));
+    List<KeyValue> results = Lists.newArrayList();
+
+    assertTrue(scanner.next(results));
+    Result r = new Result(results);
+    assertTrue(Bytes.equals(r.getRow(), prefixes[0]));
+
+    results.clear();
+
+    assertTrue(scanner.next(results));
+    r = new Result(results);
+    assertTrue(Bytes.equals(r.getRow(), prefixes[1]));
+
+    assertFalse(scanner.next(results));
+  }
+
   public void testRowKeyPrefixBloomFilter() throws Exception {
     FileSystem fs = FileSystem.getLocal(conf);
     final float ERROR_RATE = (float) 0.01;



Mime
View raw message