hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject hbase git commit: HBASE-14355 Scan different TimeRange for each column family - fixing memstore bug
Date Thu, 26 Nov 2015 06:04:29 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 89e34f36a -> 0cfd893e1


HBASE-14355 Scan different TimeRange for each column family - fixing memstore bug

Signed-off-by: stack <stack@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0cfd893e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0cfd893e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0cfd893e

Branch: refs/heads/branch-1.2
Commit: 0cfd893e12ee56cf025d950e4163b00dff6985fc
Parents: 89e34f3
Author: rahulgidwani <rahul.gidwani@flurry.com>
Authored: Wed Nov 25 14:43:12 2015 -0800
Committer: stack <stack@apache.org>
Committed: Wed Nov 25 22:04:21 2015 -0800

----------------------------------------------------------------------
 .../hbase/regionserver/DefaultMemStore.java     | 18 ++++++++---
 .../hbase/regionserver/ScanQueryMatcher.java    |  9 ++++--
 .../hbase/regionserver/TestDefaultMemStore.java | 32 ++++++++++++--------
 3 files changed, 39 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0cfd893e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 164638b..c929e71 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.util.ByteRange;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -665,12 +666,19 @@ public class DefaultMemStore implements MemStore {
 
   /**
    * Check if this memstore may contain the required keys
-   * @param scan
+   * @param scan scan
+   * @param store holds reference to cf
+   * @param oldestUnexpiredTS
    * @return False if the key definitely does not exist in this Memstore
    */
-  public boolean shouldSeek(Scan scan, long oldestUnexpiredTS) {
-    return (timeRangeTracker.includesTimeRange(scan.getTimeRange()) ||
-        snapshotTimeRangeTracker.includesTimeRange(scan.getTimeRange()))
+  public boolean shouldSeek(Scan scan, Store store, long oldestUnexpiredTS) {
+    byte[] cf = store.getFamily().getName();
+    TimeRange timeRange = scan.getColumnFamilyTimeRange().get(cf);
+    if (timeRange == null) {
+      timeRange = scan.getTimeRange();
+    }
+    return (timeRangeTracker.includesTimeRange(timeRange) ||
+        snapshotTimeRangeTracker.includesTimeRange(timeRange))
         && (Math.max(timeRangeTracker.getMaximumTimestamp(),
                      snapshotTimeRangeTracker.getMaximumTimestamp()) >=
             oldestUnexpiredTS);
@@ -943,7 +951,7 @@ public class DefaultMemStore implements MemStore {
 
     @Override
     public boolean shouldUseScanner(Scan scan, Store store, long oldestUnexpiredTS) {
-      return shouldSeek(scan, oldestUnexpiredTS);
+      return shouldSeek(scan, store, oldestUnexpiredTS);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/0cfd893e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index 032b4ce..9c91017 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -160,7 +160,12 @@ public class ScanQueryMatcher {
   public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
       ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS,
       long now, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
-    this.tr = scan.getTimeRange();
+    TimeRange timeRange = scan.getColumnFamilyTimeRange().get(scanInfo.getFamily());
+    if (timeRange == null) {
+      this.tr = scan.getTimeRange();
+    } else {
+      this.tr = timeRange;
+    }
     this.rowComparator = scanInfo.getComparator();
     this.regionCoprocessorHost = regionCoprocessorHost;
     this.deletes =  instantiateDeleteTracker();
@@ -275,7 +280,7 @@ public class ScanQueryMatcher {
    *      caused by a data corruption.
    */
   public MatchCode match(Cell cell) throws IOException {
-    if (filter != null && filter.filterAllRemaining()) {
+      if (filter != null && filter.filterAllRemaining()) {
       return MatchCode.DONE_SCAN;
     }
     int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,

http://git-wip-us.apache.org/repos/asf/hbase/blob/0cfd893e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index 897b724..c8d9ef4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -58,6 +58,9 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 /** memstore test case */
 @Category(MediumTests.class)
 public class TestDefaultMemStore extends TestCase {
@@ -741,29 +744,32 @@ public class TestDefaultMemStore extends TestCase {
   /**
    * Test to ensure correctness when using Memstore with multiple timestamps
    */
-  public void testMultipleTimestamps() throws IOException {
+  public void testMultipleTimestamps() throws Exception {
     long[] timestamps = new long[] {20,10,5,1};
     Scan scan = new Scan();
 
     for (long timestamp: timestamps)
       addRows(memstore,timestamp);
 
-    scan.setTimeRange(0, 2);
-    assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+    byte[] fam = Bytes.toBytes("fam");
+    HColumnDescriptor hcd = mock(HColumnDescriptor.class);
+    when(hcd.getName()).thenReturn(fam);
+    Store store = mock(Store.class);
+    when(store.getFamily()).thenReturn(hcd);
+    scan.setColumnFamilyTimeRange(fam, 0, 2);
+    assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE));
 
-    scan.setTimeRange(20, 82);
-    assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+    scan.setColumnFamilyTimeRange(fam, 20, 82);
+    assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE));
 
-    scan.setTimeRange(10, 20);
-    assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+    scan.setColumnFamilyTimeRange(fam, 10, 20);
+    assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE));
 
-    scan.setTimeRange(8, 12);
-    assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+    scan.setColumnFamilyTimeRange(fam, 8, 12);
+    assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE));
 
-    /*This test is not required for correctness but it should pass when
-     * timestamp range optimization is on*/
-    //scan.setTimeRange(28, 42);
-    //assertTrue(!memstore.shouldSeek(scan));
+    scan.setColumnFamilyTimeRange(fam, 28, 42);
+    assertTrue(!memstore.shouldSeek(scan, store, Long.MIN_VALUE));
   }
 
   ////////////////////////////////////


Mime
View raw message