hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenh...@apache.org
Subject hbase git commit: HBASE-14782 FuzzyRowFilter skips valid rows (Vladimir Rodionov)
Date Thu, 19 Nov 2015 03:55:16 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 5fe779a61 -> 735a5bd87


HBASE-14782 FuzzyRowFilter skips valid rows (Vladimir Rodionov)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/735a5bd8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/735a5bd8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/735a5bd8

Branch: refs/heads/0.98
Commit: 735a5bd87176c57d611e21b2831200e094636feb
Parents: 5fe779a
Author: chenheng <chenheng@apache.org>
Authored: Thu Nov 19 11:53:53 2015 +0800
Committer: chenheng <chenheng@apache.org>
Committed: Thu Nov 19 11:53:53 2015 +0800

----------------------------------------------------------------------
 .../hadoop/hbase/filter/FuzzyRowFilter.java     | 27 ++++++++++-
 .../hadoop/hbase/filter/TestFuzzyRowFilter.java |  8 +--
 .../filter/TestFuzzyRowFilterEndToEnd.java      | 51 ++++++++++++++++++++
 3 files changed, 81 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/735a5bd8/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index d6ad3a0..312af97 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -151,6 +151,7 @@ public class FuzzyRowFilter extends FilterBase {
     }
     // NOT FOUND -> seek next using hint
     lastFoundIndex = -1;
+
     return ReturnCode.SEEK_NEXT_USING_HINT;
 
   }
@@ -584,7 +585,31 @@ public class FuzzyRowFilter extends FilterBase {
       }
     }
 
-    return result;
+    return reverse? result: trimTrailingZeroes(result, fuzzyKeyMeta, toInc);
+  }
+
+  /**
+   * For forward scanner, next cell hint should  not contain any trailing zeroes
+   * unless they are part of fuzzyKeyMeta
+   * hint = '\x01\x01\x01\x00\x00'
+   * will skip valid row '\x01\x01\x01'
+   * 
+   * @param result
+   * @param fuzzyKeyMeta
+   * @param toInc - position of incremented byte
+   * @return trimmed version of result
+   */
+  
+  private static byte[] trimTrailingZeroes(byte[] result, byte[] fuzzyKeyMeta, int toInc)
{
+    int off = fuzzyKeyMeta.length >= result.length? result.length -1:
+           fuzzyKeyMeta.length -1;  
+    for( ; off >= 0; off--){
+      if(fuzzyKeyMeta[off] != 0) break;
+    }
+    if (off < toInc)  off = toInc;
+    byte[] retValue = new byte[off+1];
+    System.arraycopy(result, 0, retValue, 0, retValue.length);
+    return retValue;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/735a5bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
index 8799ebf..e65f91c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
@@ -229,7 +229,7 @@ public class TestFuzzyRowFilter {
             new byte[]{0, 1, 2}, // fuzzy row
             new byte[]{0, -1, -1}, // mask
             new byte[]{1, 2, 1, 0, 1}, // current
-            new byte[]{2, 1, 2, 0, 0}); // expected next
+            new byte[]{2, 1, 2}); // expected next
 
     assertNext(false,
             new byte[]{0, 1, 2}, // fuzzy row
@@ -241,13 +241,13 @@ public class TestFuzzyRowFilter {
             new byte[]{0, 1, 0, 2, 0}, // fuzzy row
             new byte[]{0, -1, 0, -1, 0}, // mask
             new byte[]{1, 0, 2, 0, 1}, // current
-            new byte[]{1, 1, 0, 2, 0}); // expected next
+            new byte[]{1, 1, 0, 2}); // expected next
 
     assertNext(false,
             new byte[]{1, 0, 1},
             new byte[]{-1, 0, -1},
             new byte[]{1, (byte) 128, 2, 0, 1},
-            new byte[]{1, (byte) 129, 1, 0, 0});
+            new byte[]{1, (byte) 129, 1});
 
     assertNext(false,
             new byte[]{0, 1, 0, 1},
@@ -313,7 +313,7 @@ public class TestFuzzyRowFilter {
             new byte[]{1, 1, 0, 0},
             new byte[]{-1, -1, 0, 0},
             new byte[]{0, 1, 3, 2},
-            new byte[]{1, 1, 0, 0});
+            new byte[]{1, 1});
 
     // No next for this one
     Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(

http://git-wip-us.apache.org/repos/asf/hbase/blob/735a5bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
index c31bb93..533c4fc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
@@ -109,6 +109,57 @@ public class TestFuzzyRowFilterEndToEnd {
     // Nothing to do.
   }
 
+  @Test  
+  public void testHBASE14782() throws IOException
+  {
+    String cf = "f";
+    String cq = "q";
+    String table = "HBASE14872";
+
+    HTable ht =
+        TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
+    // Load data
+    String[] rows = new String[]{
+        "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
+        "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", 
+        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
+        "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
+        "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", 
+        "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", 
+    };
+    
+    String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
+    
+    for(int i=0; i < rows.length; i++){
+      Put p = new Put(Bytes.toBytesBinary(rows[i]));
+      p.add(cf.getBytes(), cq.getBytes(), "value".getBytes());
+      ht.put(p);            
+    }
+    
+    Put p = new Put(Bytes.toBytesBinary(badRow));
+    p.add(cf.getBytes(), cq.getBytes(), "value".getBytes());
+    ht.put(p);            
+
+    TEST_UTIL.flush();
+
+    List<Pair<byte[], byte[]>> data =  new ArrayList<Pair<byte[], byte[]>>();
+    byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
+    byte[] mask = new byte[] { 1,0,0,0};
+    data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
+    FuzzyRowFilter filter = new FuzzyRowFilter(data);
+    
+    Scan scan = new Scan();
+    scan.setFilter(filter);
+    
+    ResultScanner scanner = ht.getScanner(scan);
+    int total = 0;
+    while(scanner.next() != null){
+      total++;
+    }    
+    assertEquals(rows.length, total);
+    TEST_UTIL.deleteTable(TableName.valueOf(table));
+  }
+  
   @Test
   public void testEndToEnd() throws Exception {
     String cf = "f";


Mime
View raw message