Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CCEAF183EF for ; Thu, 19 Nov 2015 03:37:51 +0000 (UTC) Received: (qmail 49982 invoked by uid 500); 19 Nov 2015 03:37:51 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 49939 invoked by uid 500); 19 Nov 2015 03:37:51 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 49930 invoked by uid 99); 19 Nov 2015 03:37:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Nov 2015 03:37:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 899D6DFF73; Thu, 19 Nov 2015 03:37:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chenheng@apache.org To: commits@hbase.apache.org Message-Id: <323487e40273426886da2e475b847608@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-14782 FuzzyRowFilter skips valid rows (Vladimir Rodionov) Date: Thu, 19 Nov 2015 03:37:51 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-1.0 40ae5bec2 -> 38256f0f6 HBASE-14782 FuzzyRowFilter skips valid rows (Vladimir Rodionov) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/38256f0f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/38256f0f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/38256f0f Branch: refs/heads/branch-1.0 Commit: 38256f0f63e9225d5b898847e0e5fb65858b0982 Parents: 40ae5be Author: chenheng Authored: Thu Nov 19 10:56:18 2015 +0800 Committer: chenheng Committed: Thu Nov 19 11:33:24 2015 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/filter/FuzzyRowFilter.java | 27 ++++++++++- .../hadoop/hbase/filter/TestFuzzyRowFilter.java | 8 +-- .../filter/TestFuzzyRowFilterEndToEnd.java | 51 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/38256f0f/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java index 1f125e5..3bc5b74 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java @@ -151,6 +151,7 @@ public class FuzzyRowFilter extends FilterBase { } // NOT FOUND -> seek next using hint lastFoundIndex = -1; + return ReturnCode.SEEK_NEXT_USING_HINT; } @@ -584,7 +585,31 @@ public class FuzzyRowFilter extends FilterBase { } } - return result; + return reverse? result: trimTrailingZeroes(result, fuzzyKeyMeta, toInc); + } + + /** + * For forward scanner, next cell hint should not contain any trailing zeroes + * unless they are part of fuzzyKeyMeta + * hint = '\x01\x01\x01\x00\x00' + * will skip valid row '\x01\x01\x01' + * + * @param result + * @param fuzzyKeyMeta + * @param toInc - position of incremented byte + * @return trimmed version of result + */ + + private static byte[] trimTrailingZeroes(byte[] result, byte[] fuzzyKeyMeta, int toInc) { + int off = fuzzyKeyMeta.length >= result.length? result.length -1: + fuzzyKeyMeta.length -1; + for( ; off >= 0; off--){ + if(fuzzyKeyMeta[off] != 0) break; + } + if (off < toInc) off = toInc; + byte[] retValue = new byte[off+1]; + System.arraycopy(result, 0, retValue, 0, retValue.length); + return retValue; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/38256f0f/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java index 4e44d2c..c259979 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java @@ -230,7 +230,7 @@ public class TestFuzzyRowFilter { new byte[]{0, 1, 2}, // fuzzy row new byte[]{0, -1, -1}, // mask new byte[]{1, 2, 1, 0, 1}, // current - new byte[]{2, 1, 2, 0, 0}); // expected next + new byte[]{2, 1, 2}); // expected next assertNext(false, new byte[]{0, 1, 2}, // fuzzy row @@ -242,13 +242,13 @@ public class TestFuzzyRowFilter { new byte[]{0, 1, 0, 2, 0}, // fuzzy row new byte[]{0, -1, 0, -1, 0}, // mask new byte[]{1, 0, 2, 0, 1}, // current - new byte[]{1, 1, 0, 2, 0}); // expected next + new byte[]{1, 1, 0, 2}); // expected next assertNext(false, new byte[]{1, 0, 1}, new byte[]{-1, 0, -1}, new byte[]{1, (byte) 128, 2, 0, 1}, - new byte[]{1, (byte) 129, 1, 0, 0}); + new byte[]{1, (byte) 129, 1}); assertNext(false, new byte[]{0, 1, 0, 1}, @@ -314,7 +314,7 @@ public class TestFuzzyRowFilter { new byte[]{1, 1, 0, 0}, new byte[]{-1, -1, 0, 0}, new byte[]{0, 1, 3, 2}, - new byte[]{1, 1, 0, 0}); + new byte[]{1, 1}); // No next for this one Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( http://git-wip-us.apache.org/repos/asf/hbase/blob/38256f0f/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java index cc2f7b8..9179549 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java @@ -110,6 +110,57 @@ public class TestFuzzyRowFilterEndToEnd { // Nothing to do. } + @Test + public void testHBASE14782() throws IOException + { + String cf = "f"; + String cq = "q"; + String table = "HBASE14872"; + + Table ht = + TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE); + // Load data + String[] rows = new String[]{ + "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00", + "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", + "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00", + "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00", + "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", + "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", + }; + + String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX"; + + for(int i=0; i < rows.length; i++){ + Put p = new Put(Bytes.toBytesBinary(rows[i])); + p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes()); + ht.put(p); + } + + Put p = new Put(Bytes.toBytesBinary(badRow)); + p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes()); + ht.put(p); + + TEST_UTIL.flush(); + + List> data = new ArrayList>(); + byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044"); + byte[] mask = new byte[] { 1,0,0,0}; + data.add(new Pair(fuzzyKey, mask)); + FuzzyRowFilter filter = new FuzzyRowFilter(data); + + Scan scan = new Scan(); + scan.setFilter(filter); + + ResultScanner scanner = ht.getScanner(scan); + int total = 0; + while(scanner.next() != null){ + total++; + } + assertEquals(rows.length, total); + TEST_UTIL.deleteTable(TableName.valueOf(table)); + } + @Test public void testEndToEnd() throws Exception { String cf = "f";