hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-12817 Data missing while scanning using PREFIX_TREE data block encoding (Duo Zhang)
Date Thu, 08 Jan 2015 03:35:34 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 3a71fc93e -> 6ef814e6d


HBASE-12817 Data missing while scanning using PREFIX_TREE data block encoding (Duo Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6ef814e6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6ef814e6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6ef814e6

Branch: refs/heads/0.98
Commit: 6ef814e6d0bae2ba081a2c7a396f523689e7a15a
Parents: 3a71fc9
Author: tedyu <yuzhihong@gmail.com>
Authored: Wed Jan 7 19:35:19 2015 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Jan 7 19:35:19 2015 -0800

----------------------------------------------------------------------
 .../codec/prefixtree/PrefixTreeSeeker.java      |   8 +-
 .../PrefixTreeArrayReversibleScanner.java       |   1 +
 .../decode/PrefixTreeArrayScanner.java          |  21 +-
 .../hbase/io/encoding/TestPrefixTree.java       | 253 ++++++++++---------
 4 files changed, 134 insertions(+), 149 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/6ef814e6/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
index c077357..ac12b70 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
@@ -120,14 +120,9 @@ public class PrefixTreeSeeker implements EncodedSeeker {
 
   @Override
   public boolean next() {
-    boolean advance = ptSearcher.advance();
-    if (ptSearcher.hasMovedToPreviousAsPartOfSeek()) {
-      ptSearcher.setMovedToPreviousAsPartOfSeek(false);
-    }
-    return advance;
+    return ptSearcher.advance();
   }
 
-//  @Override
   public boolean advance() {
     return ptSearcher.advance();
   }
@@ -210,7 +205,6 @@ public class PrefixTreeSeeker implements EncodedSeeker {
     if(CellScannerPosition.AFTER == position){
       if(!ptSearcher.isBeforeFirst()){
         ptSearcher.previous();
-        ptSearcher.setMovedToPreviousAsPartOfSeek(true);
       }
       return 1;
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/6ef814e6/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
index 584a8fb..5a817cb 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
@@ -115,6 +115,7 @@ public class PrefixTreeArrayReversibleScanner extends PrefixTreeArrayScanner
imp
         }
       }
       if (currentRowNode.hasOccurrences()) {// escape clause
+        currentRowNode.resetFanIndex();
         return true;// found some values
       }
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/6ef814e6/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
index 44615c7..d1859da 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
@@ -61,7 +61,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
 
   protected boolean nubCellsRemain;
   protected int currentCellIndex;
-  protected boolean movedToPrevious;
 
 
   /*********************** construct ******************************/
@@ -262,12 +261,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements
CellScanne
           return true;
         }
       } else {
-        if (movedToPrevious && currentRowNode.hasOccurrences()
-            && currentRowNode.getFanIndex() == getNextFanIndex()) {
-          followFan(getNextFanIndex());
-        } else {
-          discardCurrentRowNode(true);
-        }
+        discardCurrentRowNode(true);
       }
     }
     return false;// went past the end
@@ -304,10 +298,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements
CellScanne
     followFan(currentFanPosition);
   }
 
-  protected int getNextFanIndex() {
-    return rowNodes[rowNodeStackIndex + 1].getFanIndex();
-  }
-
   protected void followNextFan() {
     int nextFanPosition = currentRowNode.getFanIndex() + 1;
     followFan(nextFanPosition);
@@ -538,13 +528,4 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements
CellScanne
   public int getTagBufferLength() {
     return tagsBuffer.length;
   }
-
-  public void setMovedToPreviousAsPartOfSeek(boolean movedToPrevious) {
-    this.movedToPrevious = movedToPrevious;
-  }
-
-  public boolean hasMovedToPreviousAsPartOfSeek() {
-    return this.movedToPrevious;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/6ef814e6/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
index fd1e933..2aab662 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
@@ -18,32 +18,37 @@
 
 package org.apache.hadoop.hbase.io.encoding;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
 
-@Category(MediumTests.class)
+@Category(SmallTests.class)
 public class TestPrefixTree {
+
   private static final String row4 = "a-b-B-2-1402397300-1402416535";
   private static final byte[] row4_bytes = Bytes.toBytes(row4);
   private static final String row3 = "a-b-A-1-1402397227-1402415999";
@@ -52,129 +57,133 @@ public class TestPrefixTree {
   private static final byte[] row2_bytes = Bytes.toBytes(row2);
   private static final String row1 = "a-b-A-1";
   private static final byte[] row1_bytes = Bytes.toBytes(row1);
-  public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
   private final static byte[] fam = Bytes.toBytes("cf_1");
   private final static byte[] qual1 = Bytes.toBytes("qf_1");
   private final static byte[] qual2 = Bytes.toBytes("qf_2");
-  public static Configuration conf;
-
-  @Rule
-  public final TestName TEST_NAME = new TestName();
-
-  @BeforeClass
-  public static void setupBeforeClass() throws Exception {
-    // setup configuration
-    conf = TEST_UTIL.getConfiguration();
-    conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
-    conf.setBoolean("hbase.online.schema.update.enable", true);
-    conf.setInt("hbase.client.scanner.timeout.period", 600000);
-    TEST_UTIL.startMiniCluster(2);
+
+  private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+
+  private HRegion region;
+
+  @Before
+  public void setUp() throws Exception {
+    TableName tableName = TableName.valueOf(getClass().getSimpleName());
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(fam).setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE));
+    HRegionInfo info = new HRegionInfo(tableName, null, null, false);
+    Path path = testUtil.getDataTestDir(getClass().getSimpleName());
+    region = HRegion.createHRegion(info, path, testUtil.getConfiguration(), htd);
   }
 
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
+  @After
+  public void tearDown() throws Exception {
+    region.close(true);
+    testUtil.cleanupTestDir();
   }
 
   @Test
   public void testHBASE11728() throws Exception {
-    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
-    HTable table = null;
-    try {
-      HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
-      HColumnDescriptor colDesc = new HColumnDescriptor(fam);
-      HTableDescriptor desc = new HTableDescriptor(tableName);
-      colDesc.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE);
-      colDesc.setTimeToLive(15552000);
-      desc.addFamily(colDesc);
-      hBaseAdmin.createTable(desc);
-      table = new HTable(conf, tableName);
-      Put put = new Put(Bytes.toBytes("a-b-0-0"));
-      put.add(fam, qual1, Bytes.toBytes("c1-value"));
-      table.put(put);
-      table.flushCommits();
-      put = new Put(row1_bytes);
-      put.add(fam, qual1, Bytes.toBytes("c1-value"));
-      table.put(put);
-      table.flushCommits();
-      put = new Put(row2_bytes);
-      put.add(fam, qual2, Bytes.toBytes("c2-value"));
-      table.put(put);
-      table.flushCommits();
-      put = new Put(row3_bytes);
-      put.add(fam, qual2, Bytes.toBytes("c2-value-2"));
-      table.put(put);
-      table.flushCommits();
-      put = new Put(row4_bytes);
-      put.add(fam, qual2, Bytes.toBytes("c2-value-3"));
-      table.put(put);
-      table.flushCommits();
-      hBaseAdmin.flush(tableName.getNameAsString());
-      String[] rows = new String[3];
-      rows[0] = row1;
-      rows[1] = row2;
-      rows[2] = row3;
-      Scan scan = new Scan();
-      scan.setStartRow(row1_bytes);
-      scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
-      ResultScanner scanner = table.getScanner(scan);
-      Result[] next = scanner.next(10);
-      assertEquals(3, next.length);
-      int i = 0;
-      for (Result res : next) {
-        CellScanner cellScanner = res.cellScanner();
-        while (cellScanner.advance()) {
-          assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
-              .current().getRowOffset(), cellScanner.current().getRowLength()));
-        }
-        i++;
+    Put put = new Put(Bytes.toBytes("a-b-0-0"));
+    put.add(fam, qual1, Bytes.toBytes("c1-value"));
+    region.put(put);
+    put = new Put(row1_bytes);
+    put.add(fam, qual1, Bytes.toBytes("c1-value"));
+    region.put(put);
+    put = new Put(row2_bytes);
+    put.add(fam, qual2, Bytes.toBytes("c2-value"));
+    region.put(put);
+    put = new Put(row3_bytes);
+    put.add(fam, qual2, Bytes.toBytes("c2-value-2"));
+    region.put(put);
+    put = new Put(row4_bytes);
+    put.add(fam, qual2, Bytes.toBytes("c2-value-3"));
+    region.put(put);
+    region.flushcache();
+    String[] rows = new String[3];
+    rows[0] = row1;
+    rows[1] = row2;
+    rows[2] = row3;
+    byte[][] val = new byte[3][];
+    val[0] = Bytes.toBytes("c1-value");
+    val[1] = Bytes.toBytes("c2-value");
+    val[2] = Bytes.toBytes("c2-value-2");
+    Scan scan = new Scan();
+    scan.setStartRow(row1_bytes);
+    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+
+    RegionScanner scanner = region.getScanner(scan);
+    List<Cell> cells = new ArrayList<Cell>();
+    for (int i = 0; i < 3; i++) {
+      assertEquals(i < 2, scanner.next(cells));
+      CellScanner cellScanner = Result.create(cells).cellScanner();
+      while (cellScanner.advance()) {
+        assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
+            .current().getRowOffset(), cellScanner.current().getRowLength()));
+        assertEquals(Bytes.toString(val[i]), Bytes.toString(cellScanner.current().getValueArray(),
+          cellScanner.current().getValueOffset(), cellScanner.current().getValueLength()));
       }
-      scanner.close();
-      // Add column
-      scan = new Scan();
-      scan.addColumn(fam, qual2);
-      scan.setStartRow(row1_bytes);
-      scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
-      scanner = table.getScanner(scan);
-      next = scanner.next(10);
-      assertEquals(2, next.length);
-      i = 1;
-      for (Result res : next) {
-        CellScanner cellScanner = res.cellScanner();
-        while (cellScanner.advance()) {
-          assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
-              .current().getRowOffset(), cellScanner.current().getRowLength()));
-        }
-        i++;
+      cells.clear();
+    }
+    scanner.close();
+
+    // Add column
+    scan = new Scan();
+    scan.addColumn(fam, qual2);
+    scan.setStartRow(row1_bytes);
+    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+    scanner = region.getScanner(scan);
+    for (int i = 1; i < 3; i++) {
+      assertEquals(i < 2, scanner.next(cells));
+      CellScanner cellScanner = Result.create(cells).cellScanner();
+      while (cellScanner.advance()) {
+        assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
+            .current().getRowOffset(), cellScanner.current().getRowLength()));
       }
-      scanner.close();
-      i = 1;
-      scan = new Scan();
-      scan.addColumn(fam, qual2);
-      scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
-      scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
-      scanner = table.getScanner(scan);
-      next = scanner.next(10);
-      assertEquals(2, next.length);
-      for (Result res : next) {
-        CellScanner cellScanner = res.cellScanner();
-        while (cellScanner.advance()) {
-          assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
-              .current().getRowOffset(), cellScanner.current().getRowLength()));
-        }
-        i++;
+      cells.clear();
+    }
+    scanner.close();
+
+    scan = new Scan();
+    scan.addColumn(fam, qual2);
+    scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
+    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+    scanner = region.getScanner(scan);
+    for (int i = 1; i < 3; i++) {
+      assertEquals(i < 2, scanner.next(cells));
+      CellScanner cellScanner = Result.create(cells).cellScanner();
+      while (cellScanner.advance()) {
+        assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
+            .current().getRowOffset(), cellScanner.current().getRowLength()));
       }
-      scanner.close();
-      scan = new Scan();
-      scan.addColumn(fam, qual2);
-      scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
-      scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
-      scanner = table.getScanner(scan);
-      next = scanner.next(10);
-      assertEquals(1, next.length);
-      scanner.close();
-    } finally {
-      table.close();
+      cells.clear();
+    }
+    scanner.close();
+
+    scan = new Scan();
+    scan.addColumn(fam, qual2);
+    scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
+    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+    scanner = region.getScanner(scan);
+    assertFalse(scanner.next(cells));
+    assertFalse(cells.isEmpty());
+    scanner.close();
+  }
+
+  @Test
+  public void testHBASE12817() throws IOException {
+    for (int i = 0; i < 100; i++) {
+      region.put(new Put(Bytes.toBytes("obj" + (2900 + i))).add(fam, qual1, Bytes.toBytes(i)));
     }
+    region.put(new Put(Bytes.toBytes("obj299")).add(fam, qual1, Bytes.toBytes("whatever")));
+    region.put(new Put(Bytes.toBytes("obj29")).add(fam, qual1, Bytes.toBytes("whatever")));
+    region.put(new Put(Bytes.toBytes("obj2")).add(fam, qual1, Bytes.toBytes("whatever")));
+    region.put(new Put(Bytes.toBytes("obj3")).add(fam, qual1, Bytes.toBytes("whatever")));
+    region.flushcache();
+    Scan scan = new Scan(Bytes.toBytes("obj29995"));
+    RegionScanner scanner = region.getScanner(scan);
+    List<Cell> cells = new ArrayList<Cell>();
+    assertFalse(scanner.next(cells));
+    assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow());
   }
 }


Mime
View raw message