hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1229808 - in /hbase/branches/0.92: ./ src/main/java/org/apache/hadoop/hbase/regionserver/ src/test/java/org/apache/hadoop/hbase/regionserver/
Date Tue, 10 Jan 2012 23:40:50 GMT
Author: larsh
Date: Tue Jan 10 23:40:49 2012
New Revision: 1229808

URL: http://svn.apache.org/viewvc?rev=1229808&view=rev
Log:
HBASE-5121 MajorCompaction may affect scan's correctness (chunhui shen and Lars H)

Modified:
    hbase/branches/0.92/CHANGES.txt
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java

Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1229808&r1=1229807&r2=1229808&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Tue Jan 10 23:40:49 2012
@@ -516,6 +516,7 @@ Release 0.92.0 - Unreleased
    HBASE-5152  Region is on service before completing initialization when doing rollback
of split,
                it will affect read correctness (Chunhui)
    HBASE-5137  MasterFileSystem.splitLog() should abort even if waitOnSafeMode() throws IOException
(Ram & Ted)
+   HBASE-5121  MajorCompaction may affect scan's correctness (chunhui shen and Lars H)
 
   TESTS
    HBASE-4492  TestRollingRestart fails intermittently

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java?rev=1229808&r1=1229807&r2=1229808&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
Tue Jan 10 23:40:49 2012
@@ -20,14 +20,14 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.KVComparator;
-
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.List;
 import java.util.PriorityQueue;
 
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+
 /**
  * Implements a heap merge across any number of KeyValueScanners.
  * <p>
@@ -135,7 +135,7 @@ public class KeyValueHeap implements Key
       return false;
     }
     InternalScanner currentAsInternal = (InternalScanner)this.current;
-    boolean mayContainsMoreRows = currentAsInternal.next(result, limit);
+    boolean mayContainMoreRows = currentAsInternal.next(result, limit);
     KeyValue pee = this.current.peek();
     /*
      * By definition, any InternalScanner must return false only when it has no
@@ -144,7 +144,7 @@ public class KeyValueHeap implements Key
      * more efficient to close scanners which are not needed than keep them in
      * the heap. This is also required for certain optimizations.
      */
-    if (pee == null || !mayContainsMoreRows) {
+    if (pee == null || !mayContainMoreRows) {
       this.current.close();
     } else {
       this.heap.add(this.current);

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1229808&r1=1229807&r2=1229808&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Tue Jan 10 23:40:49 2012
@@ -228,7 +228,9 @@ class StoreScanner implements KeyValueSc
   public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException
{
     //DebugPrint.println("SS.next");
 
-    checkReseek();
+    if (checkReseek()) {
+      return true;
+    }
 
     // if the heap was left null, then the scanners had previously run out anyways, close
and
     // return.
@@ -376,12 +378,25 @@ class StoreScanner implements KeyValueSc
     // Let the next() call handle re-creating and seeking
   }
 
-  private void checkReseek() throws IOException {
+  /**
+   * @return true if top of heap has changed (and KeyValueHeap has to try the
+   *         next KV)
+   * @throws IOException
+   */
+  private boolean checkReseek() throws IOException {
     if (this.heap == null && this.lastTop != null) {
       resetScannerStack(this.lastTop);
+      if (this.heap.peek() == null
+          || store.comparator.compare(this.lastTop, this.heap.peek()) != 0) {
+        LOG.debug("Storescanner.peek() is changed where before = "
+            + this.lastTop.toString() + ",and after = " + this.heap.peek());
+        this.lastTop = null;
+        return true;
+      }
       this.lastTop = null; // gone!
     }
     // else dont need to reseek
+    return false;
   }
 
   private void resetScannerStack(KeyValue lastTopKey) throws IOException {

Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=1229808&r1=1229807&r2=1229808&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
(original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
Tue Jan 10 23:40:49 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionIn
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -45,7 +46,6 @@ import org.apache.hadoop.hbase.filter.Wh
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 
 /**
  * Test of a long-lived scanner validating as we go.
@@ -81,6 +81,23 @@ public class TestScanner extends HBaseTe
 
   private HRegion r;
   private HRegionIncommon region;
+  
+  private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
+  final private byte[] col1, col2;
+
+  public TestScanner() throws Exception {
+    super();
+
+    firstRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
+    secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
+    // Increment the least significant character so we get to next row.
+    secondRowBytes[START_KEY_BYTES.length - 1]++;
+    thirdRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
+    thirdRowBytes[START_KEY_BYTES.length - 1]++;
+    thirdRowBytes[START_KEY_BYTES.length - 1]++;
+    col1 = "column1".getBytes(HConstants.UTF8_ENCODING);
+    col2 = "column2".getBytes(HConstants.UTF8_ENCODING);
+  }
 
   /**
    * Test basic stop row filter works.
@@ -470,6 +487,68 @@ public class TestScanner extends HBaseTe
     }
   }
 
+  /**
+   * Make sure scanner returns correct result when we run a major compaction
+   * with deletes.
+   * 
+   * @throws Exception
+   */
+  @SuppressWarnings("deprecation")
+  public void testScanAndConcurrentMajorCompact() throws Exception {
+    HTableDescriptor htd = createTableDescriptor(getName());
+    this.r = createNewHRegion(htd, null, null);
+    HRegionIncommon hri = new HRegionIncommon(r);
+
+    try {
+      addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
+          firstRowBytes, secondRowBytes);
+      addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
+          firstRowBytes, secondRowBytes);
+
+      Delete dc = new Delete(firstRowBytes);
+      /* delete column1 of firstRow */
+      dc.deleteColumns(fam1, col1);
+      r.delete(dc, null, true);
+      r.flushcache();
+
+      addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
+          secondRowBytes, thirdRowBytes);
+      addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
+          secondRowBytes, thirdRowBytes);
+      r.flushcache();
+
+      InternalScanner s = r.getScanner(new Scan());
+      // run a major compact, column1 of firstRow will be cleaned.
+      r.compactStores(true);
+
+      List<KeyValue> results = new ArrayList<KeyValue>();
+      s.next(results);
+
+      // make sure returns column2 of firstRow
+      assertTrue("result is not correct, keyValues : " + results,
+          results.size() == 1);
+      assertTrue(Bytes.BYTES_COMPARATOR.compare(firstRowBytes, results.get(0)
+          .getRow()) == 0);
+      assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(0)
+          .getFamily()) == 0);
+
+      results = new ArrayList<KeyValue>();
+      s.next(results);
+
+      // get secondRow
+      assertTrue(results.size() == 2);
+      assertTrue(Bytes.BYTES_COMPARATOR.compare(secondRowBytes, results.get(0)
+          .getRow()) == 0);
+      assertTrue(Bytes.BYTES_COMPARATOR.compare(fam1, results.get(0)
+          .getFamily()) == 0);
+      assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(1)
+          .getFamily()) == 0);
+    } finally {
+      this.r.close();
+      this.r.getLog().closeAndDelete();
+    }
+  }
+
 
   /*
    * @param hri Region



Mime
View raw message