hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1584514 - in /hbase/branches/0.94/src: docbkx/book.xml main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
Date Fri, 04 Apr 2014 04:30:25 GMT
Author: larsh
Date: Fri Apr  4 04:30:25 2014
New Revision: 1584514

URL: http://svn.apache.org/r1584514
Log:
HBASE-10118 Major compact keeps deletes with future timestamps. (Liu Shaohui)

Modified:
    hbase/branches/0.94/src/docbkx/book.xml
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java

Modified: hbase/branches/0.94/src/docbkx/book.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/docbkx/book.xml?rev=1584514&r1=1584513&r2=1584514&view=diff
==============================================================================
--- hbase/branches/0.94/src/docbkx/book.xml (original)
+++ hbase/branches/0.94/src/docbkx/book.xml Fri Apr  4 04:30:25 2014
@@ -485,6 +485,15 @@ htable.put(put);
           up on the user mailing list.</para>
           <para>Also see <xref linkend="keyvalue"/> for more information on the
internal KeyValue format.
           </para>
+          <para>Delete markers are purged during the major compaction of store, 
+          unless the KEEP_DELETED_CELLS is set in the column family. In some 
+          scenarios, users want to keep the deletes for a time and you can set the 
+          delete TTL: hbase.hstore.time.to.purge.deletes in the configuration. 
+          If this delete TTL is not set or set to 0, all delete markers including those 
+          with future timestamp are purged during the later major compaction. 
+          Otherwise, a delete marker is kept until the major compaction after 
+          marker's timestamp + delete TTL. 
+          </para>
         </section>
        </section>
 

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1584514&r1=1584513&r2=1584514&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Fri Apr  4 04:30:25 2014
@@ -296,9 +296,11 @@ public class ScanQueryMatcher {
         }
         // Can't early out now, because DelFam come before any other keys
       }
-      if (retainDeletesInOutput
-          || (!isUserScan && (EnvironmentEdgeManager.currentTimeMillis() - timestamp)
<= timeToPurgeDeletes)
-          || kv.getMemstoreTS() > maxReadPointToTrackVersions) {
+      if ((!isUserScan)
+          && timeToPurgeDeletes > 0
+          && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes)
{
+        return MatchCode.INCLUDE;
+      } else if (retainDeletesInOutput || kv.getMemstoreTS() > maxReadPointToTrackVersions)
{
         // always include or it is not time yet to check whether it is OK
         // to purge deltes or not
         if (!isUserScan) {

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java?rev=1584514&r1=1584513&r2=1584514&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
(original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
Fri Apr  4 04:30:25 2014
@@ -223,6 +223,72 @@ public class TestFromClientSide {
      h.close();
    }
 
+  /**
+   * Basic client side validation of HBASE-10118
+   */
+  @Test
+  public void testPurgeFutureDeletes() throws Exception {
+    final byte[] TABLENAME = Bytes.toBytes("testPurgeFutureDeletes");
+    final byte[] ROW = Bytes.toBytes("row");
+    final byte[] FAMILY = Bytes.toBytes("family");
+    final byte[] COLUMN = Bytes.toBytes("column");
+    final byte[] VALUE = Bytes.toBytes("value");
+
+    HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
+
+    // future timestamp
+    long ts = System.currentTimeMillis() * 2;
+    Put put = new Put(ROW, ts);
+    put.add(FAMILY, COLUMN, VALUE);
+    table.put(put);
+
+    Get get = new Get(ROW);
+    Result result = table.get(get);
+    assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
+
+    Delete del = new Delete(ROW);
+    del.deleteColumn(FAMILY, COLUMN, ts);
+    table.delete(del);
+
+    get = new Get(ROW);
+    result = table.get(get);
+    assertNull(result.getValue(FAMILY, COLUMN));
+
+    // major compaction, purged future deletes
+    TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
+    TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
+
+    // waiting for the major compaction to complete
+    int sleepTime = 0;
+    while (true) {
+      Thread.sleep(200);
+      sleepTime += 200;
+
+      Scan s = new Scan();
+      s.setRaw(true);
+      ResultScanner scanner = table.getScanner(s);
+      if (scanner.next() == null) {
+        scanner.close();
+        break;
+      }
+      scanner.close();
+
+      if (sleepTime > 6000) {
+        throw new IOException("Major compaction time is larger than 6000s");
+      }
+    }
+
+    put = new Put(ROW, ts);
+    put.add(FAMILY, COLUMN, VALUE);
+    table.put(put);
+
+    get = new Get(ROW);
+    result = table.get(get);
+    assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
+
+    table.close();
+  }
+
    /**
    * HBASE-2468 use case 1 and 2: region info de/serialization
    */



Mime
View raw message