hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1346862 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/regionserver/Store.java test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Date Wed, 06 Jun 2012 12:50:45 GMT
Author: mbautin
Date: Wed Jun  6 12:50:45 2012
New Revision: 1346862

URL: http://svn.apache.org/viewvc?rev=1346862&view=rev
Log:
[HBASE-4721] [0.89-fb] Fix retainDeletes to work well with large timestamps.

Author: aaiyer

Summary:
The previous implementation to retain delete timestamps introduced a bug,
    where if this feature is not used, we would only purge deletes upto the
    System.currentTimeMillis() instead of Long.MAX_VALUE.

    (See: https://phabricator.fb.com/D479953 for the details/discussion)

    The existing behavior (before this feature was introduced behaved as
        if the deletes were purged until Long.MAX_VALUE).

    Adding a unit test, and fixing the implementation accordingly.

Test Plan:
Added a test (i) verified that the test fails before the fix, and
  (ii) verified that the test passes after the fix.

  tbd: run all the tests on MR to look for other compatability issues.

Reviewers: kannan

Reviewed By: kannan

CC: pkhemani, hbase-eng@, levi

Differential Revision: https://phabricator.fb.com/D486039

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1346862&r1=1346861&r2=1346862&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Wed
Jun  6 12:50:45 2012
@@ -205,7 +205,7 @@ public class Store extends SchemaConfigu
       this.ttl *= 1000;
     }
     // used by ScanQueryMatcher
-    long timeToPurgeDeletes =
+    timeToPurgeDeletes =
         Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
     LOG.info("time to purge deletes set to " + timeToPurgeDeletes +
         "ms in store " + this);
@@ -1369,7 +1369,7 @@ public class Store extends SchemaConfigu
         scan.setMaxVersions(family.getMaxVersions());
         /* include deletes, unless we are doing a major compaction */
         long retainDeletesUntil = (majorCompaction)?
-          (System.currentTimeMillis() - this.timeToPurgeDeletes)
+          (this.timeToPurgeDeletes <= 0 ? Long.MAX_VALUE: (System.currentTimeMillis()
- this.timeToPurgeDeletes))
           : Long.MIN_VALUE;
         scanner = new StoreScanner(this, scan, scanners, smallestReadPoint,
             retainDeletesUntil);
@@ -2035,7 +2035,7 @@ public class Store extends SchemaConfigu
 
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (16 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (16 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG)
           + (6 * Bytes.SIZEOF_INT) + 2 * Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1346862&r1=1346861&r2=1346862&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Wed Jun  6 12:50:45 2012
@@ -127,6 +127,7 @@ public class TestCompaction extends HBas
     for (int i = 0; i < compactionThreshold; i++) {
       createStoreFile(r);
     }
+
     // Now delete everything.
     InternalScanner s = r.getScanner(new Scan());
     do {
@@ -275,6 +276,27 @@ public class TestCompaction extends HBas
     assertTrue("Should not see anything after TTL has expired", count == 0);
   }
 
+  public void testDeleteRetentionDuringMajorCompaction() throws Exception {
+    // add a delete with a large TS
+    Delete delete = new Delete(secondRowBytes);
+    byte [][] famAndQf = {COLUMN_FAMILY, null};
+    delete.deleteColumns(famAndQf[0], col1, Long.MAX_VALUE - 100);
+    r.delete(delete, null, true);
+
+    r.flushcache();
+
+    // after minor compaction, delete should be there
+    r.compactStores(false);
+    int count = count();
+    assertTrue("Delete should not be removed in a minor compaction", count != 0);
+
+    // after major compaction, the delete should disappear
+    r.compactStores(true);
+
+    count = count();
+    assertTrue("Deletes should be removed after a minor compaction", count == 0);
+  }
+
   public void testTimeBasedMajorCompaction() throws Exception {
     // create 2 storefiles and force a major compaction to reset the time
     int delay = 10 * 1000; // 10 sec



Mime
View raw message