hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1451314 - in /hbase/trunk: hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactio...
Date Thu, 28 Feb 2013 19:46:25 GMT
Author: sershe
Date: Thu Feb 28 19:46:24 2013
New Revision: 1451314

URL: http://svn.apache.org/r1451314
Log:
HBASE-7902 deletes may be removed during minor compaction, in non-standard compaction schemes

Modified:
    hbase/trunk/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java

Modified: hbase/trunk/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
(original)
+++ hbase/trunk/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
Thu Feb 28 19:46:24 2013
@@ -201,7 +201,7 @@ public class ZooKeeperScanPolicyObserver
     Scan scan = new Scan();
     scan.setMaxVersions(scanInfo.getMaxVersions());
     return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
-        ScanType.MINOR_COMPACT, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
+        ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
   }
 
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
Thu Feb 28 19:46:24 2013
@@ -817,7 +817,7 @@ public class HStore implements Store {
       Scan scan = new Scan();
       scan.setMaxVersions(scanInfo.getMaxVersions());
       scanner = new StoreScanner(this, scanInfo, scan,
-          Collections.singletonList(memstoreScanner), ScanType.MINOR_COMPACT,
+          Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
           this.region.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
     }
     if (this.region.getCoprocessorHost() != null) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Thu Feb 28 19:46:24 2013
@@ -155,7 +155,7 @@ public class ScanQueryMatcher {
     // keep deleted cells: if compaction or raw scan
     this.keepDeletedCells = (scanInfo.getKeepDeletedCells() && !isUserScan) || scan.isRaw();
     // retain deletes: if minor compaction or raw scan
-    this.retainDeletesInOutput = scanType == ScanType.MINOR_COMPACT || scan.isRaw();
+    this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw();
     // seePastDeleteMarker: user initiated scans
     this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() && isUserScan;
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
Thu Feb 28 19:46:24 2013
@@ -24,7 +24,7 @@ import org.apache.hadoop.classification.
  */
 @InterfaceAudience.Private
 public enum ScanType {
-  MAJOR_COMPACT,
-  MINOR_COMPACT,
+  COMPACT_DROP_DELETES,
+  COMPACT_RETAIN_DELETES,
   USER_SCAN
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java
Thu Feb 28 19:46:24 2013
@@ -125,9 +125,7 @@ public class DefaultCompactionPolicy ext
     }
     candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction);
     CompactionRequest result = new CompactionRequest(candidateSelection);
-    if (!majorCompaction && !candidateSelection.isEmpty()) {
-      result.setOffPeak(mayUseOffPeak);
-    }
+    result.setOffPeak(!candidateSelection.isEmpty() && !majorCompaction &&
mayUseOffPeak);
     return result;
   }
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
Thu Feb 28 19:46:24 2013
@@ -71,6 +71,8 @@ public class DefaultCompactor extends Co
     // Also calculate earliest put timestamp if major compaction
     int maxKeyCount = 0;
     long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+    ScanType scanType = request.isMajor()
+        ? ScanType.COMPACT_DROP_DELETES : ScanType.COMPACT_RETAIN_DELETES;
     for (StoreFile file: filesToCompact) {
       StoreFile.Reader r = file.getReader();
       if (r == null) {
@@ -85,7 +87,7 @@ public class DefaultCompactor extends Co
       // For major compactions calculate the earliest put timestamp of all
       // involved storefiles. This is used to remove family delete marker during
       // compaction.
-      if (majorCompaction) {
+      if (scanType == ScanType.COMPACT_DROP_DELETES) {
         byte [] tmp = r.loadFileInfo().get(StoreFile.EARLIEST_PUT_TS);
         if (tmp == null) {
           // There's a file with no information, must be an old one
@@ -131,13 +133,9 @@ public class DefaultCompactor extends Co
       InternalScanner scanner = null;
       try {
         if (store.getCoprocessorHost() != null) {
-          scanner = store
-              .getCoprocessorHost()
-              .preCompactScannerOpen(store, scanners,
-                majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs,
-                request);
+          scanner = store.getCoprocessorHost()
+              .preCompactScannerOpen(store, scanners, scanType, earliestPutTs, request);
         }
-        ScanType scanType = majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT;
         if (scanner == null) {
           Scan scan = new Scan();
           scan.setMaxVersions(store.getFamily().getMaxVersions());

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
Thu Feb 28 19:46:24 2013
@@ -408,7 +408,7 @@ public class HFileReadWriteTest {
 
       // Include deletes
       scanner = new StoreScanner(store, store.getScanInfo(), scan, scanners,
-          ScanType.MAJOR_COMPACT, Long.MIN_VALUE, Long.MIN_VALUE);
+          ScanType.COMPACT_DROP_DELETES, Long.MIN_VALUE, Long.MIN_VALUE);
 
       ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java
Thu Feb 28 19:46:24 2013
@@ -32,7 +32,7 @@ public class NoOpScanPolicyObserver exte
     Scan scan = new Scan();
     scan.setMaxVersions(oldSI.getMaxVersions());
     return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
-        ScanType.MINOR_COMPACT, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
+        ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
   }
 
   /**

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Thu Feb 28 19:46:24 2013
@@ -555,7 +555,7 @@ public class TestStoreScanner extends Te
         KeyValue.COMPARATOR);
       StoreScanner scanner =
         new StoreScanner(scan, scanInfo,
-          ScanType.MAJOR_COMPACT, null, scanners,
+          ScanType.COMPACT_DROP_DELETES, null, scanners,
           HConstants.OLDEST_TIMESTAMP);
       List<KeyValue> results = new ArrayList<KeyValue>();
       results = new ArrayList<KeyValue>();

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
Thu Feb 28 19:46:24 2013
@@ -574,7 +574,7 @@ public class TestAccessController {
     PrivilegedExceptionAction action = new PrivilegedExceptionAction() {
       public Object run() throws Exception {
         ACCESS_CONTROLLER.preCompact(ObserverContext.createAndPrepare(RCP_ENV, null), null,
null,
-          ScanType.MINOR_COMPACT);
+          ScanType.COMPACT_RETAIN_DELETES);
         return null;
       }
     };

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java?rev=1451314&r1=1451313&r2=1451314&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
Thu Feb 28 19:46:24 2013
@@ -247,7 +247,7 @@ public class TestCoprocessorScanPolicy {
       Scan scan = new Scan();
       scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
       return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
-          ScanType.MINOR_COMPACT, store.getSmallestReadPoint(),
+          ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(),
           HConstants.OLDEST_TIMESTAMP);
     }
 



Mime
View raw message