hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject hbase git commit: HBASE-17057 Minor compactions should also drop page cache (Ashu Pachauri)
Date Fri, 24 Feb 2017 23:31:57 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 b0780bdc6 -> 4b8195f22


HBASE-17057 Minor compactions should also drop page cache (Ashu Pachauri)

Signed-off-by: Gary Helmling <garyh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b8195f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b8195f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b8195f2

Branch: refs/heads/branch-1
Commit: 4b8195f22c1adf414d96f7e9f819c95ea6333479
Parents: b0780bd
Author: Ashu Pachauri <ashu210890@gmail.com>
Authored: Wed Nov 30 15:47:59 2016 -0800
Committer: Gary Helmling <garyh@apache.org>
Committed: Fri Feb 24 15:31:36 2017 -0800

----------------------------------------------------------------------
 .../src/main/resources/hbase-default.xml        | 18 ++++++++++++++
 .../regionserver/compactions/Compactor.java     | 26 ++++++++++++++++----
 2 files changed, 39 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/4b8195f2/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 147e243..7cd0015 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -753,6 +753,24 @@ possible configurations would overwhelm and obscure the important.
     </description>
   </property>
   <property>
+    <name>hbase.regionserver.majorcompaction.pagecache.drop</name>
+    <value>true</value>
+    <description>Specifies whether to drop pages read/written into the system page
cache by
+      major compactions. Setting it to true helps prevent major compactions from
+      polluting the page cache, which is almost always required, especially for clusters
+      with low/moderate memory to storage ratio.</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.minorcompaction.pagecache.drop</name>
+    <value>true</value>
+    <description>Specifies whether to drop pages read/written into the system page
cache by
+      minor compactions. Setting it to true helps prevent minor compactions from
+      polluting the page cache, which is most beneficial on clusters with low
+      memory to storage ratio or very write heavy clusters. You may want to set it to
+      false under moderate to low write workload when bulk of the reads are
+      on the most recently written data.</description>
+  </property>
+  <property>
     <name>hbase.storescanner.parallel.seek.enable</name>
     <value>false</value>
     <description>

http://git-wip-us.apache.org/repos/asf/hbase/blob/4b8195f2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index d1ab800..af8ffe8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -77,6 +77,15 @@ public abstract class Compactor<T extends CellSink> {
   /** specify how many days to keep MVCC values during major compaction **/ 
   protected int keepSeqIdPeriod;
 
+  // Configs that drive whether we drop page cache behind compactions
+  protected static final String  MAJOR_COMPACTION_DROP_CACHE =
+      "hbase.regionserver.majorcompaction.pagecache.drop";
+  protected static final String MINOR_COMPACTION_DROP_CACHE =
+      "hbase.regionserver.minorcompaction.pagecache.drop";
+
+  private boolean dropCacheMajor;
+  private boolean dropCacheMinor;
+
   //TODO: depending on Store is not good but, realistically, all compactors currently do.
   Compactor(final Configuration conf, final Store store) {
     this.conf = conf;
@@ -87,6 +96,8 @@ public abstract class Compactor<T extends CellSink> {
         Compression.Algorithm.NONE : this.store.getFamily().getCompactionCompression();
     this.keepSeqIdPeriod = Math.max(this.conf.getInt(HConstants.KEEP_SEQID_PERIOD, 
       HConstants.MIN_KEEP_SEQID_PERIOD), HConstants.MIN_KEEP_SEQID_PERIOD);
+    this.dropCacheMajor = conf.getBoolean(MAJOR_COMPACTION_DROP_CACHE, true);
+    this.dropCacheMinor = conf.getBoolean(MINOR_COMPACTION_DROP_CACHE, true);
   }
 
   public interface CellSink {
@@ -267,6 +278,13 @@ public abstract class Compactor<T extends CellSink> {
     List<StoreFileScanner> scanners;
     Collection<StoreFile> readersToClose;
     T writer = null;
+    boolean dropCache;
+    if (request.isMajor() || request.isAllFiles()) {
+      dropCache = this.dropCacheMajor;
+    } else {
+      dropCache = this.dropCacheMinor;
+    }
+
     if (this.conf.getBoolean("hbase.regionserver.compaction.private.readers", true)) {
       // clone all StoreFiles, so we'll do the compaction on a independent copy of StoreFiles,
       // HFiles, and their readers
@@ -274,12 +292,10 @@ public abstract class Compactor<T extends CellSink> {
       for (StoreFile f : request.getFiles()) {
         readersToClose.add(f.cloneForReader());
       }
-      scanners = createFileScanners(readersToClose, smallestReadPoint,
-        store.throttleCompaction(request.getSize()));
+      scanners = createFileScanners(readersToClose, smallestReadPoint, dropCache);
     } else {
       readersToClose = Collections.emptyList();
-      scanners = createFileScanners(request.getFiles(), smallestReadPoint,
-        store.throttleCompaction(request.getSize()));
+      scanners = createFileScanners(request.getFiles(), smallestReadPoint, dropCache);
     }
     InternalScanner scanner = null;
     boolean finished = false;
@@ -301,7 +317,7 @@ public abstract class Compactor<T extends CellSink> {
         smallestReadPoint = Math.min(fd.minSeqIdToKeep, smallestReadPoint);
         cleanSeqId = true;
       }
-      writer = sinkFactory.createWriter(scanner, fd, store.throttleCompaction(request.getSize()));
+      writer = sinkFactory.createWriter(scanner, fd, dropCache);
       finished =
           performCompaction(scanner, writer, smallestReadPoint, cleanSeqId, throughputController);
       if (!finished) {


Mime
View raw message