hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramkris...@apache.org
Subject hbase git commit: HBASE-16454 Compactor's shipping logic decision should be based on the current store's block size (Ram)
Date Tue, 23 Aug 2016 05:36:32 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 2aae923c3 -> ff6182b02


HBASE-16454 Compactor's shipping logic decision should be based on the
current store's block size (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff6182b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff6182b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff6182b0

Branch: refs/heads/master
Commit: ff6182b020c0d75f8d077eaeef7ae84124e1d4ce
Parents: 2aae923
Author: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
Authored: Tue Aug 23 11:06:02 2016 +0530
Committer: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
Committed: Tue Aug 23 11:06:02 2016 +0530

----------------------------------------------------------------------
 .../hadoop/hbase/mob/DefaultMobStoreCompactor.java      |  5 +++--
 .../hbase/regionserver/compactions/Compactor.java       | 12 +++++-------
 2 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/ff6182b0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index 711f31d..55aea00 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -160,12 +160,13 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
    * @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
    * @param throughputController The compaction throughput controller.
    * @param major Is a major compaction.
+   * @param numofFilesToCompact the number of files to compact
    * @return Whether compaction ended; false if it was interrupted for any reason.
    */
   @Override
   protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
-      long smallestReadPoint, boolean cleanSeqId,
-      ThroughputController throughputController,  boolean major) throws IOException {
+      long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController,
+      boolean major, int numofFilesToCompact) throws IOException {
     if (!(scanner instanceof MobCompactionStoreScanner)) {
       throw new IllegalArgumentException(
         "The scanner should be an instance of MobCompactionStoreScanner");

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff6182b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 1796bca..c695788 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -310,7 +310,7 @@ public abstract class Compactor<T extends CellSink> {
       }
       writer = sinkFactory.createWriter(scanner, fd, store.throttleCompaction(request.getSize()));
       finished = performCompaction(fd, scanner, writer, smallestReadPoint, cleanSeqId,
-        throughputController, request.isAllFiles());
+        throughputController, request.isAllFiles(), request.getFiles().size());
       if (!finished) {
         throw new InterruptedIOException("Aborting compaction of store " + store + " in region
"
             + store.getRegionInfo().getRegionNameAsString() + " because it was interrupted.");
@@ -385,11 +385,12 @@ public abstract class Compactor<T extends CellSink> {
    * @param cleanSeqId When true, remove seqId(used to be mvcc) value which is &lt;=
    *          smallestReadPoint
    * @param major Is a major compaction.
+   * @param numofFilesToCompact the number of files to compact
    * @return Whether compaction ended; false if it was interrupted for some reason.
    */
   protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
-      long smallestReadPoint, boolean cleanSeqId,
-      ThroughputController throughputController, boolean major) throws IOException {
+      long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController,
+      boolean major, int numofFilesToCompact) throws IOException {
     long bytesWrittenProgressForCloseCheck = 0;
     long bytesWrittenProgressForLog = 0;
     long bytesWrittenProgressForShippedCall = 0;
@@ -409,10 +410,7 @@ public abstract class Compactor<T extends CellSink> {
 
     throughputController.start(compactionName);
     KeyValueScanner kvs = (scanner instanceof KeyValueScanner)? (KeyValueScanner)scanner
: null;
-    long minFilesToCompact = Math.max(2L,
-        conf.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY,
-            /* old name */ conf.getInt("hbase.hstore.compactionThreshold", 3)));
-    long shippedCallSizeLimit = (long) minFilesToCompact * HConstants.DEFAULT_BLOCKSIZE;
+    long shippedCallSizeLimit = (long) numofFilesToCompact * this.store.getFamily().getBlocksize();
     try {
       do {
         hasMore = scanner.next(cells, scannerContext);


Mime
View raw message