hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-17241 Avoid compacting already compacted mob files with _del files (huaxiang sun)
Date Mon, 05 Dec 2016 04:23:13 GMT
Repository: hbase
Updated Branches:
  refs/heads/master c73180d45 -> 0d6b872d9


HBASE-17241 Avoid compacting already compacted mob files with _del files (huaxiang sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d6b872d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d6b872d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d6b872d

Branch: refs/heads/master
Commit: 0d6b872d9dca4c040a172eed7b0153ece36c0a0d
Parents: c73180d
Author: tedyu <yuzhihong@gmail.com>
Authored: Sun Dec 4 20:23:06 2016 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Sun Dec 4 20:23:06 2016 -0800

----------------------------------------------------------------------
 .../PartitionedMobCompactionRequest.java        |  4 +++
 .../compactions/PartitionedMobCompactor.java    | 20 +++++++++++
 .../TestPartitionedMobCompactor.java            | 38 +++++++++++++++++---
 3 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
index e288571..665b5e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
@@ -86,6 +86,10 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest
{
     public List<FileStatus> listFiles() {
       return Collections.unmodifiableList(files);
     }
+
+    public int getFileCount () {
+      return files.size();
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
index 33f5e71..2f7d961 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
@@ -25,6 +25,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -187,6 +188,25 @@ public class PartitionedMobCompactor extends MobCompactor {
         selectedFileCount++;
       }
     }
+
+    /*
+     * If it is not a major mob compaction with del files, and the file number in Partition
is 1,
+     * remove the partition from filesToCompact list to avoid re-compacting files which has
been
+     * compacted with del files.
+     */
+    if (!allFiles && (allDelFiles.size() > 0)) {
+      Iterator<Map.Entry<CompactionPartitionId, CompactionPartition>> it =
+          filesToCompact.entrySet().iterator();
+
+      while(it.hasNext()) {
+        Map.Entry<CompactionPartitionId, CompactionPartition> entry = it.next();
+        if (entry.getValue().getFileCount() == 1) {
+          it.remove();
+          --selectedFileCount;
+        }
+      }
+    }
+
     PartitionedMobCompactionRequest request = new PartitionedMobCompactionRequest(
       filesToCompact.values(), allDelFiles);
     if (candidates.size() == (allDelFiles.size() + selectedFileCount + irrelevantFileCount))
{

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
index 7da8544..c112289 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
@@ -106,11 +106,23 @@ public class TestPartitionedMobCompactor {
   @Test
   public void testCompactionSelectWithAllFiles() throws Exception {
     String tableName = "testCompactionSelectWithAllFiles";
+    // If there is only 1 file, it will not be compacted with _del files, so
+    // It wont be CompactionType.ALL_FILES in this case, do not create with _del files.
     testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD,
-        CompactionType.ALL_FILES, false);
+        CompactionType.ALL_FILES, false, false);
   }
 
   @Test
+  public void testCompactionSelectToAvoidCompactOneFileWithDelete() throws Exception {
+    String tableName = "testCompactionSelectToAvoidCompactOneFileWithDelete";
+    // If there is only 1 file, it will not be compacted with _del files, so
+    // It wont be CompactionType.ALL_FILES in this case, and expected compact file count
will be 0.
+    testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD,
+        CompactionType.PART_FILES, false);
+  }
+
+
+  @Test
   public void testCompactionSelectWithPartFiles() throws Exception {
     String tableName = "testCompactionSelectWithPartFiles";
     testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false);
@@ -125,20 +137,36 @@ public class TestPartitionedMobCompactor {
   private void testCompactionAtMergeSize(final String tableName,
       final long mergeSize, final CompactionType type, final boolean isForceAllFiles)
       throws Exception {
+    testCompactionAtMergeSize(tableName, mergeSize, type, isForceAllFiles, true);
+  }
+
+  private void testCompactionAtMergeSize(final String tableName,
+      final long mergeSize, final CompactionType type, final boolean isForceAllFiles,
+      final boolean createDelFiles)
+      throws Exception {
     resetConf();
     init(tableName);
     int count = 10;
     // create 10 mob files.
     createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
+
+    if (createDelFiles) {
+      // create 10 del files
+      createStoreFiles(basePath, family, qf, count, Type.Delete);
+    }
+
     listFiles();
     List<String> expectedStartKeys = new ArrayList<>();
     for(FileStatus file : mobFiles) {
       if(file.getLen() < mergeSize) {
         String fileName = file.getPath().getName();
         String startKey = fileName.substring(0, 32);
-        expectedStartKeys.add(startKey);
+
+        // If it is not an major mob compaction and del files are there,
+        // these mob files wont be compacted.
+        if (isForceAllFiles || !createDelFiles) {
+          expectedStartKeys.add(startKey);
+        }
       }
     }
     // set the mob compaction mergeable threshold
@@ -150,7 +178,7 @@ public class TestPartitionedMobCompactor {
   public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
     String tableName = "testCompactDelFilesWithDefaultBatchSize";
     testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE,
-      MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
+        MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
   }
 
   @Test


Mime
View raw message