hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject [1/4] hbase git commit: HBASE-13959 Region splitting uses a single thread in most common cases. (Hari Krishna Dara)
Date Tue, 30 Jun 2015 01:10:29 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 254ef1624 -> 163ddbf03
  refs/heads/branch-1.1 fb0cd5cb4 -> 0548da62d
  refs/heads/branch-1.2 39de65d26 -> 38f43acc4
  refs/heads/master 4f06279ca -> f8bd578b8


HBASE-13959 Region splitting uses a single thread in most common cases. (Hari Krishna Dara)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8bd578b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8bd578b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8bd578b

Branch: refs/heads/master
Commit: f8bd578b80b4e656d799c82ca1b6191e35bb0ae4
Parents: 4f06279
Author: Lars Hofhansl <larsh@apache.org>
Authored: Mon Jun 29 18:08:46 2015 -0700
Committer: Lars Hofhansl <larsh@apache.org>
Committed: Mon Jun 29 18:08:46 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/HConstants.java     |  7 +++++
 .../regionserver/SplitTransactionImpl.java      | 30 ++++++++++++++++----
 2 files changed, 32 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/f8bd578b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index c1316a9..32f07cb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1185,6 +1185,13 @@ public final class HConstants {
       "hbase.heap.occupancy.high_water_mark";
   public static final float DEFAULT_HEAP_OCCUPANCY_HIGH_WATERMARK = 0.98f;
 
+  /**
+   * The max number of threads used for splitting storefiles in parallel during
+   * the region split process.
+   */
+  public static final String REGION_SPLIT_THREADS_MAX =
+    "hbase.regionserver.region.split.threads.max";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8bd578b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
index 8695c77..95be873 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
@@ -568,17 +568,29 @@ public class SplitTransactionImpl implements SplitTransaction {
     // The following code sets up a thread pool executor with as many slots as
     // there's files to split. It then fires up everything, waits for
     // completion and finally checks for any exception
-    int nbFiles = hstoreFilesToSplit.size();
+    int nbFiles = 0;
+    for (Map.Entry<byte[], List<StoreFile>> entry: hstoreFilesToSplit.entrySet())
{
+        nbFiles += entry.getValue().size();
+    }
     if (nbFiles == 0) {
       // no file needs to be splitted.
       return new Pair<Integer, Integer>(0,0);
     }
-    LOG.info("Preparing to split " + nbFiles + " storefiles for region " + this.parent);
+    // Default max #threads to use is the smaller of table's configured number of blocking
store
+    // files or the available number of logical cores.
+    int defMaxThreads = Math.min(parent.conf.getInt(HStore.BLOCKING_STOREFILES_KEY,
+                HStore.DEFAULT_BLOCKING_STOREFILE_COUNT),
+            Runtime.getRuntime().availableProcessors());
+    // Max #threads is the smaller of the number of storefiles or the default max determined
above.
+    int maxThreads = Math.min(parent.conf.getInt(HConstants.REGION_SPLIT_THREADS_MAX,
+                defMaxThreads), nbFiles);
+    LOG.info("Preparing to split " + nbFiles + " storefiles for region " + this.parent +
+            " using " + maxThreads + " threads");
     ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
     builder.setNameFormat("StoreFileSplitter-%1$d");
     ThreadFactory factory = builder.build();
     ThreadPoolExecutor threadPool =
-      (ThreadPoolExecutor) Executors.newFixedThreadPool(nbFiles, factory);
+      (ThreadPoolExecutor) Executors.newFixedThreadPool(maxThreads, factory);
     List<Future<Pair<Path,Path>>> futures = new ArrayList<Future<Pair<Path,Path>>>
(nbFiles);
 
     // Split each store file.
@@ -624,14 +636,18 @@ public class SplitTransactionImpl implements SplitTransaction {
     }
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Split storefiles for region " + this.parent + " Daugther A: " + created_a
-          + " storefiles, Daugther B: " + created_b + " storefiles.");
+      LOG.debug("Split storefiles for region " + this.parent + " Daughter A: " + created_a
+          + " storefiles, Daughter B: " + created_b + " storefiles.");
     }
     return new Pair<Integer, Integer>(created_a, created_b);
   }
 
   private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
       throws IOException {
+    if (LOG.isDebugEnabled()) {
+        LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " +
+                  this.parent);
+    }
     HRegionFileSystem fs = this.parent.getRegionFileSystem();
     String familyName = Bytes.toString(family);
     Path path_a =
@@ -640,6 +656,10 @@ public class SplitTransactionImpl implements SplitTransaction {
     Path path_b =
         fs.splitStoreFile(this.hri_b, familyName, sf, this.splitrow, true,
           this.parent.getSplitPolicy());
+    if (LOG.isDebugEnabled()) {
+        LOG.debug("Splitting complete for store file: " + sf.getPath() + " for region: "
+
+                  this.parent);
+    }
     return new Pair<Path,Path>(path_a, path_b);
   }
 


Mime
View raw message