hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-12223 MultiTableInputFormatBase.getSplits is too slow (Yuanbo Peng)
Date Thu, 18 Dec 2014 13:09:08 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 83e4bfaf7 -> 15cf0a6e7


HBASE-12223 MultiTableInputFormatBase.getSplits is too slow (Yuanbo Peng)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15cf0a6e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15cf0a6e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15cf0a6e

Branch: refs/heads/master
Commit: 15cf0a6e7b10882f7fd205b65c2ef265a690597d
Parents: 83e4bfa
Author: tedyu <yuzhihong@gmail.com>
Authored: Thu Dec 18 05:09:01 2014 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Thu Dec 18 05:09:01 2014 -0800

----------------------------------------------------------------------
 .../mapreduce/MultiTableInputFormatBase.java    | 101 +++++++++++--------
 1 file changed, 60 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/15cf0a6e/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
index 5c253cb..b9a2db7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -46,6 +46,9 @@ import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
 /**
  * A base for {@link MultiTableInputFormat}s. Receives a list of
  * {@link Scan} instances that define the input tables and
@@ -129,67 +132,82 @@ public abstract class MultiTableInputFormatBase extends
     if (scans.isEmpty()) {
       throw new IOException("No scans were provided.");
     }
-    List<InputSplit> splits = new ArrayList<InputSplit>();
 
+    Map<TableName, List<Scan>> tableMaps = new HashMap<TableName, List<Scan>>();
     for (Scan scan : scans) {
       byte[] tableNameBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
       if (tableNameBytes == null)
         throw new IOException("A scan object did not have a table name");
 
       TableName tableName = TableName.valueOf(tableNameBytes);
+
+      List<Scan> scanList = tableMaps.get(tableName);
+      if (scanList == null) {
+        scanList = new ArrayList<Scan>();
+        tableMaps.put(tableName, scanList);
+      }
+      scanList.add(scan);
+    }
+
+    List<InputSplit> splits = new ArrayList<InputSplit>();
+    Iterator iter = tableMaps.entrySet().iterator();
+    while (iter.hasNext()) {
+      Map.Entry<TableName, List<Scan>> entry = (Map.Entry<TableName, List<Scan>>)
iter.next();
+      TableName tableName = entry.getKey();
+      List<Scan> scanList = entry.getValue();
       Table table = null;
       RegionLocator regionLocator = null;
       Connection conn = null;
-      try {
+
+      try{
         conn = ConnectionFactory.createConnection(context.getConfiguration());
         table = conn.getTable(tableName);
-        regionLocator = conn.getRegionLocator(tableName);
         regionLocator = (RegionLocator) table;
+        RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
+                regionLocator, conn.getAdmin());
         Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys();
-        if (keys == null || keys.getFirst() == null ||
-            keys.getFirst().length == 0) {
-          throw new IOException("Expecting at least one region for table : "
-              + tableName.getNameAsString());
-        }
-        int count = 0;
+        for (Scan scan : scanList) {
+          if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
+            throw new IOException("Expecting at least one region for table : "
+                    + tableName.getNameAsString());
+          }
+          int count = 0;
 
-        byte[] startRow = scan.getStartRow();
-        byte[] stopRow = scan.getStopRow();
+          byte[] startRow = scan.getStartRow();
+          byte[] stopRow = scan.getStopRow();
 
-        RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(
-            regionLocator, conn.getAdmin());
+          for (int i = 0; i < keys.getFirst().length; i++) {
+            if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+              continue;
+            }
 
-        for (int i = 0; i < keys.getFirst().length; i++) {
-          if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
-            continue;
-          }
-          HRegionLocation hregionLocation = regionLocator.getRegionLocation(
-              keys.getFirst()[i], false);
-          String regionHostname = hregionLocation.getHostname();
-          HRegionInfo regionInfo = hregionLocation.getRegionInfo();
+            if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+                    Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+                    (stopRow.length == 0 || Bytes.compareTo(stopRow,
+                            keys.getFirst()[i]) > 0)) {
+              byte[] splitStart = startRow.length == 0 ||
+                      Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ?
+                      keys.getFirst()[i] : startRow;
+              byte[] splitStop = (stopRow.length == 0 ||
+                      Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) &&
+                      keys.getSecond()[i].length > 0 ?
+                      keys.getSecond()[i] : stopRow;
+
+              HRegionLocation hregionLocation = regionLocator.getRegionLocation(
+                      keys.getFirst()[i], false);
+              String regionHostname = hregionLocation.getHostname();
+              HRegionInfo regionInfo = hregionLocation.getRegionInfo();
+              long regionSize = sizeCalculator.getRegionSize(
+                      regionInfo.getRegionName());
 
-          // determine if the given start and stop keys fall into the range
-          if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
-              Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
-              (stopRow.length == 0 ||
-                  Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
-            byte[] splitStart =
-                startRow.length == 0 ||
-                    Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
-                    .getFirst()[i] : startRow;
-            byte[] splitStop =
-                (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
-                    stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
-                    .getSecond()[i] : stopRow;
+              TableSplit split = new TableSplit(table.getName(),
+                      scan, splitStart, splitStop, regionHostname, regionSize);
 
-            long regionSize = sizeCalculator.getRegionSize(regionInfo.getRegionName());
-            TableSplit split =
-                new TableSplit(regionLocator.getName(),
-                    scan, splitStart, splitStop, regionHostname, regionSize);
+              splits.add(split);
 
-            splits.add(split);
-            if (LOG.isDebugEnabled())
-              LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+              if (LOG.isDebugEnabled())
+                LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+            }
           }
         }
       } finally {
@@ -198,6 +216,7 @@ public abstract class MultiTableInputFormatBase extends
         if (null != conn) conn.close();
       }
     }
+
     return splits;
   }
 


Mime
View raw message