hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [2/4] hbase git commit: HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
Date Fri, 16 Jan 2015 10:27:04 GMT
HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52b4db86
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52b4db86
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52b4db86

Branch: refs/heads/branch-1
Commit: 52b4db860780fea6cf5ec0dabb34a581090a535d
Parents: 2e08bb3
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Thu Jan 15 23:13:47 2015 +0000
Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Committed: Thu Jan 15 23:14:29 2015 +0000

----------------------------------------------------------------------
 .../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java    | 11 +++++++++--
 .../org/apache/hadoop/hbase/regionserver/HStore.java     |  6 ++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/52b4db86/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index b6e8d9c..9903fe9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
       }
       Path familyDir = stat.getPath();
       byte[] family = familyDir.getName().getBytes();
-      Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
-      for (Path hfile : hfiles) {
+      FileStatus[] hfileStatuses = fs.listStatus(familyDir);
+      for (FileStatus hfileStatus : hfileStatuses) {
+        long length = hfileStatus.getLen();
+        Path hfile = hfileStatus.getPath();
         // Skip "_", reference, HFileLink
         String fileName = hfile.getName();
         if (fileName.startsWith("_")) continue;
@@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
           LOG.warn("Skipping HFileLink " + fileName);
           continue;
         }
+        if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+            HConstants.DEFAULT_MAX_FILE_SIZE)) {
+          LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
+              length + " bytes can be problematic as it may lead to oversplitting.");
+        }
         ret.add(new LoadQueueItem(family, hfile));
       }
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/52b4db86/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index ad701b7..6a65038 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -709,6 +709,12 @@ public class HStore implements Store {
             + this.getRegionInfo().getRegionNameAsString());
       }
 
+      if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+          HConstants.DEFAULT_MAX_FILE_SIZE)) {
+        LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
+            reader.length() + " bytes can be problematic as it may lead to oversplitting.");
+      }
+
       if (verifyBulkLoads) {
         long verificationStartTime = EnvironmentEdgeManager.currentTime();
         LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());


Mime
View raw message