hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chu...@apache.org
Subject [1/2] hbase git commit: Revert due to missing jira ticket "Add more flexibility for input directory structure to LoadIncrementalHFiles"
Date Thu, 11 Jan 2018 19:07:56 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-2 5d4140e09 -> 3849db8f1


Revert due to missing jira ticket  "Add more flexibility for input directory structure to
LoadIncrementalHFiles"

This reverts commit d524768528cd15151ba1ebb82e32609da5308128.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/482d413e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/482d413e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/482d413e

Branch: refs/heads/branch-2
Commit: 482d413e0abbabcb0afcdeab7c8ad761218e1df2
Parents: 5d4140e
Author: Rahul Gidwani <churro@apache.org>
Authored: Thu Jan 11 11:07:10 2018 -0800
Committer: Rahul Gidwani <churro@apache.org>
Committed: Thu Jan 11 11:07:10 2018 -0800

----------------------------------------------------------------------
 .../hbase/tool/LoadIncrementalHFiles.java       | 50 ++++-----------
 .../hbase/tool/TestLoadIncrementalHFiles.java   | 67 ++++++--------------
 2 files changed, 32 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/482d413e/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
index c14c944..119ac49 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.tool;
 
+import static java.lang.String.format;
+
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
@@ -46,7 +48,7 @@ import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
-import static java.lang.String.format;
+
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -58,6 +60,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.ClientServiceCallable;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
@@ -89,20 +94,14 @@ import org.apache.hadoop.hbase.security.UserProvider;
 import org.apache.hadoop.hbase.security.token.FsDelegationToken;
 import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
 import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
 import org.apache.hbase.thirdparty.com.google.common.collect.Multimaps;
 import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Tool to load the output of HFileOutputFormat into an existing table.
@@ -182,12 +181,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
   }
 
   private void usage() {
-    System.err.println("usage: " + NAME + " /path/to/hfileoutputformat-output tablename -loadTable"
-        + "\n -D" + CREATE_TABLE_CONF_KEY + "=no - can be used to avoid creation of table
by "
-        + "this tool\n  Note: if you set this to 'no', then the target table must already
exist "
-        + "in HBase\n -loadTable implies your baseDirectory to store file has a depth of
3 ,you"
-        + " must have an existing table\n-D" + IGNORE_UNMATCHED_CF_CONF_KEY + "=yes - can
be used "
-        + "to ignore unmatched column families\n" +
+    System.err.println("usage: " + NAME + " /path/to/hfileoutputformat-output tablename"
+ "\n -D" +
+        CREATE_TABLE_CONF_KEY + "=no - can be used to avoid creation of table by this tool\n"
+
+        "  Note: if you set this to 'no', then the target table must already exist in HBase\n
-D" +
+        IGNORE_UNMATCHED_CF_CONF_KEY + "=yes - can be used to ignore unmatched column families\n"
+
         "\n");
   }
 
@@ -1153,8 +1150,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
       }
       try (Table table = connection.getTable(tableName);
           RegionLocator locator = connection.getRegionLocator(tableName)) {
-        return doBulkLoad(new Path(hfofDir), admin, table, locator, isSilence(),
-            isAlwaysCopyFiles());
+        return doBulkLoad(new Path(hfofDir), admin, table, locator, isSilence(), isAlwaysCopyFiles());
       }
     }
   }
@@ -1182,33 +1178,13 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
 
   @Override
   public int run(String[] args) throws Exception {
-    if (args.length != 2 && args.length != 3) {
+    if (args.length < 2) {
       usage();
       return -1;
     }
     String dirPath = args[0];
     TableName tableName = TableName.valueOf(args[1]);
-
-
-    if (args.length == 2) {
-      return !run(dirPath, tableName).isEmpty() ? 0 : -1;
-    } else {
-      Map<byte[], List<Path>> family2Files = Maps.newHashMap();
-      FileSystem fs = FileSystem.get(getConf());
-      for (FileStatus regionDir : fs.listStatus(new Path(dirPath))) {
-        FSVisitor.visitRegionStoreFiles(fs, regionDir.getPath(), (region, family, hfileName)
-> {
-          Path path = new Path(regionDir.getPath(), new Path(family, hfileName));
-          byte[] familyName = Bytes.toBytes(family);
-          if (family2Files.containsKey(familyName)) {
-            family2Files.get(familyName).add(path);
-          } else {
-            family2Files.put(familyName, Lists.newArrayList(path));
-          }
-        });
-      }
-      return !run(family2Files, tableName).isEmpty() ? 0 : -1;
-    }
-
+    return !run(dirPath, tableName).isEmpty() ? 0 : -1;
   }
 
   public static void main(String[] args) throws Exception {

http://git-wip-us.apache.org/repos/asf/hbase/blob/482d413e/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
index 8b1c96e..7e4d40e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -29,6 +30,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.TreeMap;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -51,7 +53,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.LoadQueueItem;
@@ -130,10 +131,10 @@ public class TestLoadIncrementalHFiles {
   public void testSimpleLoadWithFileCopy() throws Exception {
     String testName = tn.getMethodName();
     final byte[] TABLE_NAME = Bytes.toBytes("mytable_" + testName);
-    runTest(testName, buildHTD(TableName.valueOf(TABLE_NAME), BloomType.NONE),
-        false, null, new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("cccc")
},
+    runTest(testName, buildHTD(TableName.valueOf(TABLE_NAME), BloomType.NONE), BloomType.NONE,
+      false, null, new byte[][][] { new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("cccc")
},
           new byte[][] { Bytes.toBytes("ddd"), Bytes.toBytes("ooo") }, },
-      false, true, 2);
+      false, true);
   }
 
   /**
@@ -256,55 +257,29 @@ public class TestLoadIncrementalHFiles {
     // Run the test bulkloading the table to the default namespace
     final TableName TABLE_WITHOUT_NS = TableName.valueOf(TABLE_NAME);
     runTest(testName, TABLE_WITHOUT_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges,
-      useMap, 2);
-
-
-    /* Run the test bulkloading the table from a depth of 3
-      directory structure is now
-      baseDirectory
-          -- regionDir
-            -- familyDir
-              -- storeFileDir
-    */
-    if (preCreateTable) {
-      runTest(testName + 2, TABLE_WITHOUT_NS, bloomType, true, tableSplitKeys, hfileRanges,
-          false, 3);
-    }
+      useMap);
 
     // Run the test bulkloading the table to the specified namespace
     final TableName TABLE_WITH_NS = TableName.valueOf(Bytes.toBytes(NAMESPACE), TABLE_NAME);
     runTest(testName, TABLE_WITH_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges,
-      useMap, 2);
+      useMap);
   }
 
   private void runTest(String testName, TableName tableName, BloomType bloomType,
-      boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges,
-      boolean useMap, int depth) throws Exception {
+      boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap)
+      throws Exception {
     TableDescriptor htd = buildHTD(tableName, bloomType);
-    runTest(testName, htd, preCreateTable, tableSplitKeys, hfileRanges, useMap, false, depth);
+    runTest(testName, htd, bloomType, preCreateTable, tableSplitKeys, hfileRanges, useMap,
false);
   }
 
   public static int loadHFiles(String testName, TableDescriptor htd, HBaseTestingUtility
util,
       byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys,
       byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles,
       int initRowCount, int factor) throws Exception {
-    return loadHFiles(testName, htd, util, fam, qual, preCreateTable, tableSplitKeys, hfileRanges,
-        useMap, deleteFile, copyFiles, initRowCount, factor, 2);
-  }
-
-  public static int loadHFiles(String testName, TableDescriptor htd, HBaseTestingUtility
util,
-      byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys,
-      byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles,
-      int initRowCount, int factor, int depth) throws Exception {
-    Path baseDirectory = util.getDataTestDirOnTestFS(testName);
+    Path dir = util.getDataTestDirOnTestFS(testName);
     FileSystem fs = util.getTestFileSystem();
-    baseDirectory = baseDirectory.makeQualified(fs.getUri(), fs.getWorkingDirectory());
-    Path parentDir = baseDirectory;
-    if (depth == 3) {
-      assert !useMap;
-      parentDir = new Path(baseDirectory, "someRegion");
-    }
-    Path familyDir = new Path(parentDir, Bytes.toString(fam));
+    dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+    Path familyDir = new Path(dir, Bytes.toString(fam));
 
     int hfileIdx = 0;
     Map<byte[], List<Path>> map = null;
@@ -339,11 +314,7 @@ public class TestLoadIncrementalHFiles {
       conf.setBoolean(LoadIncrementalHFiles.ALWAYS_COPY_FILES, true);
     }
     LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
-    List<String> args = Lists.newArrayList(baseDirectory.toString(), tableName.toString());
-    if (depth == 3) {
-      args.add("-loadTable");
-    }
-
+    String[] args = { dir.toString(), tableName.toString() };
     if (useMap) {
       if (deleteFile) {
         fs.delete(last, true);
@@ -358,7 +329,7 @@ public class TestLoadIncrementalHFiles {
         }
       }
     } else {
-      loader.run(args.toArray(new String[]{}));
+      loader.run(args);
     }
 
     if (copyFiles) {
@@ -377,11 +348,11 @@ public class TestLoadIncrementalHFiles {
     return expectedRows;
   }
 
-  private void runTest(String testName, TableDescriptor htd,
+  private void runTest(String testName, TableDescriptor htd, BloomType bloomType,
       boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap,
-      boolean copyFiles, int depth) throws Exception {
+      boolean copyFiles) throws Exception {
     loadHFiles(testName, htd, util, FAMILY, QUALIFIER, preCreateTable, tableSplitKeys, hfileRanges,
-      useMap, true, copyFiles, 0, 1000, depth);
+      useMap, true, copyFiles, 0, 1000);
 
     final TableName tableName = htd.getTableName();
     // verify staging folder has been cleaned up
@@ -459,7 +430,7 @@ public class TestLoadIncrementalHFiles {
         .build();
 
     try {
-      runTest(testName, htd, true, SPLIT_KEYS, hFileRanges, false, false, 2);
+      runTest(testName, htd, BloomType.NONE, true, SPLIT_KEYS, hFileRanges, false, false);
       assertTrue("Loading into table with non-existent family should have failed", false);
     } catch (Exception e) {
       assertTrue("IOException expected", e instanceof IOException);


Mime
View raw message