hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/2] hbase git commit: HBASE-15038 ExportSnapshot should support separate configs for source and destination
Date Tue, 05 Jan 2016 21:02:12 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 8aa53800e -> 8eed167fc


HBASE-15038 ExportSnapshot should support separate configs for source and destination


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5b958029
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5b958029
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5b958029

Branch: refs/heads/0.98
Commit: 5b958029e1341adf2f9036b458d18f497af1a15e
Parents: 8aa5380
Author: Gary Helmling <garyh@apache.org>
Authored: Wed Dec 23 18:49:58 2015 -0800
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Mon Jan 4 17:29:21 2016 -0800

----------------------------------------------------------------------
 .../hadoop/hbase/snapshot/ExportSnapshot.java   | 33 ++++++++++++++------
 1 file changed, 23 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/5b958029/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 37c247d..0ef38c3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -92,6 +92,10 @@ import org.apache.hadoop.util.ToolRunner;
 @InterfaceStability.Evolving
 public class ExportSnapshot extends Configured implements Tool {
   public static final String NAME = "exportsnapshot";
+  /** Configuration prefix for overrides for the source filesystem */
+  public static final String CONF_SOURCE_PREFIX = NAME + ".from.";
+  /** Configuration prefix for overrides for the destination filesystem */
+  public static final String CONF_DEST_PREFIX = NAME + ".to.";
 
   private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
 
@@ -146,6 +150,9 @@ public class ExportSnapshot extends Configured implements Tool {
     @Override
     public void setup(Context context) throws IOException {
       Configuration conf = context.getConfiguration();
+      Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
+      Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
+
       verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
 
       filesGroup = conf.get(CONF_FILES_GROUP);
@@ -160,15 +167,15 @@ public class ExportSnapshot extends Configured implements Tool {
       testFailures = conf.getBoolean(CONF_TEST_FAILURE, false);
 
       try {
-        conf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
-        inputFs = FileSystem.get(inputRoot.toUri(), conf);
+        srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache",
true);
+        inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
       } catch (IOException e) {
         throw new IOException("Could not get the input FileSystem with root=" + inputRoot,
e);
       }
 
       try {
-        conf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
-        outputFs = FileSystem.get(outputRoot.toUri(), conf);
+        destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache",
true);
+        outputFs = FileSystem.get(outputRoot.toUri(), destConf);
       } catch (IOException e) {
         throw new IOException("Could not get the output FileSystem with root="+ outputRoot,
e);
       }
@@ -798,8 +805,12 @@ public class ExportSnapshot extends Configured implements Tool {
     job.setNumReduceTasks(0);
 
     // Acquire the delegation Tokens
+    Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
+    TokenCache.obtainTokensForNamenodes(job.getCredentials(),
+      new Path[] { inputRoot }, srcConf);
+    Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
     TokenCache.obtainTokensForNamenodes(job.getCredentials(),
-      new Path[] { inputRoot, outputRoot }, conf);
+        new Path[] { outputRoot }, destConf);
 
     // Run the MR Job
     if (!job.waitForCompletion(true)) {
@@ -906,11 +917,13 @@ public class ExportSnapshot extends Configured implements Tool {
       targetName = snapshotName;
     }
 
-    conf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
-    FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf);
+    Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
+    srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
+    FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
     LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
-    conf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
-    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
+    Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
+    destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
+    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf);
     LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
 
     boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
@@ -994,7 +1007,7 @@ public class ExportSnapshot extends Configured implements Tool {
       // Step 4 - Verify snapshot integrity
       if (verifyTarget) {
         LOG.info("Verify snapshot integrity");
-        verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir);
+        verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);
       }
 
       LOG.info("Export Completed: " + targetName);


Mime
View raw message