Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5198118C18 for ; Mon, 4 Jan 2016 16:56:52 +0000 (UTC) Received: (qmail 75181 invoked by uid 500); 4 Jan 2016 16:56:48 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 75118 invoked by uid 500); 4 Jan 2016 16:56:48 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 73358 invoked by uid 99); 4 Jan 2016 16:56:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Jan 2016 16:56:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 32066E07E9; Mon, 4 Jan 2016 16:56:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: syuanjiang@apache.org To: commits@hbase.apache.org Date: Mon, 04 Jan 2016 16:57:13 -0000 Message-Id: <1236234cd8e54cc5b0aa36a4a1498132@git.apache.org> In-Reply-To: <51767155c1144ae68c5c50a871d63d25@git.apache.org> References: <51767155c1144ae68c5c50a871d63d25@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [28/29] hbase git commit: HBASE-15038 ExportSnapshot should support separate configs for source and destination HBASE-15038 ExportSnapshot should support separate configs for source and destination Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9589a7d8 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9589a7d8 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9589a7d8 Branch: refs/heads/hbase-12439 Commit: 9589a7d8be4d29bffcb0c711e5bd6573e8df712c Parents: a82f7fc Author: Gary Helmling Authored: Wed Dec 23 18:49:58 2015 -0800 Committer: Gary Helmling Committed: Mon Jan 4 00:10:19 2016 -0800 ---------------------------------------------------------------------- .../hadoop/hbase/snapshot/ExportSnapshot.java | 33 ++++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/9589a7d8/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index beddce3..a574410 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -89,6 +89,10 @@ import org.apache.hadoop.util.ToolRunner; @InterfaceStability.Evolving public class ExportSnapshot extends Configured implements Tool { public static final String NAME = "exportsnapshot"; + /** Configuration prefix for overrides for the source filesystem */ + public static final String CONF_SOURCE_PREFIX = NAME + ".from."; + /** Configuration prefix for overrides for the destination filesystem */ + public static final String CONF_DEST_PREFIX = NAME + ".to."; private static final Log LOG = LogFactory.getLog(ExportSnapshot.class); @@ -141,6 +145,9 @@ public class ExportSnapshot extends Configured implements Tool { @Override public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); + Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); + Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); + verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true); filesGroup = conf.get(CONF_FILES_GROUP); @@ -155,15 +162,15 @@ public class ExportSnapshot extends Configured implements Tool { testFailures = conf.getBoolean(CONF_TEST_FAILURE, false); try { - conf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); - inputFs = FileSystem.get(inputRoot.toUri(), conf); + srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); + inputFs = FileSystem.get(inputRoot.toUri(), srcConf); } catch (IOException e) { throw new IOException("Could not get the input FileSystem with root=" + inputRoot, e); } try { - conf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); - outputFs = FileSystem.get(outputRoot.toUri(), conf); + destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); + outputFs = FileSystem.get(outputRoot.toUri(), destConf); } catch (IOException e) { throw new IOException("Could not get the output FileSystem with root="+ outputRoot, e); } @@ -789,8 +796,12 @@ public class ExportSnapshot extends Configured implements Tool { job.setNumReduceTasks(0); // Acquire the delegation Tokens + Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); + TokenCache.obtainTokensForNamenodes(job.getCredentials(), + new Path[] { inputRoot }, srcConf); + Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); TokenCache.obtainTokensForNamenodes(job.getCredentials(), - new Path[] { inputRoot, outputRoot }, conf); + new Path[] { outputRoot }, destConf); // Run the MR Job if (!job.waitForCompletion(true)) { @@ -913,11 +924,13 @@ public class ExportSnapshot extends Configured implements Tool { targetName = snapshotName; } - conf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); - FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf); + Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); + srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); + FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf); LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot); - conf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); - FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf); + Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); + destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); + FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf); LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString()); boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false); @@ -1004,7 +1017,7 @@ public class ExportSnapshot extends Configured implements Tool { // Step 4 - Verify snapshot integrity if (verifyTarget) { LOG.info("Verify snapshot integrity"); - verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir); + verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir); } LOG.info("Export Completed: " + targetName);