Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 36697200B92 for ; Tue, 13 Sep 2016 10:29:32 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 34FBD160AD8; Tue, 13 Sep 2016 08:29:32 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 5B5D5160AD3 for ; Tue, 13 Sep 2016 10:29:31 +0200 (CEST) Received: (qmail 64978 invoked by uid 500); 13 Sep 2016 08:29:17 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 63274 invoked by uid 99); 13 Sep 2016 08:29:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Sep 2016 08:29:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C0BE4DFCC0; Tue, 13 Sep 2016 08:29:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: drankye@apache.org To: common-commits@hadoop.apache.org Date: Tue, 13 Sep 2016 08:29:58 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [44/50] [abbrv] hadoop git commit: HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic archived-at: Tue, 13 Sep 2016 08:29:32 -0000 HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9faccd10 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9faccd10 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9faccd10 Branch: refs/heads/HDFS-10285 Commit: 9faccd104672dfef123735ca8ada178fc3a6196f Parents: cc01ed70 Author: Ravi Prakash Authored: Mon Sep 12 08:26:08 2016 -0700 Committer: Ravi Prakash Committed: Mon Sep 12 08:26:08 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/tools/DistCp.java | 4 ++ .../org/apache/hadoop/tools/DistCpOptions.java | 8 ++- .../apache/hadoop/tools/TestOptionsParser.java | 64 +++++++++++++++++++- 3 files changed, 70 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 36e5ee4..be58f13 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool { private static final String PREFIX = "_distcp"; private static final String WIP_PREFIX = "._WIP_"; private static final String DISTCP_DEFAULT_XML = "distcp-default.xml"; + private static final String DISTCP_SITE_XML = "distcp-site.xml"; static final Random rand = new Random(); private boolean submitted; @@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool { public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception { Configuration config = new Configuration(configuration); config.addResource(DISTCP_DEFAULT_XML); + config.addResource(DISTCP_SITE_XML); setConf(config); this.inputOptions = inputOptions; this.metaFolder = createMetaFolderPath(); @@ -393,10 +395,12 @@ public class DistCp extends Configured implements Tool { * Loads properties from distcp-default.xml into configuration * object * @return Configuration which includes properties from distcp-default.xml + * and distcp-site.xml */ private static Configuration getDefaultConf() { Configuration config = new Configuration(); config.addResource(DISTCP_DEFAULT_XML); + config.addResource(DISTCP_SITE_XML); return config; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index e6f53f5..4c5518f 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -47,7 +47,7 @@ public class DistCpOptions { public static final int maxNumListstatusThreads = 40; private int numListstatusThreads = 0; // Indicates that flag is not set. private int maxMaps = DistCpConstants.DEFAULT_MAPS; - private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB; + private float mapBandwidth = 0; // Indicates that we should use the default. private String copyStrategy = DistCpConstants.UNIFORMSIZE; @@ -609,8 +609,10 @@ public class DistCpOptions { String.valueOf(useDiff)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC, String.valueOf(skipCRC)); - DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH, - String.valueOf(mapBandwidth)); + if (mapBandwidth > 0) { + DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH, + String.valueOf(mapBandwidth)); + } DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); if (filtersFile != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index c46bcd9..218de4e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -107,7 +107,7 @@ public class TestOptionsParser { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/"}); - Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA); + Assert.assertEquals(options.getMapBandwidth(), 0, DELTA); options = OptionsParser.parse(new String[] { "-bandwidth", @@ -389,7 +389,7 @@ public class TestOptionsParser { + "deleteMissing=false, ignoreFailures=false, overwrite=false, " + "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, " + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, " - + "mapBandwidth=100.0, " + + "mapBandwidth=0.0, " + "copyStrategy='uniformsize', preserveStatus=[], " + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, " + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, " @@ -572,6 +572,8 @@ public class TestOptionsParser { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null); DistCpOptions options = OptionsParser.parse(new String[] { "-atomic", "-i", @@ -581,7 +583,7 @@ public class TestOptionsParser { Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), - DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA); + -1.0, DELTA); conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); @@ -603,6 +605,62 @@ public class TestOptionsParser { } @Test + public void testOptionsAppendToConfDoesntOverwriteBandwidth() { + Configuration conf = new Configuration(); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null); + DistCpOptions options = OptionsParser.parse(new String[] { + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0, + DELTA); + + conf = new Configuration(); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null); + options = OptionsParser.parse(new String[] { + "-update", + "-delete", + "-pu", + "-bandwidth", + "77", + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0, + DELTA); + + conf = new Configuration(); + conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88"); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88"); + options = OptionsParser.parse(new String[] { + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0, + DELTA); + + conf = new Configuration(); + conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0"); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0"); + options = OptionsParser.parse(new String[] { + "-bandwidth", + "99", + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0, + DELTA); + } + + @Test public void testAppendOption() { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean( --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org