hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dran...@apache.org
Subject [25/32] hadoop git commit: HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic
Date Tue, 13 Sep 2016 08:16:49 GMT
HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed
by Zoran Dimitrijevic


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9faccd10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9faccd10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9faccd10

Branch: refs/heads/HADOOP-12756
Commit: 9faccd104672dfef123735ca8ada178fc3a6196f
Parents: cc01ed70
Author: Ravi Prakash <raviprak@altiscale.com>
Authored: Mon Sep 12 08:26:08 2016 -0700
Committer: Ravi Prakash <raviprak@altiscale.com>
Committed: Mon Sep 12 08:26:08 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/tools/DistCp.java    |  4 ++
 .../org/apache/hadoop/tools/DistCpOptions.java  |  8 ++-
 .../apache/hadoop/tools/TestOptionsParser.java  | 64 +++++++++++++++++++-
 3 files changed, 70 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 36e5ee4..be58f13 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool {
   private static final String PREFIX = "_distcp";
   private static final String WIP_PREFIX = "._WIP_";
   private static final String DISTCP_DEFAULT_XML = "distcp-default.xml";
+  private static final String DISTCP_SITE_XML = "distcp-site.xml";
   static final Random rand = new Random();
 
   private boolean submitted;
@@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool {
   public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception
{
     Configuration config = new Configuration(configuration);
     config.addResource(DISTCP_DEFAULT_XML);
+    config.addResource(DISTCP_SITE_XML);
     setConf(config);
     this.inputOptions = inputOptions;
     this.metaFolder   = createMetaFolderPath();
@@ -393,10 +395,12 @@ public class DistCp extends Configured implements Tool {
    * Loads properties from distcp-default.xml into configuration
    * object
    * @return Configuration which includes properties from distcp-default.xml
+   *         and distcp-site.xml
    */
   private static Configuration getDefaultConf() {
     Configuration config = new Configuration();
     config.addResource(DISTCP_DEFAULT_XML);
+    config.addResource(DISTCP_SITE_XML);
     return config;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index e6f53f5..4c5518f 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -47,7 +47,7 @@ public class DistCpOptions {
   public static final int maxNumListstatusThreads = 40;
   private int numListstatusThreads = 0;  // Indicates that flag is not set.
   private int maxMaps = DistCpConstants.DEFAULT_MAPS;
-  private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB;
+  private float mapBandwidth = 0;  // Indicates that we should use the default.
 
   private String copyStrategy = DistCpConstants.UNIFORMSIZE;
 
@@ -609,8 +609,10 @@ public class DistCpOptions {
         String.valueOf(useDiff));
     DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC,
         String.valueOf(skipCRC));
-    DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
-        String.valueOf(mapBandwidth));
+    if (mapBandwidth > 0) {
+      DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
+          String.valueOf(mapBandwidth));
+    }
     DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS,
         DistCpUtils.packAttributes(preserveStatus));
     if (filtersFile != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index c46bcd9..218de4e 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -107,7 +107,7 @@ public class TestOptionsParser {
     DistCpOptions options = OptionsParser.parse(new String[] {
         "hdfs://localhost:9820/source/first",
         "hdfs://localhost:9820/target/"});
-    Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB,
DELTA);
+    Assert.assertEquals(options.getMapBandwidth(), 0, DELTA);
 
     options = OptionsParser.parse(new String[] {
         "-bandwidth",
@@ -389,7 +389,7 @@ public class TestOptionsParser {
         + "deleteMissing=false, ignoreFailures=false, overwrite=false, "
         + "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, "
         + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
-        + "mapBandwidth=100.0, "
+        + "mapBandwidth=0.0, "
         + "copyStrategy='uniformsize', preserveStatus=[], "
         + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
         + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
@@ -572,6 +572,8 @@ public class TestOptionsParser {
     Configuration conf = new Configuration();
     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(),
false));
     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(),
false));
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
     DistCpOptions options = OptionsParser.parse(new String[] {
         "-atomic",
         "-i",
@@ -581,7 +583,7 @@ public class TestOptionsParser {
     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(),
false));
     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(),
false));
     Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1),
-        DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
+        -1.0, DELTA);
 
     conf = new Configuration();
     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(),
false));
@@ -603,6 +605,62 @@ public class TestOptionsParser {
   }
 
   @Test
+  public void testOptionsAppendToConfDoesntOverwriteBandwidth() {
+    Configuration conf = new Configuration();
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
+    DistCpOptions options = OptionsParser.parse(new String[] {
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0,
+        DELTA);
+
+    conf = new Configuration();
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
+    options = OptionsParser.parse(new String[] {
+        "-update",
+        "-delete",
+        "-pu",
+        "-bandwidth",
+        "77",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0,
+        DELTA);
+
+    conf = new Configuration();
+    conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88");
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88");
+    options = OptionsParser.parse(new String[] {
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0,
+        DELTA);
+
+    conf = new Configuration();
+    conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0");
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0");
+    options = OptionsParser.parse(new String[] {
+        "-bandwidth",
+        "99",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0,
+        DELTA);
+  }
+
+  @Test
   public void testAppendOption() {
     Configuration conf = new Configuration();
     Assert.assertFalse(conf.getBoolean(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message