hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [03/21] hadoop git commit: HADOOP-6801. io.sort.mb and io.sort.factor were renamed and moved to mapreduce but are still in CommonConfigurationKeysPublic.java and used in SequenceFile.java.
Date Mon, 06 Mar 2017 18:18:23 GMT
HADOOP-6801. io.sort.mb and io.sort.factor were renamed and moved to mapreduce but are still
in CommonConfigurationKeysPublic.java and used in SequenceFile.java.

This closes #146

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb5a1795
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb5a1795
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb5a1795

Branch: refs/heads/YARN-5734
Commit: eb5a17954a758fdb1f3f29ef34e129d5f37d3a26
Parents: 19a1fc6
Author: Harsh J <harsh@cloudera.com>
Authored: Wed Oct 26 13:21:51 2016 +0530
Committer: Akira Ajisaka <aajisaka@apache.org>
Committed: Fri Mar 3 18:38:04 2017 +0900

----------------------------------------------------------------------
 .../fs/CommonConfigurationKeysPublic.java       | 29 ++++++++-
 .../java/org/apache/hadoop/io/SequenceFile.java | 20 +++++-
 .../src/main/resources/core-default.xml         | 19 ++++++
 .../org/apache/hadoop/io/TestSequenceFile.java  | 68 +++++++++++++++++++-
 4 files changed, 131 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index e1feda1..6b267dc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -250,18 +250,43 @@ public class CommonConfigurationKeysPublic {
    * @deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
    * in mapred-default.xml
    * See https://issues.apache.org/jira/browse/HADOOP-6801
+   *
+   * For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+   * instead, see {@link #SEQ_IO_SORT_MB_KEY}.
    */
   public static final String  IO_SORT_MB_KEY = "io.sort.mb";
-  /** Default value for IO_SORT_MB_DEFAULT */
+  /** Default value for {@link #IO_SORT_MB_KEY}. */
   public static final int     IO_SORT_MB_DEFAULT = 100;
   /**
    * @deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
    * in mapred-default.xml
    * See https://issues.apache.org/jira/browse/HADOOP-6801
+   *
+   * For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+   * instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.
    */
   public static final String  IO_SORT_FACTOR_KEY = "io.sort.factor";
-  /** Default value for IO_SORT_FACTOR_DEFAULT */
+  /** Default value for {@link #IO_SORT_FACTOR_KEY}. */
   public static final int     IO_SORT_FACTOR_DEFAULT = 100;
+
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String  SEQ_IO_SORT_MB_KEY = "seq.io.sort.mb";
+  /** Default value for {@link #SEQ_IO_SORT_MB_KEY}. */
+  public static final int     SEQ_IO_SORT_MB_DEFAULT = 100;
+
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String  SEQ_IO_SORT_FACTOR_KEY = "seq.io.sort.factor";
+  /** Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}. */
+  public static final int     SEQ_IO_SORT_FACTOR_DEFAULT = 100;
+
   /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index c510ff7..de0bf4f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -2816,14 +2816,30 @@ public class SequenceFile {
     }
 
     /** Sort and merge using an arbitrary {@link RawComparator}. */
+    @SuppressWarnings("deprecation")
     public Sorter(FileSystem fs, RawComparator comparator, Class keyClass,
                   Class valClass, Configuration conf, Metadata metadata) {
       this.fs = fs;
       this.comparator = comparator;
       this.keyClass = keyClass;
       this.valClass = valClass;
-      this.memory = conf.getInt("io.sort.mb", 100) * 1024 * 1024;
-      this.factor = conf.getInt("io.sort.factor", 100);
+      // Remember to fall-back on the deprecated MB and Factor keys
+      // until they are removed away permanently.
+      if (conf.get(CommonConfigurationKeys.IO_SORT_MB_KEY) != null) {
+        this.memory = conf.getInt(CommonConfigurationKeys.IO_SORT_MB_KEY,
+          CommonConfigurationKeys.SEQ_IO_SORT_MB_DEFAULT) * 1024 * 1024;
+      } else {
+        this.memory = conf.getInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY,
+          CommonConfigurationKeys.SEQ_IO_SORT_MB_DEFAULT) * 1024 * 1024;
+      }
+      if (conf.get(CommonConfigurationKeys.IO_SORT_FACTOR_KEY) != null) {
+        this.factor = conf.getInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY,
+            CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_DEFAULT);
+      } else {
+        this.factor = conf.getInt(
+            CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY,
+            CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_DEFAULT);
+      }
       this.conf = conf;
       this.metadata = metadata;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 08ca05b..35be56b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2476,4 +2476,23 @@
       in audit logs.
     </description>
   </property>
+<!-- SequenceFile's Sorter properties -->
+  <property>
+    <name>seq.io.sort.mb</name>
+    <value>100</value>
+    <description>
+      The total amount of buffer memory to use while sorting files,
+      while using SequenceFile.Sorter, in megabytes. By default,
+      gives each merge stream 1MB, which should minimize seeks.
+    </description>
+  </property>
+  <property>
+    <name>seq.io.sort.factor</name>
+    <value>100</value>
+    <description>
+      The number of streams to merge at once while sorting
+      files using SequenceFile.Sorter.
+      This determines the number of open file handles.
+    </description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb5a1795/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
index b76cff6..e97ab6a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
@@ -38,6 +38,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import org.mockito.Mockito;
 
 
@@ -54,7 +55,72 @@ public class TestSequenceFile {
     compressedSeqFileTest(new DefaultCodec());
     LOG.info("Successfully tested SequenceFile with DefaultCodec");
   }
-  
+
+  @SuppressWarnings("deprecation")
+  public void testSorterProperties() throws IOException {
+    // Test to ensure that deprecated properties have no default
+    // references anymore.
+    Configuration config = new Configuration();
+    assertNull("The deprecated sort memory property "
+        + CommonConfigurationKeys.IO_SORT_MB_KEY
+        + " must not exist in any core-*.xml files.",
+        config.get(CommonConfigurationKeys.IO_SORT_MB_KEY));
+    assertNull("The deprecated sort factor property "
+        + CommonConfigurationKeys.IO_SORT_FACTOR_KEY
+        + " must not exist in any core-*.xml files.",
+        config.get(CommonConfigurationKeys.IO_SORT_FACTOR_KEY));
+
+    // Test deprecated property honoring
+    // Set different values for old and new property names
+    // and compare which one gets loaded
+    config = new Configuration();
+    FileSystem fs = FileSystem.get(config);
+    config.setInt(CommonConfigurationKeys.IO_SORT_MB_KEY, 10);
+    config.setInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY, 10);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY, 20);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY, 20);
+    SequenceFile.Sorter sorter = new SequenceFile.Sorter(
+        fs, Text.class, Text.class, config);
+    assertEquals("Deprecated memory conf must be honored over newer property",
+        10*1024*1024, sorter.getMemory());
+    assertEquals("Deprecated factor conf must be honored over newer property",
+        10, sorter.getFactor());
+
+    // Test deprecated properties (graceful deprecation)
+    config = new Configuration();
+    fs = FileSystem.get(config);
+    config.setInt(CommonConfigurationKeys.IO_SORT_MB_KEY, 10);
+    config.setInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY, 10);
+    sorter = new SequenceFile.Sorter(
+        fs, Text.class, Text.class, config);
+    assertEquals("Deprecated memory property "
+        + CommonConfigurationKeys.IO_SORT_MB_KEY
+        + " must get properly applied.",
+        10*1024*1024, // In bytes
+        sorter.getMemory());
+    assertEquals("Deprecated sort factor property "
+        + CommonConfigurationKeys.IO_SORT_FACTOR_KEY
+        + " must get properly applied.",
+        10, sorter.getFactor());
+
+    // Test regular properties (graceful deprecation)
+    config = new Configuration();
+    fs = FileSystem.get(config);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY, 20);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY, 20);
+    sorter = new SequenceFile.Sorter(
+        fs, Text.class, Text.class, config);
+    assertEquals("Memory property "
+        + CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY
+        + " must get properly applied if present.",
+        20*1024*1024, // In bytes
+        sorter.getMemory());
+    assertEquals("Merge factor property "
+        + CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY
+        + " must get properly applied if present.",
+        20, sorter.getFactor());
+  }
+
   public void compressedSeqFileTest(CompressionCodec codec) throws Exception {
     int count = 1024 * 10;
     int megabytes = 1;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message