hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject git commit: MAPREDUCE-6022. map_input_file is missing from streaming job environment. Contributed by Jason Lowe. (cherry picked from commit b056048114bf4701ef9dd22486db937cb589e81b)
Date Wed, 29 Oct 2014 17:30:28 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e88832dfb -> 45b6dea95


MAPREDUCE-6022. map_input_file is missing from streaming job
environment. Contributed by Jason Lowe.
(cherry picked from commit b056048114bf4701ef9dd22486db937cb589e81b)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45b6dea9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45b6dea9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45b6dea9

Branch: refs/heads/branch-2
Commit: 45b6dea95f5397282e2611f3dc6f68df0527529c
Parents: e88832d
Author: Kihwal Lee <kihwal@apache.org>
Authored: Wed Oct 29 12:30:08 2014 -0500
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Wed Oct 29 12:30:08 2014 -0500

----------------------------------------------------------------------
 .../org/apache/hadoop/conf/Configuration.java   | 26 ++++++++++++++++++++
 hadoop-mapreduce-project/CHANGES.txt            |  3 +++
 .../org/apache/hadoop/streaming/PipeMapRed.java |  4 ++-
 .../java/org/apache/hadoop/streaming/TrApp.java |  3 +++
 4 files changed, 35 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 62cc78d..1276865 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -559,6 +559,32 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   }
 
   /**
+   * Sets all deprecated properties that are not currently set but have a
+   * corresponding new property that is set. Useful for iterating the
+   * properties when all deprecated properties for currently set properties
+   * need to be present.
+   */
+  public void setDeprecatedProperties() {
+    DeprecationContext deprecations = deprecationContext.get();
+    Properties props = getProps();
+    Properties overlay = getOverlay();
+    for (Map.Entry<String, DeprecatedKeyInfo> entry :
+        deprecations.getDeprecatedKeyMap().entrySet()) {
+      String depKey = entry.getKey();
+      if (!overlay.contains(depKey)) {
+        for (String newKey : entry.getValue().newKeys) {
+          String val = overlay.getProperty(newKey);
+          if (val != null) {
+            props.setProperty(depKey, val);
+            overlay.setProperty(depKey, val);
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  /**
    * Checks for the presence of the property <code>name</code> in the
    * deprecation map. Returns the first of the list of new keys if present
    * in the deprecation map or the <code>name</code> itself. If the property

http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d60e3d5..6e245ec 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -231,6 +231,9 @@ Release 2.6.0 - UNRELEASED
     MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
     TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
 
+    MAPREDUCE-6022. map_input_file is missing from streaming job environment.
+    (jlowe via kihwal)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
index c8d099e..f47e756 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
@@ -235,7 +235,9 @@ public abstract class PipeMapRed {
     }
   }
 
-  void addJobConfToEnvironment(JobConf conf, Properties env) {
+  void addJobConfToEnvironment(JobConf jobconf, Properties env) {
+    JobConf conf = new JobConf(jobconf);
+    conf.setDeprecatedProperties();
     Iterator it = conf.iterator();
     while (it.hasNext()) {
       Map.Entry en = (Map.Entry) it.next();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/45b6dea9/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
index e413261..d058332 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
@@ -57,6 +57,9 @@ public class TrApp
     // the FileSplit context properties are not available in local hadoop..
     // so can't check them in this test.
 
+    // verify some deprecated properties appear for older stream jobs
+    expect("map_input_file", env.getProperty("mapreduce_map_input_file"));
+    expect("map_input_length", env.getProperty("mapreduce_map_input_length"));
   }
 
   // this runs in a subprocess; won't use JUnit's assertTrue()    


Mime
View raw message