hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1367102 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/conf/ src/test/java/org/apache/hadoop/conf/
Date Mon, 30 Jul 2012 14:20:35 GMT
Author: bobby
Date: Mon Jul 30 14:20:35 2012
New Revision: 1367102

URL: http://svn.apache.org/viewvc?rev=1367102&view=rev
Log:
svn merge -c 1301551,1300642,1332821,1303884 FIXES:MAPREDUCE-4010,HADOOP-8167,HADOOP-8172,HADOOP-8197
Patches to fix OOZIE-761

Modified:
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestDeprecatedKeys.java

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1367102&r1=1367101&r2=1367102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon
Jul 30 14:20:35 2012
@@ -123,6 +123,13 @@ Release 0.23.3 - UNRELEASED
     HADOOP-8613. AbstractDelegationTokenIdentifier#getUser() should set token
     auth type. (daryn)
 
+    HADOOP-8167. Configuration deprecation logic breaks backwards compatibility (tucu)
+
+    HADOOP-8172. Configuration no longer sets all keys in a deprecated key 
+    list. (Anupam Seth via bobby)
+
+    HADOOP-8197. Configuration logs WARNs on every use of a deprecated key (tucu)
+
 Release 0.23.2 - UNRELEASED 
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1367102&r1=1367101&r2=1367102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Mon Jul 30 14:20:35 2012
@@ -293,10 +293,18 @@ public class Configuration implements It
    * This is to be used only by the developers in order to add deprecation of
    * keys, and attempts to call this method after loading resources once,
    * would lead to <tt>UnsupportedOperationException</tt>
+   * 
+   * If a key is deprecated in favor of multiple keys, they are all treated as 
+   * aliases of each other, and setting any one of them resets all the others 
+   * to the new value.
+   * 
    * @param key
    * @param newKeys
    * @param customMessage
+   * @deprecated use {@link addDeprecation(String key, String newKey,
+      String customMessage)} instead
    */
+  @Deprecated
   public synchronized static void addDeprecation(String key, String[] newKeys,
       String customMessage) {
     if (key == null || key.length() == 0 ||
@@ -312,6 +320,22 @@ public class Configuration implements It
       }
     }
   }
+  
+  /**
+   * Adds the deprecated key to the deprecation map.
+   * It does not override any existing entries in the deprecation map.
+   * This is to be used only by the developers in order to add deprecation of
+   * keys, and attempts to call this method after loading resources once,
+   * would lead to <tt>UnsupportedOperationException</tt>
+   * 
+   * @param key
+   * @param newKey
+   * @param customMessage
+   */
+  public synchronized static void addDeprecation(String key, String newKey,
+	      String customMessage) {
+	  addDeprecation(key, new String[] {newKey}, customMessage);
+  }
 
   /**
    * Adds the deprecated key to the deprecation map when no custom message
@@ -321,24 +345,74 @@ public class Configuration implements It
    * keys, and attempts to call this method after loading resources once,
    * would lead to <tt>UnsupportedOperationException</tt>
    * 
+   * If a key is deprecated in favor of multiple keys, they are all treated as 
+   * aliases of each other, and setting any one of them resets all the others 
+   * to the new value.
+   * 
    * @param key Key that is to be deprecated
    * @param newKeys list of keys that take up the values of deprecated key
+   * @deprecated use {@link addDeprecation(String key, String newKey)} instead
    */
+  @Deprecated
   public synchronized static void addDeprecation(String key, String[] newKeys) {
     addDeprecation(key, newKeys, null);
   }
   
   /**
+   * Adds the deprecated key to the deprecation map when no custom message
+   * is provided.
+   * It does not override any existing entries in the deprecation map.
+   * This is to be used only by the developers in order to add deprecation of
+   * keys, and attempts to call this method after loading resources once,
+   * would lead to <tt>UnsupportedOperationException</tt>
+   * 
+   * @param key Key that is to be deprecated
+   * @param newKey key that takes up the value of deprecated key
+   */
+  public synchronized static void addDeprecation(String key, String newKey) {
+	addDeprecation(key, new String[] {newKey}, null);
+  }
+  
+  /**
    * checks whether the given <code>key</code> is deprecated.
    * 
    * @param key the parameter which is to be checked for deprecation
    * @return <code>true</code> if the key is deprecated and 
    *         <code>false</code> otherwise.
    */
-  private static boolean isDeprecated(String key) {
+  public static boolean isDeprecated(String key) {
     return deprecatedKeyMap.containsKey(key);
   }
- 
+
+  /**
+   * Returns the alternate name for a key if the property name is deprecated
+   * or if deprecates a property name.
+   *
+   * @param name property name.
+   * @return alternate name.
+   */
+  private String[] getAlternateNames(String name) {
+    String oldName, altNames[] = null;
+    DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
+    if (keyInfo == null) {
+      altNames = (reverseDeprecatedKeyMap.get(name) != null ) ? 
+        new String [] {reverseDeprecatedKeyMap.get(name)} : null;
+      if(altNames != null && altNames.length > 0) {
+    	//To help look for other new configs for this deprecated config
+    	keyInfo = deprecatedKeyMap.get(altNames[0]);
+      }      
+    } 
+    if(keyInfo != null && keyInfo.newKeys.length > 0) {
+      List<String> list = new ArrayList<String>(); 
+      if(altNames != null) {
+    	  list.addAll(Arrays.asList(altNames));
+      }
+      list.addAll(Arrays.asList(keyInfo.newKeys));
+      altNames = list.toArray(new String[list.size()]);
+    }
+    return altNames;
+  }
+
   /**
    * Checks for the presence of the property <code>name</code> in the
    * deprecation map. Returns the first of the list of new keys if present
@@ -351,31 +425,29 @@ public class Configuration implements It
    * @return the first property in the list of properties mapping
    *         the <code>name</code> or the <code>name</code> itself.
    */
-  private String handleDeprecation(String name) {
-    if (isDeprecated(name)) {
+  private String[] handleDeprecation(String name) {
+    ArrayList<String > names = new ArrayList<String>();
+	if (isDeprecated(name)) {
       DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
-      if (!keyInfo.accessed) {
-        LOG.warn(keyInfo.getWarningMessage(name));
-      }
+      warnOnceIfDeprecated(name);
       for (String newKey : keyInfo.newKeys) {
         if(newKey != null) {
-          name = newKey;
-          break;
+          names.add(newKey);
         }
       }
     }
-    String deprecatedKey = reverseDeprecatedKeyMap.get(name);
-    if (deprecatedKey != null && !getOverlay().containsKey(name) &&
-        getOverlay().containsKey(deprecatedKey)) {
-      getProps().setProperty(name, getOverlay().getProperty(deprecatedKey));
-      getOverlay().setProperty(name, getOverlay().getProperty(deprecatedKey));
-      
-      DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(deprecatedKey);
-      if (!keyInfo.accessed) {
-        LOG.warn(keyInfo.getWarningMessage(deprecatedKey));
-      }
+    if(names.size() == 0) {
+      names.add(name);
+    }
+    for(String n : names) {
+	  String deprecatedKey = reverseDeprecatedKeyMap.get(n);
+	  if (deprecatedKey != null && !getOverlay().containsKey(n) &&
+	      getOverlay().containsKey(deprecatedKey)) {
+	    getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
+	    getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
+	  }
     }
-    return name;
+    return names.toArray(new String[names.size()]);
   }
  
   private void handleDeprecation() {
@@ -626,8 +698,12 @@ public class Configuration implements It
    *         or null if no such property exists.
    */
   public String get(String name) {
-    name = handleDeprecation(name);
-    return substituteVars(getProps().getProperty(name));
+    String[] names = handleDeprecation(name);
+    String result = null;
+    for(String n : names) {
+      result = substituteVars(getProps().getProperty(n));
+    }
+    return result;
   }
   
   /**
@@ -664,14 +740,18 @@ public class Configuration implements It
    *         its replacing property and null if no such property exists.
    */
   public String getRaw(String name) {
-    name = handleDeprecation(name);
-    return getProps().getProperty(name);
+    String[] names = handleDeprecation(name);
+    String result = null;
+    for(String n : names) {
+      result = getProps().getProperty(n);
+    }
+    return result;
   }
 
   /** 
    * Set the <code>value</code> of the <code>name</code> property.
If 
-   * <code>name</code> is deprecated, it sets the <code>value</code>
to the keys
-   * that replace the deprecated key.
+   * <code>name</code> is deprecated or there is a deprecated name associated
to it,
+   * it sets the value to both names.
    * 
    * @param name property name.
    * @param value property value.
@@ -694,37 +774,47 @@ public class Configuration implements It
     if (deprecatedKeyMap.isEmpty()) {
       getProps();
     }
-    if (!isDeprecated(name)) {
-      getOverlay().setProperty(name, value);
-      getProps().setProperty(name, value);
-      if(source == null) {
-        updatingResource.put(name, new String[] {"programatically"});
-      } else {
-        updatingResource.put(name, new String[] {source});
-      }
+    getOverlay().setProperty(name, value);
+    getProps().setProperty(name, value);
+    if(source == null) {
+      updatingResource.put(name, new String[] {"programatically"});
+    } else {
+      updatingResource.put(name, new String[] {source});
     }
-    else {
-      DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
+    String[] altNames = getAlternateNames(name);
+    if (altNames != null && altNames.length > 0) {
       String altSource = "because " + name + " is deprecated";
-      LOG.warn(keyInfo.getWarningMessage(name));
-      for (String newKey : keyInfo.newKeys) {
-        if(!newKey.equals(name)) {
-          getOverlay().setProperty(newKey, value);
-          getProps().setProperty(newKey, value);
-          updatingResource.put(newKey, new String[] {altSource});
+      for(String altName : altNames) {
+        if(!altName.equals(name)) {
+          getOverlay().setProperty(altName, value);
+          getProps().setProperty(altName, value);
+          updatingResource.put(altName, new String[] {altSource});
         }
       }
     }
+    warnOnceIfDeprecated(name);
   }
-  
+
+  private void warnOnceIfDeprecated(String name) {
+    DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
+    if (keyInfo != null && !keyInfo.accessed) {
+      LOG.warn(keyInfo.getWarningMessage(name));
+    }
+  }
+
   /**
    * Unset a previously set property.
    */
   public synchronized void unset(String name) {
-    name = handleDeprecation(name);
-
+    String[] altNames = getAlternateNames(name);
     getOverlay().remove(name);
     getProps().remove(name);
+    if (altNames !=null && altNames.length > 0) {
+      for(String altName : altNames) {
+    	getOverlay().remove(altName);
+    	getProps().remove(altName);
+      }
+    }
   }
 
   /**
@@ -758,8 +848,12 @@ public class Configuration implements It
    *         doesn't exist.                    
    */
   public String get(String name, String defaultValue) {
-    name = handleDeprecation(name);
-    return substituteVars(getProps().getProperty(name, defaultValue));
+    String[] names = handleDeprecation(name);
+    String result = null;
+    for(String n : names) {
+      result = substituteVars(getProps().getProperty(n, defaultValue));
+    }
+    return result;
   }
     
   /** 

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java?rev=1367102&r1=1367101&r2=1367102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
Mon Jul 30 14:20:35 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.conf;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
@@ -27,6 +28,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.Map;
 
 import org.apache.hadoop.fs.Path;
 import org.junit.After;
@@ -162,7 +164,7 @@ public class TestConfigurationDeprecatio
     conf.set("Y", "y");
     conf.set("Z", "z");
     // get old key
-    assertEquals("y", conf.get("X"));
+    assertEquals("z", conf.get("X"));
   }
 
   /**
@@ -270,4 +272,56 @@ public class TestConfigurationDeprecatio
         new String[]{ "tests.fake-default.new-key" });
     assertEquals("hello", conf.get("tests.fake-default.new-key"));
   }
+
+  @Test
+  public void testIteratorWithDeprecatedKeys() {
+    Configuration conf = new Configuration();
+    Configuration.addDeprecation("dK", new String[]{"nK"});
+    conf.set("k", "v");
+    conf.set("dK", "V");
+    assertEquals("V", conf.get("dK"));
+    assertEquals("V", conf.get("nK"));
+    conf.set("nK", "VV");
+    assertEquals("VV", conf.get("dK"));
+    assertEquals("VV", conf.get("nK"));
+    boolean kFound = false;
+    boolean dKFound = false;
+    boolean nKFound = false;
+    for (Map.Entry<String, String> entry : conf) {
+      if (entry.getKey().equals("k")) {
+        assertEquals("v", entry.getValue());
+        kFound = true;
+      }
+      if (entry.getKey().equals("dK")) {
+        assertEquals("VV", entry.getValue());
+        dKFound = true;
+      }
+      if (entry.getKey().equals("nK")) {
+        assertEquals("VV", entry.getValue());
+        nKFound = true;
+      }
+    }
+    assertTrue("regular Key not found", kFound);
+    assertTrue("deprecated Key not found", dKFound);
+    assertTrue("new Key not found", nKFound);
+  }
+  
+  @Test
+  public void testUnsetWithDeprecatedKeys() {
+    Configuration conf = new Configuration();
+    Configuration.addDeprecation("dK", new String[]{"nK"});
+    conf.set("nK", "VV");
+    assertEquals("VV", conf.get("dK"));
+    assertEquals("VV", conf.get("nK"));
+    conf.unset("dK");
+    assertNull(conf.get("dK"));
+    assertNull(conf.get("nK"));
+    conf.set("nK", "VV");
+    assertEquals("VV", conf.get("dK"));
+    assertEquals("VV", conf.get("nK"));
+    conf.unset("nK");
+    assertNull(conf.get("dK"));
+    assertNull(conf.get("nK"));
+  }
+
 }

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestDeprecatedKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestDeprecatedKeys.java?rev=1367102&r1=1367101&r2=1367102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestDeprecatedKeys.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestDeprecatedKeys.java
Mon Jul 30 14:20:35 2012
@@ -18,10 +18,15 @@
 
 package org.apache.hadoop.conf;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.io.ByteArrayOutputStream;
+import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.junit.Test;
 
 import junit.framework.TestCase;
 
@@ -31,6 +36,7 @@ public class TestDeprecatedKeys extends 
   public void testDeprecatedKeys() throws Exception {
     Configuration conf = new Configuration();
     conf.set("topology.script.file.name", "xyz");
+    conf.set("topology.script.file.name", "xyz");
     String scriptFile = conf.get(CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY);
     assertTrue(scriptFile.equals("xyz")) ;
   }
@@ -52,4 +58,49 @@ public class TestDeprecatedKeys extends 
     assertTrue(fileContents.contains("old.config.yet.to.be.deprecated"));
     assertTrue(fileContents.contains("new.conf.to.replace.deprecated.conf"));
   }
+  
+  @Test
+  public void testIteratorWithDeprecatedKeysMappedToMultipleNewKeys() {
+    Configuration conf = new Configuration();
+    Configuration.addDeprecation("dK", new String[]{"nK1", "nK2"});
+    conf.set("k", "v");
+    conf.set("dK", "V");
+    assertEquals("V", conf.get("dK"));
+    assertEquals("V", conf.get("nK1"));
+    assertEquals("V", conf.get("nK2"));
+    conf.set("nK1", "VV");
+    assertEquals("VV", conf.get("dK"));
+    assertEquals("VV", conf.get("nK1"));
+    assertEquals("VV", conf.get("nK2"));
+    conf.set("nK2", "VVV");
+    assertEquals("VVV", conf.get("dK"));
+    assertEquals("VVV", conf.get("nK2"));
+    assertEquals("VVV", conf.get("nK1"));
+    boolean kFound = false;
+    boolean dKFound = false;
+    boolean nK1Found = false;
+    boolean nK2Found = false;
+    for (Map.Entry<String, String> entry : conf) {
+      if (entry.getKey().equals("k")) {
+        assertEquals("v", entry.getValue());
+        kFound = true;
+      }
+      if (entry.getKey().equals("dK")) {
+        assertEquals("VVV", entry.getValue());
+        dKFound = true;
+      }
+      if (entry.getKey().equals("nK1")) {
+        assertEquals("VVV", entry.getValue());
+        nK1Found = true;
+      }
+      if (entry.getKey().equals("nK2")) {
+        assertEquals("VVV", entry.getValue());
+        nK2Found = true;
+      }
+    }
+    assertTrue("regular Key not found", kFound);
+    assertTrue("deprecated Key not found", dKFound);
+    assertTrue("new Key 1 not found", nK1Found);
+    assertTrue("new Key 2 not found", nK2Found);
+  }
 }



Mime
View raw message