hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1538408 - in /hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common: ./ src/main/java/ src/main/java/org/apache/hadoop/conf/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/h...
Date Sun, 03 Nov 2013 17:51:54 GMT
Author: szetszwo
Date: Sun Nov  3 17:51:52 2013
New Revision: 1538408

URL: http://svn.apache.org/r1538408
Log:
Merge r1537584 through r1538407 from trunk.

Modified:
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt   (contents,
props changed)
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/  
(props changed)
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
    hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt Sun Nov
 3 17:51:52 2013
@@ -428,6 +428,9 @@ Release 2.2.1 - UNRELEASED
     HADOOP-10046. Print a log message when SSL is enabled.
     (David S. Wang via wang)
 
+    HADOOP-10079. log a warning message if group resolution takes too long.
+    (cmccabe)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -452,6 +455,11 @@ Release 2.2.1 - UNRELEASED
     HADOOP-10072. TestNfsExports#testMultiMatchers fails due to non-deterministic
     timing around cache expiry check. (cnauroth)
 
+    HADOOP-9898. Set SO_KEEPALIVE on all our sockets. (todd via wang)
+
+    HADOOP-9478. Fix race conditions during the initialization of Configuration
+    related to deprecatedKeyMap (cmccabe)
+
 Release 2.2.0 - 2013-10-13
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1536182-1536571,1536573-1538407

Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1536890-1538407

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Sun Nov  3 17:51:52 2013
@@ -55,6 +55,8 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -65,6 +67,7 @@ import javax.xml.transform.TransformerFa
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
+import org.apache.commons.collections.map.UnmodifiableMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -87,6 +90,7 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.w3c.dom.Text;
 import org.xml.sax.SAXException;
+
 import com.google.common.base.Preconditions;
 
 /** 
@@ -254,13 +258,13 @@ public class Configuration implements It
    * warning message which can be logged whenever the deprecated key is used.
    */
   private static class DeprecatedKeyInfo {
-    private String[] newKeys;
-    private String customMessage;
-    private boolean accessed;
+    private final String[] newKeys;
+    private final String customMessage;
+    private final AtomicBoolean accessed = new AtomicBoolean(false);
+
     DeprecatedKeyInfo(String[] newKeys, String customMessage) {
       this.newKeys = newKeys;
       this.customMessage = customMessage;
-      accessed = false;
     }
 
     /**
@@ -286,26 +290,170 @@ public class Configuration implements It
       else {
         warningMessage = customMessage;
       }
-      accessed = true;
       return warningMessage;
     }
+
+    boolean getAndSetAccessed() {
+      return accessed.getAndSet(true);
+    }
+
+    public void clearAccessed() {
+      accessed.set(false);
+    }
   }
   
   /**
-   * Stores the deprecated keys, the new keys which replace the deprecated keys
-   * and custom message(if any provided).
+   * A pending addition to the global set of deprecated keys.
    */
-  private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap = 
-      new HashMap<String, DeprecatedKeyInfo>();
+  public static class DeprecationDelta {
+    private final String key;
+    private final String[] newKeys;
+    private final String customMessage;
+
+    DeprecationDelta(String key, String[] newKeys, String customMessage) {
+      Preconditions.checkNotNull(key);
+      Preconditions.checkNotNull(newKeys);
+      Preconditions.checkArgument(newKeys.length > 0);
+      this.key = key;
+      this.newKeys = newKeys;
+      this.customMessage = customMessage;
+    }
+
+    public DeprecationDelta(String key, String newKey, String customMessage) {
+      this(key, new String[] { newKey }, customMessage);
+    }
+
+    public DeprecationDelta(String key, String newKey) {
+      this(key, new String[] { newKey }, null);
+    }
+
+    public String getKey() {
+      return key;
+    }
+
+    public String[] getNewKeys() {
+      return newKeys;
+    }
+
+    public String getCustomMessage() {
+      return customMessage;
+    }
+  }
+
+  /**
+   * The set of all keys which are deprecated.
+   *
+   * DeprecationContext objects are immutable.
+   */
+  private static class DeprecationContext {
+    /**
+     * Stores the deprecated keys, the new keys which replace the deprecated keys
+     * and custom message(if any provided).
+     */
+    private final Map<String, DeprecatedKeyInfo> deprecatedKeyMap;
+
+    /**
+     * Stores a mapping from superseding keys to the keys which they deprecate.
+     */
+    private final Map<String, String> reverseDeprecatedKeyMap;
+
+    /**
+     * Create a new DeprecationContext by copying a previous DeprecationContext
+     * and adding some deltas.
+     *
+     * @param other   The previous deprecation context to copy, or null to start
+     *                from nothing.
+     * @param deltas  The deltas to apply.
+     */
+    @SuppressWarnings("unchecked")
+    DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
+      HashMap<String, DeprecatedKeyInfo> newDeprecatedKeyMap = 
+        new HashMap<String, DeprecatedKeyInfo>();
+      HashMap<String, String> newReverseDeprecatedKeyMap =
+        new HashMap<String, String>();
+      if (other != null) {
+        for (Entry<String, DeprecatedKeyInfo> entry :
+            other.deprecatedKeyMap.entrySet()) {
+          newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
+        }
+        for (Entry<String, String> entry :
+            other.reverseDeprecatedKeyMap.entrySet()) {
+          newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
+        }
+      }
+      for (DeprecationDelta delta : deltas) {
+        if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
+          DeprecatedKeyInfo newKeyInfo =
+            new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
+          newDeprecatedKeyMap.put(delta.key, newKeyInfo);
+          for (String newKey : delta.getNewKeys()) {
+            newReverseDeprecatedKeyMap.put(newKey, delta.key);
+          }
+        }
+      }
+      this.deprecatedKeyMap =
+        UnmodifiableMap.decorate(newDeprecatedKeyMap);
+      this.reverseDeprecatedKeyMap =
+        UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
+    }
+
+    Map<String, DeprecatedKeyInfo> getDeprecatedKeyMap() {
+      return deprecatedKeyMap;
+    }
+
+    Map<String, String> getReverseDeprecatedKeyMap() {
+      return reverseDeprecatedKeyMap;
+    }
+  }
   
+  private static DeprecationDelta[] defaultDeprecations = 
+    new DeprecationDelta[] {
+      new DeprecationDelta("topology.script.file.name", 
+        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
+      new DeprecationDelta("topology.script.number.args", 
+        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
+      new DeprecationDelta("hadoop.configured.node.mapping", 
+        CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
+      new DeprecationDelta("topology.node.switch.mapping.impl", 
+        CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
+      new DeprecationDelta("dfs.df.interval", 
+        CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
+      new DeprecationDelta("hadoop.native.lib", 
+        CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY),
+      new DeprecationDelta("fs.default.name", 
+        CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
+      new DeprecationDelta("dfs.umaskmode",
+        CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY)
+    };
+
+  /**
+   * The global DeprecationContext.
+   */
+  private static AtomicReference<DeprecationContext> deprecationContext =
+      new AtomicReference<DeprecationContext>(
+          new DeprecationContext(null, defaultDeprecations));
+
   /**
-   * Stores a mapping from superseding keys to the keys which they deprecate.
+   * Adds a set of deprecated keys to the global deprecations.
+   *
+   * This method is lockless.  It works by means of creating a new
+   * DeprecationContext based on the old one, and then atomically swapping in
+   * the new context.  If someone else updated the context in between us reading
+   * the old context and swapping in the new one, we try again until we win the
+   * race.
+   *
+   * @param deltas   The deprecations to add.
    */
-  private static Map<String, String> reverseDeprecatedKeyMap =
-      new HashMap<String, String>();
+  public static void addDeprecations(DeprecationDelta[] deltas) {
+    DeprecationContext prev, next;
+    do {
+      prev = deprecationContext.get();
+      next = new DeprecationContext(prev, deltas);
+    } while (!deprecationContext.compareAndSet(prev, next));
+  }
 
   /**
-   * Adds the deprecated key to the deprecation map.
+   * Adds the deprecated key to the global deprecation map.
    * It does not override any existing entries in the deprecation map.
    * This is to be used only by the developers in order to add deprecation of
    * keys, and attempts to call this method after loading resources once,
@@ -314,6 +462,9 @@ public class Configuration implements It
    * If a key is deprecated in favor of multiple keys, they are all treated as 
    * aliases of each other, and setting any one of them resets all the others 
    * to the new value.
+   *
+   * If you have multiple deprecation entries to add, it is more efficient to
+   * use #addDeprecations(DeprecationDelta[] deltas) instead.
    * 
    * @param key
    * @param newKeys
@@ -322,41 +473,35 @@ public class Configuration implements It
       String customMessage)} instead
    */
   @Deprecated
-  public synchronized static void addDeprecation(String key, String[] newKeys,
+  public static void addDeprecation(String key, String[] newKeys,
       String customMessage) {
-    if (key == null || key.length() == 0 ||
-        newKeys == null || newKeys.length == 0) {
-      throw new IllegalArgumentException();
-    }
-    if (!isDeprecated(key)) {
-      DeprecatedKeyInfo newKeyInfo;
-      newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage);
-      deprecatedKeyMap.put(key, newKeyInfo);
-      for (String newKey : newKeys) {
-        reverseDeprecatedKeyMap.put(newKey, key);
-      }
-    }
+    addDeprecations(new DeprecationDelta[] {
+      new DeprecationDelta(key, newKeys, customMessage)
+    });
   }
-  
+
   /**
-   * Adds the deprecated key to the deprecation map.
+   * Adds the deprecated key to the global deprecation map.
    * It does not override any existing entries in the deprecation map.
    * This is to be used only by the developers in order to add deprecation of
    * keys, and attempts to call this method after loading resources once,
    * would lead to <tt>UnsupportedOperationException</tt>
    * 
+   * If you have multiple deprecation entries to add, it is more efficient to
+   * use #addDeprecations(DeprecationDelta[] deltas) instead.
+   *
    * @param key
    * @param newKey
    * @param customMessage
    */
-  public synchronized static void addDeprecation(String key, String newKey,
+  public static void addDeprecation(String key, String newKey,
 	      String customMessage) {
 	  addDeprecation(key, new String[] {newKey}, customMessage);
   }
 
   /**
-   * Adds the deprecated key to the deprecation map when no custom message
-   * is provided.
+   * Adds the deprecated key to the global deprecation map when no custom
+   * message is provided.
    * It does not override any existing entries in the deprecation map.
    * This is to be used only by the developers in order to add deprecation of
    * keys, and attempts to call this method after loading resources once,
@@ -366,28 +511,34 @@ public class Configuration implements It
    * aliases of each other, and setting any one of them resets all the others 
    * to the new value.
    * 
+   * If you have multiple deprecation entries to add, it is more efficient to
+   * use #addDeprecations(DeprecationDelta[] deltas) instead.
+   *
    * @param key Key that is to be deprecated
    * @param newKeys list of keys that take up the values of deprecated key
    * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
    */
   @Deprecated
-  public synchronized static void addDeprecation(String key, String[] newKeys) {
+  public static void addDeprecation(String key, String[] newKeys) {
     addDeprecation(key, newKeys, null);
   }
   
   /**
-   * Adds the deprecated key to the deprecation map when no custom message
-   * is provided.
+   * Adds the deprecated key to the global deprecation map when no custom
+   * message is provided.
    * It does not override any existing entries in the deprecation map.
    * This is to be used only by the developers in order to add deprecation of
    * keys, and attempts to call this method after loading resources once,
    * would lead to <tt>UnsupportedOperationException</tt>
    * 
+   * If you have multiple deprecation entries to add, it is more efficient to
+   * use #addDeprecations(DeprecationDelta[] deltas) instead.
+   *
    * @param key Key that is to be deprecated
    * @param newKey key that takes up the value of deprecated key
    */
-  public synchronized static void addDeprecation(String key, String newKey) {
-	addDeprecation(key, new String[] {newKey}, null);
+  public static void addDeprecation(String key, String newKey) {
+    addDeprecation(key, new String[] {newKey}, null);
   }
   
   /**
@@ -398,7 +549,7 @@ public class Configuration implements It
    *         <code>false</code> otherwise.
    */
   public static boolean isDeprecated(String key) {
-    return deprecatedKeyMap.containsKey(key);
+    return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
   }
 
   /**
@@ -410,13 +561,14 @@ public class Configuration implements It
    */
   private String[] getAlternateNames(String name) {
     String altNames[] = null;
-    DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
+    DeprecationContext cur = deprecationContext.get();
+    DeprecatedKeyInfo keyInfo = cur.getDeprecatedKeyMap().get(name);
     if (keyInfo == null) {
-      altNames = (reverseDeprecatedKeyMap.get(name) != null ) ? 
-        new String [] {reverseDeprecatedKeyMap.get(name)} : null;
+      altNames = (cur.getReverseDeprecatedKeyMap().get(name) != null ) ? 
+        new String [] {cur.getReverseDeprecatedKeyMap().get(name)} : null;
       if(altNames != null && altNames.length > 0) {
     	//To help look for other new configs for this deprecated config
-    	keyInfo = deprecatedKeyMap.get(altNames[0]);
+    	keyInfo = cur.getDeprecatedKeyMap().get(altNames[0]);
       }      
     } 
     if(keyInfo != null && keyInfo.newKeys.length > 0) {
@@ -442,11 +594,12 @@ public class Configuration implements It
    * @return the first property in the list of properties mapping
    *         the <code>name</code> or the <code>name</code> itself.
    */
-  private String[] handleDeprecation(String name) {
+  private String[] handleDeprecation(DeprecationContext deprecations,
+      String name) {
     ArrayList<String > names = new ArrayList<String>();
 	if (isDeprecated(name)) {
-      DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
-      warnOnceIfDeprecated(name);
+      DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
+      warnOnceIfDeprecated(deprecations, name);
       for (String newKey : keyInfo.newKeys) {
         if(newKey != null) {
           names.add(newKey);
@@ -457,7 +610,7 @@ public class Configuration implements It
     	names.add(name);
     }
     for(String n : names) {
-	  String deprecatedKey = reverseDeprecatedKeyMap.get(n);
+	  String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
 	  if (deprecatedKey != null && !getOverlay().containsKey(n) &&
 	      getOverlay().containsKey(deprecatedKey)) {
 	    getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
@@ -469,11 +622,12 @@ public class Configuration implements It
  
   private void handleDeprecation() {
     LOG.debug("Handling deprecation for all properties in config...");
+    DeprecationContext deprecations = deprecationContext.get();
     Set<Object> keys = new HashSet<Object>();
     keys.addAll(getProps().keySet());
     for (Object item: keys) {
       LOG.debug("Handling deprecation for " + (String)item);
-      handleDeprecation((String)item);
+      handleDeprecation(deprecations, (String)item);
     }
   }
  
@@ -492,13 +646,6 @@ public class Configuration implements It
     }
     addDefaultResource("core-default.xml");
     addDefaultResource("core-site.xml");
-    //Add code for managing deprecated key mapping
-    //for example
-    //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"});
-    //adds deprecation for oldKey1 to two new keys(newkey1, newkey2).
-    //so get or set of oldKey1 will correctly populate/access values of 
-    //newkey1 and newkey2
-    addDeprecatedKeys();
   }
   
   private Properties properties;
@@ -721,7 +868,7 @@ public class Configuration implements It
    *         or null if no such property exists.
    */
   public String get(String name) {
-    String[] names = handleDeprecation(name);
+    String[] names = handleDeprecation(deprecationContext.get(), name);
     String result = null;
     for(String n : names) {
       result = substituteVars(getProps().getProperty(n));
@@ -778,7 +925,7 @@ public class Configuration implements It
    *         its replacing property and null if no such property exists.
    */
   public String getRaw(String name) {
-    String[] names = handleDeprecation(name);
+    String[] names = handleDeprecation(deprecationContext.get(), name);
     String result = null;
     for(String n : names) {
       result = getProps().getProperty(n);
@@ -816,7 +963,8 @@ public class Configuration implements It
     Preconditions.checkArgument(
         value != null,
         "Property value must not be null");
-    if (deprecatedKeyMap.isEmpty()) {
+    DeprecationContext deprecations = deprecationContext.get();
+    if (deprecations.getDeprecatedKeyMap().isEmpty()) {
       getProps();
     }
     getOverlay().setProperty(name, value);
@@ -837,12 +985,12 @@ public class Configuration implements It
         }
       }
     }
-    warnOnceIfDeprecated(name);
+    warnOnceIfDeprecated(deprecations, name);
   }
 
-  private void warnOnceIfDeprecated(String name) {
-    DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
-    if (keyInfo != null && !keyInfo.accessed) {
+  private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
+    DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
+    if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
       LOG_DEPRECATION.info(keyInfo.getWarningMessage(name));
     }
   }
@@ -893,7 +1041,7 @@ public class Configuration implements It
    *         doesn't exist.                    
    */
   public String get(String name, String defaultValue) {
-    String[] names = handleDeprecation(name);
+    String[] names = handleDeprecation(deprecationContext.get(), name);
     String result = null;
     for(String n : names) {
       result = substituteVars(getProps().getProperty(n, defaultValue));
@@ -2100,6 +2248,7 @@ public class Configuration implements It
       if (!"configuration".equals(root.getTagName()))
         LOG.fatal("bad conf file: top-level element not <configuration>");
       NodeList props = root.getChildNodes();
+      DeprecationContext deprecations = deprecationContext.get();
       for (int i = 0; i < props.getLength(); i++) {
         Node propNode = props.item(i);
         if (!(propNode instanceof Element))
@@ -2137,9 +2286,10 @@ public class Configuration implements It
         
         // Ignore this parameter if it has already been marked as 'final'
         if (attr != null) {
-          if (deprecatedKeyMap.containsKey(attr)) {
-            DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr);
-            keyInfo.accessed = false;
+          if (deprecations.getDeprecatedKeyMap().containsKey(attr)) {
+            DeprecatedKeyInfo keyInfo =
+                deprecations.getDeprecatedKeyMap().get(attr);
+            keyInfo.clearAccessed();
             for (String key:keyInfo.newKeys) {
               // update new keys with deprecated key's value 
               loadProperty(toAddTo, name, key, value, finalParameter, 
@@ -2433,26 +2583,6 @@ public class Configuration implements It
     return result;
   }
 
-  //Load deprecated keys in common
-  private static void addDeprecatedKeys() {
-    Configuration.addDeprecation("topology.script.file.name", 
-               new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY});
-    Configuration.addDeprecation("topology.script.number.args", 
-               new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY});
-    Configuration.addDeprecation("hadoop.configured.node.mapping", 
-               new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY});
-    Configuration.addDeprecation("topology.node.switch.mapping.impl", 
-               new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY});
-    Configuration.addDeprecation("dfs.df.interval", 
-               new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY});
-    Configuration.addDeprecation("hadoop.native.lib", 
-               new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY});
-    Configuration.addDeprecation("fs.default.name", 
-               new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY});
-    Configuration.addDeprecation("dfs.umaskmode",
-        new String[]{CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY});
-  }
-  
   /**
    * A unique class which is used as a sentinel value in the caching
    * for getClassByName. {@link Configuration#getClassByNameOrNull(String)}
@@ -2460,7 +2590,9 @@ public class Configuration implements It
   private static abstract class NegativeCacheSentinel {}
 
   public static void dumpDeprecatedKeys() {
-    for (Map.Entry<String, DeprecatedKeyInfo> entry : deprecatedKeyMap.entrySet())
{
+    DeprecationContext deprecations = deprecationContext.get();
+    for (Map.Entry<String, DeprecatedKeyInfo> entry :
+        deprecations.getDeprecatedKeyMap().entrySet()) {
       StringBuilder newKeys = new StringBuilder();
       for (String newKey : entry.getValue().newKeys) {
         newKeys.append(newKey).append("\t");

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
Sun Nov  3 17:51:52 2013
@@ -240,6 +240,14 @@ public class CommonConfigurationKeysPubl
   public static final String  HADOOP_SECURITY_GROUPS_CACHE_SECS =
     "hadoop.security.groups.cache.secs";
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final long HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT =
+    300;
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS =
+    "hadoop.security.groups.cache.warn.after.ms";
+  public static final long HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT =
+    5000;
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String  HADOOP_SECURITY_AUTHENTICATION =
     "hadoop.security.authentication";
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Sun Nov  3 17:51:52 2013
@@ -575,6 +575,7 @@ public class Client {
         try {
           this.socket = socketFactory.createSocket();
           this.socket.setTcpNoDelay(tcpNoDelay);
+          this.socket.setKeepAlive(true);
           
           /*
            * Bind the socket to the host specified in the principal name of the

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Sun Nov  3 17:51:52 2013
@@ -762,6 +762,7 @@ public abstract class Server {
 
         channel.configureBlocking(false);
         channel.socket().setTcpNoDelay(tcpNoDelay);
+        channel.socket().setKeepAlive(true);
         
         Reader reader = getReader();
         try {

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
Sun Nov  3 17:51:52 2013
@@ -50,6 +50,7 @@ public class Groups {
   private final Map<String, CachedGroups> userToGroupsMap = 
     new ConcurrentHashMap<String, CachedGroups>();
   private final long cacheTimeout;
+  private final long warningDeltaMs;
 
   public Groups(Configuration conf) {
     impl = 
@@ -60,11 +61,16 @@ public class Groups {
           conf);
     
     cacheTimeout = 
-      conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 5*60) * 1000;
+      conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 
+          CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT) * 1000;
+    warningDeltaMs =
+      conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS,
+        CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT);
     
     if(LOG.isDebugEnabled())
       LOG.debug("Group mapping impl=" + impl.getClass().getName() + 
-          "; cacheTimeout=" + cacheTimeout);
+          "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" +
+          warningDeltaMs);
   }
   
   /**
@@ -76,17 +82,24 @@ public class Groups {
   public List<String> getGroups(String user) throws IOException {
     // Return cached value if available
     CachedGroups groups = userToGroupsMap.get(user);
-    long now = Time.now();
+    long startMs = Time.monotonicNow();
     // if cache has a value and it hasn't expired
-    if (groups != null && (groups.getTimestamp() + cacheTimeout > now)) {
+    if (groups != null && (groups.getTimestamp() + cacheTimeout > startMs)) {
       if(LOG.isDebugEnabled()) {
         LOG.debug("Returning cached groups for '" + user + "'");
       }
       return groups.getGroups();
     }
-    
+
     // Create and cache user's groups
-    groups = new CachedGroups(impl.getGroups(user));
+    List<String> groupList = impl.getGroups(user);
+    long endMs = Time.monotonicNow();
+    long deltaMs = endMs - startMs ;
+    if (deltaMs > warningDeltaMs) {
+      LOG.warn("Potential performance problem: getGroups(user=" + user +") " +
+          "took " + deltaMs + " milliseconds.");
+    }
+    groups = new CachedGroups(groupList, endMs);
     if (groups.getGroups().isEmpty()) {
       throw new IOException("No groups found for user " + user);
     }
@@ -133,9 +146,9 @@ public class Groups {
     /**
      * Create and initialize group cache
      */
-    CachedGroups(List<String> groups) {
+    CachedGroups(List<String> groups, long timestamp) {
       this.groups = groups;
-      this.timestamp = Time.now();
+      this.timestamp = timestamp;
     }
 
     /**

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Sun Nov  3 17:51:52 2013
@@ -106,6 +106,15 @@
 </property>
 
 <property>
+  <name>hadoop.security.groups.cache.warn.after.ms</name>
+  <value>5000</value>
+  <description>
+    If looking up a single user to group takes longer than this amount of
+    milliseconds, we will log a warning message.
+  </description>
+</property>
+
+<property>
   <name>hadoop.security.group.mapping.ldap.url</name>
   <value></value>
   <description>

Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
Sun Nov  3 17:51:52 2013
@@ -26,13 +26,28 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import junit.framework.Assert;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.common.util.concurrent.Uninterruptibles;
+
  
 public class TestConfigurationDeprecation {
   private Configuration conf;
@@ -320,4 +335,68 @@ public class TestConfigurationDeprecatio
     assertNull(conf.get("nK"));
   }
 
+  private static String getTestKeyName(int threadIndex, int testIndex) {
+    return "testConcurrentDeprecateAndManipulate.testKey." +
+                  threadIndex + "." + testIndex;
+  }
+  
+  /**
+   * Run a set of threads making changes to the deprecations
+   * concurrently with another set of threads calling get()
+   * and set() on Configuration objects.
+   */
+  @SuppressWarnings("deprecation")
+  @Test(timeout=60000)
+  public void testConcurrentDeprecateAndManipulate() throws Exception {
+    final int NUM_THREAD_IDS = 10;
+    final int NUM_KEYS_PER_THREAD = 1000;
+    ScheduledThreadPoolExecutor executor =
+      new ScheduledThreadPoolExecutor(2 * NUM_THREAD_IDS,
+      new ThreadFactoryBuilder().setDaemon(true).
+      setNameFormat("testConcurrentDeprecateAndManipulate modification thread %d").
+      build());
+    final CountDownLatch latch = new CountDownLatch(1);
+    final AtomicInteger highestModificationThreadId = new AtomicInteger(1);
+    List<Future<Void>> futures = new LinkedList<Future<Void>>();
+    for (int i = 0; i < NUM_THREAD_IDS; i++) {
+      futures.add(executor.schedule(new Callable<Void>() {
+        @Override
+        public Void call() throws Exception {
+          latch.await();
+          int threadIndex = highestModificationThreadId.addAndGet(1);
+          for (int i = 0; i < NUM_KEYS_PER_THREAD; i++) {
+            String testKey = getTestKeyName(threadIndex, i);
+            String testNewKey = testKey + ".new";
+            Configuration.addDeprecations(
+              new DeprecationDelta[] {
+                new DeprecationDelta(testKey, testNewKey)
+              });
+          }
+          return null;
+        }
+      }, 0, TimeUnit.SECONDS));
+    }
+    final AtomicInteger highestAccessThreadId = new AtomicInteger(1);
+    for (int i = 0; i < NUM_THREAD_IDS; i++) {
+      futures.add(executor.schedule(new Callable<Void>() {
+        @Override
+        public Void call() throws Exception {
+          Configuration conf = new Configuration();
+          latch.await();
+          int threadIndex = highestAccessThreadId.addAndGet(1);
+          for (int i = 0; i < NUM_KEYS_PER_THREAD; i++) {
+            String testNewKey = getTestKeyName(threadIndex, i) + ".new";
+            String value = "value." + threadIndex + "." + i;
+            conf.set(testNewKey, value);
+            Assert.assertEquals(value, conf.get(testNewKey));
+          }
+          return null;
+        }
+      }, 0, TimeUnit.SECONDS));
+    }
+    latch.countDown(); // allow all threads to proceed
+    for (Future<Void> future : futures) {
+      Uninterruptibles.getUninterruptibly(future);
+    }
+  }
 }



Mime
View raw message