hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1207490 - in /hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common: ./ src/main/docs/ src/main/docs/changes/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/http/ src/main/java/org/apache/...
Date Mon, 28 Nov 2011 18:21:54 GMT
Author: atm
Date: Mon Nov 28 18:21:46 2011
New Revision: 1207490

URL: http://svn.apache.org/viewvc?rev=1207490&view=rev
Log:
Merge trunk into HA branch.

Added:
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
      - copied unchanged from r1207485, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestStaticMapping.java
      - copied unchanged from r1207485, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestStaticMapping.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java
      - copied unchanged from r1207485, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java
Modified:
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt   (contents,
props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/  
(props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/  
(props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/  
(props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov
28 18:21:46 2011
@@ -3,7 +3,12 @@ Hadoop Change Log
 Trunk (unreleased changes)
 
   INCOMPATIBLE CHANGES
-   
+
+  NEW FEATURES
+
+    HADOOP-7777 Implement a base class for DNSToSwitchMapping implementations 
+    that can offer extra topology information. (stevel)
+
   IMPROVEMENTS
 
     HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)
@@ -118,6 +123,9 @@ Release 0.23.1 - Unreleased
     HADOOP-7802. Hadoop scripts unconditionally source
     "$bin"/../libexec/hadoop-config.sh. (Bruno Mahé via tomwhite)
 
+    HADOOP-7858. Drop some info logging to DEBUG level in IPC,
+    metrics, and HTTP. (todd via eli)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -131,6 +139,8 @@ Release 0.23.1 - Unreleased
    HADOOP-6614. RunJar should provide more diags when it can't create
    a temp file. (Jonathan Hsieh via eli)
 
+   HADOOP-7859. TestViewFsHdfs.testgetFileLinkStatus is failing an assert. (eli)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES
@@ -1096,6 +1106,12 @@ Release 0.22.0 - Unreleased
 
     HADOOP-7786. Remove HDFS-specific config keys defined in FsConfig. (eli)
 
+    HADOOP-7358. Improve log levels when exceptions caught in RPC handler
+    (Todd Lipcon via shv)
+
+    HADOOP-7861. changes2html.pl generates links to HADOOP, HDFS, and MAPREDUCE
+    jiras. (shv)
+
   OPTIMIZATIONS
 
     HADOOP-6884. Add LOG.isDebugEnabled() guard for each LOG.debug(..).

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1207485
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1207485
 /hadoop/core/branches/branch-0.19/src/docs:713112

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
Mon Nov 28 18:21:46 2011
@@ -242,7 +242,11 @@ for my $rel (@releases) {
 
       $item =~ s:\s*(\([^)"]+?\))\s*$:<br />$1:;       # Separate attribution
       $item =~ s:\n{2,}:\n<p/>\n:g;                    # Keep paragraph breaks
-      $item =~ s{(?:${jira_url_prefix})?(HADOOP-\d+)}  # Link to JIRA
+      $item =~ s{(?:${jira_url_prefix})?(HADOOP-\d+)}  # Link to JIRA Common
+                {<a href="${jira_url_prefix}$1">$1</a>}g;
+      $item =~ s{(?:${jira_url_prefix})?(HDFS-\d+)}    # Link to JIRA Hdfs
+                {<a href="${jira_url_prefix}$1">$1</a>}g;
+      $item =~ s{(?:${jira_url_prefix})?(MAPREDUCE-\d+)}  # Link to JIRA MR
                 {<a href="${jira_url_prefix}$1">$1</a>}g;
       print "      <li>$item</li>\n";
     }

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1207485
 /hadoop/core/branches/branch-0.19/core/src/java:713112
 /hadoop/core/trunk/src/core:776175-785643,785929-786278

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Mon Nov 28 18:21:46 2011
@@ -1099,18 +1099,10 @@ public final class FileContext {
    */
   private Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
     Path pathWithLink, Path target) {
-    /* NB: makeQualified uses the target's scheme and authority, if
-     * specified, and the scheme and authority of pathFS, if not. If
-     * the path does have a scheme and authority we assert they match
-     * those of pathFS since resolve updates the file system of a path
-     * that contains links each time a link is encountered.
-     */
+    // NB: makeQualified uses the target's scheme and authority, if
+    // specified, and the scheme and authority of pathFS, if not.
     final String scheme = target.toUri().getScheme();
     final String auth   = target.toUri().getAuthority();
-    if (scheme != null && auth != null) {
-      assert scheme.equals(pathFS.getUri().getScheme());
-      assert auth.equals(pathFS.getUri().getAuthority());
-    }
     return (scheme == null && auth == null)
       ? target.makeQualified(pathFS.getUri(), pathWithLink.getParent())
       : target;

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
Mon Nov 28 18:21:46 2011
@@ -644,12 +644,12 @@ public class HttpServer implements Filte
         while (true) {
           try {
             port = webServer.getConnectors()[0].getLocalPort();
-            LOG.info("Port returned by webServer.getConnectors()[0]." +
+            LOG.debug("Port returned by webServer.getConnectors()[0]." +
             		"getLocalPort() before open() is "+ port + 
             		". Opening the listener on " + oriPort);
             listener.open();
             port = listener.getLocalPort();
-            LOG.info("listener.getLocalPort() returned " + listener.getLocalPort() + 
+            LOG.debug("listener.getLocalPort() returned " + listener.getLocalPort() + 
                   " webServer.getConnectors()[0].getLocalPort() returned " +
                   webServer.getConnectors()[0].getLocalPort());
             //Workaround to handle the problem reported in HADOOP-4744

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Mon Nov 28 18:21:46 2011
@@ -1498,7 +1498,7 @@ public abstract class Server {
 
     @Override
     public void run() {
-      LOG.info(getName() + ": starting");
+      LOG.debug(getName() + ": starting");
       SERVER.set(Server.this);
       ByteArrayOutputStream buf = 
         new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
@@ -1536,7 +1536,16 @@ public abstract class Server {
                   );
             }
           } catch (Throwable e) {
-            LOG.info(getName() + ", call: " + call + ", error: ", e);
+            String logMsg = getName() + ", call " + call + ": error: " + e;
+            if (e instanceof RuntimeException || e instanceof Error) {
+              // These exception types indicate something is probably wrong
+              // on the server side, as opposed to just a normal exceptional
+              // result.
+              LOG.warn(logMsg, e);
+            } else {
+              LOG.info(logMsg, e);
+            }
+
             errorClass = e.getClass().getName();
             error = StringUtils.stringifyException(e);
             // Remove redundant error class name from the beginning of the stack trace
@@ -1571,7 +1580,7 @@ public abstract class Server {
           LOG.info(getName() + " caught an exception", e);
         }
       }
-      LOG.info(getName() + ": exiting");
+      LOG.debug(getName() + ": exiting");
     }
 
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
Mon Nov 28 18:21:46 2011
@@ -389,7 +389,7 @@ public class WritableRpcEngine implement
       }
       protocolImplMap.put(new ProtoNameVer(protocolName, version),
           new ProtoClassProtoImpl(protocolClass, protocolImpl)); 
-      LOG.info("Protocol Name = " + protocolName +  " version=" + version +
+      LOG.debug("Protocol Name = " + protocolName +  " version=" + version +
           " ProtocolImpl=" + protocolImpl.getClass().getName() + 
           " protocolClass=" + protocolClass.getName());
     }

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
Mon Nov 28 18:21:46 2011
@@ -241,7 +241,7 @@ public class MetricsSystemImpl extends M
           injectedTags, period, config.subset(SOURCE_KEY));
     sources.put(name, sa);
     sa.start();
-    LOG.info("Registered source "+ name);
+    LOG.debug("Registered source "+ name);
   }
 
   @Override public synchronized <T extends MetricsSink>
@@ -405,8 +405,8 @@ public class MetricsSystemImpl extends M
   private synchronized void stopSources() {
     for (Entry<String, MetricsSourceAdapter> entry : sources.entrySet()) {
       MetricsSourceAdapter sa = entry.getValue();
-      LOG.info("Stopping metrics source "+ entry.getKey());
-      LOG.debug(sa.source().getClass());
+      LOG.debug("Stopping metrics source "+ entry.getKey() +
+          ": class=" + sa.source().getClass());
       sa.stop();
     }
     sysSource.stop();
@@ -416,8 +416,8 @@ public class MetricsSystemImpl extends M
   private synchronized void stopSinks() {
     for (Entry<String, MetricsSinkAdapter> entry : sinks.entrySet()) {
       MetricsSinkAdapter sa = entry.getValue();
-      LOG.info("Stopping metrics sink "+ entry.getKey());
-      LOG.debug(sa.sink().getClass());
+      LOG.debug("Stopping metrics sink "+ entry.getKey() +
+          ": class=" + sa.sink().getClass());
       sa.stop();
     }
     sinks.clear();

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
Mon Nov 28 18:21:46 2011
@@ -34,9 +34,13 @@ import org.apache.hadoop.classification.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
+public class CachedDNSToSwitchMapping extends AbstractDNSToSwitchMapping {
   private Map<String, String> cache = new ConcurrentHashMap<String, String>();
-  protected DNSToSwitchMapping rawMapping;
+
+  /**
+   * The uncached mapping
+   */
+  protected final DNSToSwitchMapping rawMapping;
 
   /**
    * cache a raw DNS mapping
@@ -118,4 +122,14 @@ public class CachedDNSToSwitchMapping im
     return getCachedHosts(names);
 
   }
+
+  /**
+   * Delegate the switch topology query to the raw mapping, via
+   * {@link AbstractDNSToSwitchMapping#isMappingSingleSwitch(DNSToSwitchMapping)}
+   * @return true iff the raw mapper is considered single-switch.
+   */
+  @Override
+  public boolean isSingleSwitch() {
+    return isMappingSingleSwitch(rawMapping);
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
Mon Nov 28 18:21:46 2011
@@ -40,6 +40,12 @@ public interface DNSToSwitchMapping {
    * Note the hostname/ip-address is not part of the returned path.
    * The network topology of the cluster would determine the number of
    * components in the network path.
+   * <p/>
+   *
+   * If a name cannot be resolved to a rack, the implementation
+   * should return {@link NetworkTopology#DEFAULT_RACK}. This
+   * is what the bundled implementations do, though it is not a formal requirement
+   *
    * @param names the list of hosts to resolve (can be empty)
    * @return list of resolved network paths.
    * If <i>names</i> is empty, the returned list is also empty

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
Mon Nov 28 18:21:46 2011
@@ -32,16 +32,21 @@ import org.apache.hadoop.fs.CommonConfig
 
 /**
  * This class implements the {@link DNSToSwitchMapping} interface using a 
- * script configured via the {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY}
+ * script configured via the
+ * {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
+ * <p/>
+ * It contains a static class <code>RawScriptBasedMapping</code> that performs
+ * the work: reading the configuration parameters, executing any defined
+ * script, handling errors and such like. The outer
+ * class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ * queries.
+ * <p/>
+ * This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ * true if and only if a script is defined.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public final class ScriptBasedMapping extends CachedDNSToSwitchMapping 
-implements Configurable
-{
-  public ScriptBasedMapping() {
-    super(new RawScriptBasedMapping());
-  }
+public final class ScriptBasedMapping extends CachedDNSToSwitchMapping {
 
   /**
    * Minimum number of arguments: {@value}
@@ -66,6 +71,18 @@ implements Configurable
                      CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY ;
 
   /**
+   * Create an instance with the default configuration.
+   * </p>
+   * Calling {@link #setConf(Configuration)} will trigger a
+   * re-evaluation of the configuration settings and so be used to
+   * set up the mapping script.
+   *
+   */
+  public ScriptBasedMapping() {
+    super(new RawScriptBasedMapping());
+  }
+
+  /**
    * Create an instance from the given configuration
    * @param conf configuration
    */
@@ -74,14 +91,31 @@ implements Configurable
     setConf(conf);
   }
 
+  /**
+   * Get the cached mapping and convert it to its real type
+   * @return the inner raw script mapping.
+   */
+  private RawScriptBasedMapping getRawMapping() {
+    return (RawScriptBasedMapping)rawMapping;
+  }
+
   @Override
   public Configuration getConf() {
-    return ((RawScriptBasedMapping)rawMapping).getConf();
+    return getRawMapping().getConf();
   }
 
+  /**
+   * {@inheritDoc}
+   * <p/>
+   * This will get called in the superclass constructor, so a check is needed
+   * to ensure that the raw mapping is defined before trying to relaying a null
+   * configuration.
+   * @param conf
+   */
   @Override
   public void setConf(Configuration conf) {
-    ((RawScriptBasedMapping)rawMapping).setConf(conf);
+    super.setConf(conf);
+    getRawMapping().setConf(conf);
   }
 
   /**
@@ -89,29 +123,26 @@ implements Configurable
    * by the superclass {@link CachedDNSToSwitchMapping}
    */
   private static final class RawScriptBasedMapping
-      implements DNSToSwitchMapping {
+      extends AbstractDNSToSwitchMapping {
     private String scriptName;
-    private Configuration conf;
     private int maxArgs; //max hostnames per call of the script
-    private static Log LOG =
+    private static final Log LOG =
         LogFactory.getLog(ScriptBasedMapping.class);
 
     /**
-     * Set the configuration and
-     * @param conf extract the configuration parameters of interest
+     * Set the configuration and extract the configuration parameters of interest
+     * @param conf the new configuration
      */
+    @Override
     public void setConf (Configuration conf) {
-      this.scriptName = conf.get(SCRIPT_FILENAME_KEY);
-      this.maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
-      this.conf = conf;
-    }
-
-    /**
-     * Get the configuration
-     * @return the configuration
-     */
-    public Configuration getConf () {
-      return conf;
+      super.setConf(conf);
+      if (conf != null) {
+        scriptName = conf.get(SCRIPT_FILENAME_KEY);
+        maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
+      } else {
+        scriptName = null;
+        maxArgs = 0;
+      }
     }
 
     /**
@@ -122,42 +153,42 @@ implements Configurable
 
     @Override
     public List<String> resolve(List<String> names) {
-    List <String> m = new ArrayList<String>(names.size());
-    
-    if (names.isEmpty()) {
-      return m;
-    }
+      List<String> m = new ArrayList<String>(names.size());
 
-    if (scriptName == null) {
-      for (int i = 0; i < names.size(); i++) {
-        m.add(NetworkTopology.DEFAULT_RACK);
+      if (names.isEmpty()) {
+        return m;
       }
-      return m;
-    }
-    
-    String output = runResolveCommand(names);
-    if (output != null) {
-      StringTokenizer allSwitchInfo = new StringTokenizer(output);
-      while (allSwitchInfo.hasMoreTokens()) {
-        String switchInfo = allSwitchInfo.nextToken();
-        m.add(switchInfo);
-      }
-      
-      if (m.size() != names.size()) {
-        // invalid number of entries returned by the script
-        LOG.error("Script " + scriptName + " returned "
-            + Integer.toString(m.size()) + " values when "
-            + Integer.toString(names.size()) + " were expected.");
+
+      if (scriptName == null) {
+        for (String name : names) {
+          m.add(NetworkTopology.DEFAULT_RACK);
+        }
+        return m;
+      }
+
+      String output = runResolveCommand(names);
+      if (output != null) {
+        StringTokenizer allSwitchInfo = new StringTokenizer(output);
+        while (allSwitchInfo.hasMoreTokens()) {
+          String switchInfo = allSwitchInfo.nextToken();
+          m.add(switchInfo);
+        }
+
+        if (m.size() != names.size()) {
+          // invalid number of entries returned by the script
+          LOG.error("Script " + scriptName + " returned "
+              + Integer.toString(m.size()) + " values when "
+              + Integer.toString(names.size()) + " were expected.");
+          return null;
+        }
+      } else {
+        // an error occurred. return null to signify this.
+        // (exn was already logged in runResolveCommand)
         return null;
       }
-    } else {
-      // an error occurred. return null to signify this.
-      // (exn was already logged in runResolveCommand)
-      return null;
+
+      return m;
     }
-    
-    return m;
-  }
 
     /**
      * Build and execute the resolution command. The command is
@@ -195,10 +226,10 @@ implements Configurable
           dir = new File(userDir);
         }
         ShellCommandExecutor s = new ShellCommandExecutor(
-            cmdList.toArray(new String[0]), dir);
+            cmdList.toArray(new String[cmdList.size()]), dir);
         try {
           s.execute();
-          allOutput.append(s.getOutput() + " ");
+          allOutput.append(s.getOutput()).append(" ");
         } catch (Exception e) {
           LOG.warn("Exception: ", e);
           return null;
@@ -207,5 +238,15 @@ implements Configurable
       }
       return allOutput.toString();
     }
+
+    /**
+     * Declare that the mapper is single-switched if a script was not named
+     * in the configuration.
+     * @return true iff there is no script
+     */
+    @Override
+    public boolean isSingleSwitch() {
+      return scriptName == null;
+    }
   }
 }

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1207485
 /hadoop/core/branches/branch-0.19/core/src/test/core:713112
 /hadoop/core/trunk/src/test/core:776175-785643,785929-786278

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
Mon Nov 28 18:21:46 2011
@@ -17,34 +17,80 @@
  */
 package org.apache.hadoop.net;
 
-import java.util.*;
-
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Implements the {@link DNSToSwitchMapping} via static mappings. Used
- * in testcases that simulate racks.
+ * in testcases that simulate racks, and in the
+ * {@link org.apache.hadoop.hdfs.MiniDFSCluster}
+ *
+ * A shared, static mapping is used; to reset it call {@link #resetMap()}.
  *
+ * When an instance of the class has its {@link #setConf(Configuration)}
+ * method called, nodes listed in the configuration will be added to the map.
+ * These do not get removed when the instance is garbage collected.
  */
-public class StaticMapping extends Configured implements DNSToSwitchMapping {
-  public void setconf(Configuration conf) {
-    String[] mappings = conf.getStrings("hadoop.configured.node.mapping");
-    if (mappings != null) {
-      for (int i = 0; i < mappings.length; i++) {
-        String str = mappings[i];
-        String host = str.substring(0, str.indexOf('='));
-        String rack = str.substring(str.indexOf('=') + 1);
-        addNodeToRack(host, rack);
+public class StaticMapping extends AbstractDNSToSwitchMapping  {
+
+  /**
+   * key to define the node mapping as a comma-delimited list of host=rack
+   * mappings, e.g. <code>host1=r1,host2=r1,host3=r2</code>.
+   * </p>
+   * <b>Important: </b>spaces not trimmed and are considered significant.
+   */
+  public static final String KEY_HADOOP_CONFIGURED_NODE_MAPPING =
+      "hadoop.configured.node.mapping";
+
+  /**
+   * Configure the mapping by extracting any mappings defined in the
+   * {@link #KEY_HADOOP_CONFIGURED_NODE_MAPPING} field
+   * @param conf new configuration
+   */
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    if (conf != null) {
+      String[] mappings = conf.getStrings(KEY_HADOOP_CONFIGURED_NODE_MAPPING);
+      if (mappings != null) {
+        for (String str : mappings) {
+          String host = str.substring(0, str.indexOf('='));
+          String rack = str.substring(str.indexOf('=') + 1);
+          addNodeToRack(host, rack);
+        }
       }
     }
   }
+
+  /**
+   * retained lower case setter for compatibility reasons; relays to
+   * {@link #setConf(Configuration)}
+   * @param conf new configuration
+   */
+  public void setconf(Configuration conf) {
+    setConf(conf);
+  }
+
   /* Only one instance per JVM */
-  private static Map<String, String> nameToRackMap = new HashMap<String, String>();
-  
-  static synchronized public void addNodeToRack(String name, String rackId) {
-    nameToRackMap.put(name, rackId);
+  private static final Map<String, String> nameToRackMap = new HashMap<String, String>();
+
+  /**
+   * Add a node to the static map. The moment any entry is added to the map,
+   * the map goes multi-rack.
+   * @param name node name
+   * @param rackId rack ID
+   */
+  public static void addNodeToRack(String name, String rackId) {
+    synchronized (nameToRackMap) {
+      nameToRackMap.put(name, rackId);
+    }
   }
+
+  @Override
   public List<String> resolve(List<String> names) {
     List<String> m = new ArrayList<String>();
     synchronized (nameToRackMap) {
@@ -59,4 +105,24 @@ public class StaticMapping extends Confi
       return m;
     }
   }
+
+  /**
+   * This mapping is only single switch if the map is empty
+   * @return the current switching status
+   */
+  @Override
+  public boolean isSingleSwitch() {
+    synchronized (nameToRackMap) {
+      return nameToRackMap.isEmpty();
+    }
+  }
+
+  /**
+   * Clear the map and revert to being a single switch
+   */
+  public static void resetMap() {
+    synchronized (nameToRackMap) {
+      nameToRackMap.clear();
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java
Mon Nov 28 18:21:46 2011
@@ -23,30 +23,59 @@ import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 
 import junit.framework.TestCase;
+import org.junit.Test;
 
 public class TestScriptBasedMapping extends TestCase {
 
-  private ScriptBasedMapping mapping;
-  private Configuration conf;
-  private List<String> names;
+
   
   public TestScriptBasedMapping() {
-    mapping = new ScriptBasedMapping();
-
-    conf = new Configuration();
-    conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
-        ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
-    conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
 
-    mapping.setConf(conf);    
   }
 
+  @Test
   public void testNoArgsMeansNoResult() {
-    names = new ArrayList<String>();
+    Configuration conf = new Configuration();
+    conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
+                ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
+    conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+    conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+    ScriptBasedMapping mapping = createMapping(conf);
+    List<String> names = new ArrayList<String>();
     names.add("some.machine.name");
     names.add("other.machine.name");
     List<String> result = mapping.resolve(names);
-    assertNull(result);
+    assertNull("Expected an empty list", result);
   }
 
+  @Test
+  public void testNoFilenameMeansSingleSwitch() throws Throwable {
+    Configuration conf = new Configuration();
+    ScriptBasedMapping mapping = createMapping(conf);
+    assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+    assertTrue("Expected to be single switch",
+               AbstractDNSToSwitchMapping.isMappingSingleSwitch(mapping));
+  }
+
+  @Test
+  public void testFilenameMeansMultiSwitch() throws Throwable {
+    Configuration conf = new Configuration();
+    conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+    ScriptBasedMapping mapping = createMapping(conf);
+    assertFalse("Expected to be multi switch", mapping.isSingleSwitch());
+    mapping.setConf(new Configuration());
+    assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+  }
+
+  @Test
+  public void testNullConfig() throws Throwable {
+    ScriptBasedMapping mapping = createMapping(null);
+    assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+
+  }
+  private ScriptBasedMapping createMapping(Configuration conf) {
+    ScriptBasedMapping mapping = new ScriptBasedMapping();
+    mapping.setConf(conf);
+    return mapping;
+  }
 }



Mime
View raw message