hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1408972 - in /hadoop/common/branches/branch-1-win: ./ src/core/ src/core/org/apache/hadoop/net/ src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/org/apache/hadoop/net/
Date Tue, 13 Nov 2012 21:36:57 GMT
Author: szetszwo
Date: Tue Nov 13 21:36:55 2012
New Revision: 1408972

URL: http://svn.apache.org/viewvc?rev=1408972&view=rev
Log:
svn merge -c 1408968 from branch-1 for HADOOP-8820. Backport HADOOP-8469 and HADOOP-8470:
Make NetworkTopology class pluggable and add NetworkTopologyWithNodeGroup, a 4-layer implementation
of NetworkTopology.

Added:
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
      - copied unchanged from r1408968, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
      - copied unchanged from r1408968, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Modified:
    hadoop/common/branches/branch-1-win/   (props changed)
    hadoop/common/branches/branch-1-win/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/branch-1-win/src/core/core-default.xml
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/net/NetworkTopology.java
    hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
    hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

Propchange: hadoop/common/branches/branch-1-win/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1:r1408968

Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1408972&r1=1408971&r2=1408972&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Tue Nov 13 21:36:55 2012
@@ -8,6 +8,10 @@ Release 1.2.0 - unreleased
     module external to HDFS to specify how HDFS blocks should be placed.
     (Sumadhur Reddy Bolli via szetszwo)
  
+    HADOOP-8820. Backport HADOOP-8469 and HADOOP-8470: Make NetworkTopology
+    class pluggable and add NetworkTopologyWithNodeGroup, a 4-layer
+    implementation of NetworkTopology.  (Junping Du and Jing Zhao via szetszwo)
+
     HADOOP-7868. Hadoop native fails to compile when default linker
     option is -Wl,--as-needed. (Trevor Robinson via eli)
 

Propchange: hadoop/common/branches/branch-1-win/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1/CHANGES.txt:r1408968

Modified: hadoop/common/branches/branch-1-win/src/core/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/core-default.xml?rev=1408972&r1=1408971&r2=1408972&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/core-default.xml (original)
+++ hadoop/common/branches/branch-1-win/src/core/core-default.xml Tue Nov 13 21:36:55 2012
@@ -446,7 +446,8 @@
   </description>
 </property>
 
-<!-- Rack Configuration -->
+
+<!-- Topology Configuration -->
 
 <property>
   <name>topology.node.switch.mapping.impl</name>
@@ -458,6 +459,13 @@
   </description>
 </property>
 
+ <property>
+  <name>net.topology.impl</name>
+  <value>org.apache.hadoop.net.NetworkTopology</value>
+  <description> The default implementation of NetworkTopology which is classic three
layer one.
+  </description>
+</property>
+
 <property>
   <name>topology.script.file.name</name>
   <value></value>

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/net/NetworkTopology.java?rev=1408972&r1=1408971&r2=1408972&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/net/NetworkTopology.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/net/NetworkTopology.java
Tue Nov 13 21:36:55 2012
@@ -47,8 +47,8 @@ public class NetworkTopology {
   /* Inner Node represent a switch/router of a data center or rack.
    * Different from a leave node, it has non-null children.
    */
-  private class InnerNode extends NodeBase {
-    private ArrayList<Node> children=new ArrayList<Node>();
+  static class InnerNode extends NodeBase {
+    protected List<Node> children=new ArrayList<Node>();
     private int numOfLeaves;
         
     /** Construct an InnerNode from a path-like string */
@@ -68,7 +68,7 @@ public class NetworkTopology {
     }
         
     /** Get its children */
-    Collection<Node> getChildren() {return children;}
+    List<Node> getChildren() {return children;}
         
     /** Return the number of children this node has */
     int getNumOfChildren() {
@@ -161,8 +161,7 @@ public class NetworkTopology {
         }
         if (parentNode == null) {
           // create a new InnerNode
-          parentNode = new InnerNode(parentName, getPath(this),
-                                     this, this.getLevel()+1);
+          parentNode = createParentNode(parentName);
           children.add(parentNode);
         }
         // add n to the subtree of the next ancestor node
@@ -174,7 +173,23 @@ public class NetworkTopology {
         }
       }
     }
-        
+
+    /**
+     * Creates a parent node to be added to the list of children.  
+     * Creates a node using the InnerNode four argument constructor specifying 
+     * the name, location, parent, and level of this node.
+     * 
+     * <p>To be overridden in subclasses for specific InnerNode implementations,
+     * as alternative to overriding the full {@link #add(Node)} method.
+     * 
+     * @param parentName The name of the parent node
+     * @return A new inner node
+     * @see InnerNode#InnerNode(String, String, InnerNode, int)
+     */
+    protected InnerNode createParentNode(String parentName) {
+      return new InnerNode(parentName, getPath(this), this, this.getLevel()+1);
+    }
+
     /** Remove node <i>n</i> from the subtree of this node
      * @param n node to be deleted 
      * @return true if the node is deleted; false otherwise
@@ -246,7 +261,7 @@ public class NetworkTopology {
         
     /** get <i>leafIndex</i> leaf of this subtree 
      * if it is not in the <i>excludedNode</i>*/
-    private Node getLeaf(int leafIndex, Node excludedNode) {
+    protected Node getLeaf(int leafIndex, Node excludedNode) {
       int count=0;
       // check if the excluded node a leaf
       boolean isLeaf =
@@ -254,7 +269,7 @@ public class NetworkTopology {
       // calculate the total number of excluded leaf nodes
       int numOfExcludedLeaves =
         isLeaf ? 1 : ((InnerNode)excludedNode).getNumOfLeaves();
-      if (isRack()) { // children are leaves
+      if (isLeafParent()) { // children are leaves
         if (isLeaf) { // excluded node is a leaf node
           int excludedIndex = children.indexOf(excludedNode);
           if (excludedIndex != -1 && leafIndex >= 0) {
@@ -291,20 +306,35 @@ public class NetworkTopology {
         return null;
       }
     }
-        
+
+    protected boolean isLeafParent() {
+      return isRack();
+    }
+
+    /**
+     * Determine if children a leaves, default implementation calls {@link #isRack()}
+     * <p>To be overridden in subclasses for specific InnerNode implementations,
+     * as alternative to overriding the full {@link #getLeaf(int, Node)} method.
+     * 
+     * @return true if children are leaves, false otherwise
+     */
+    protected boolean areChildrenLeaves() {
+      return isRack();
+    }
+
     int getNumOfLeaves() {
       return numOfLeaves;
     }
   } // end of InnerNode
     
-  InnerNode clusterMap = new InnerNode(InnerNode.ROOT); // the root
-  private int numOfRacks = 0;  // rack counter
-  private ReadWriteLock netlock;
+  InnerNode clusterMap;
+  protected int numOfRacks = 0;  // rack counter
+  protected ReadWriteLock netlock = new ReentrantReadWriteLock();
     
   public NetworkTopology() {
-    netlock = new ReentrantReadWriteLock();
+    clusterMap = new InnerNode(InnerNode.ROOT);
   }
-    
+
   /** Add a leaf node
    * Update node counter & rack counter if neccessary
    * @param node
@@ -320,7 +350,7 @@ public class NetworkTopology {
     }
     netlock.writeLock().lock();
     try {
-      Node rack = getNode(node.getNetworkLocation());
+      Node rack = getNodeForNetworkLocation(node);
       if (rack != null && !(rack instanceof InnerNode)) {
         throw new IllegalArgumentException("Unexpected data node " 
                                            + node.toString() 
@@ -337,7 +367,26 @@ public class NetworkTopology {
       netlock.writeLock().unlock();
     }
   }
-    
+
+  /**
+   * Return a reference to the node given its string representation.
+   * Default implementation delegates to {@link #getNode(String)}.
+   * 
+   * <p>To be overridden in subclasses for specific NetworkTopology 
+   * implementations, as alternative to overriding the full {@link #add(Node)}
+   *  method.
+   * 
+   * @param node The string representation of this node's network location is
+   * used to retrieve a Node object. 
+   * @return a reference to the node; null if the node is not in the tree
+   * 
+   * @see #add(Node)
+   * @see #getNode(String)
+   */
+  protected Node getNodeForNetworkLocation(Node node) {
+    return getNode(node.getNetworkLocation());
+  }
+  
   /** Remove a node
    * Update node counter & rack counter if neccessary
    * @param node
@@ -403,7 +452,21 @@ public class NetworkTopology {
       netlock.readLock().unlock();
     }
   }
-    
+  
+  /** Given a string representation of a rack for a specific network
+   *  location
+   * 
+   * To be overridden in subclasses for specific NetworkTopology 
+   * implementations, as alternative to overriding the full 
+   * {@link #getRack(String)} method.
+   * @param loc
+   *          a path-like string representation of a network location
+   * @return a rack string
+   */
+  public String getRack(String loc) {
+    return loc;
+  }
+
   /** Return the total number of racks */
   public int getNumOfRacks() {
     netlock.readLock().lock();
@@ -516,13 +579,44 @@ public class NetworkTopology {
       
     netlock.readLock().lock();
     try {
-      return node1.getParent()==node2.getParent();
+      return isSameParents(node1, node2);
     } finally {
       netlock.readLock().unlock();
     }
   }
+  
+  /**
+   * Check if network topology is aware of NodeGroup
+   */
+  public boolean isNodeGroupAware() {
+    return false;
+  }
+  
+  /** 
+   * Return false directly as not aware of NodeGroup, to be override in sub-class
+   */
+  public boolean isOnSameNodeGroup(Node node1, Node node2) {
+    return false;
+  }
+
+  /**
+   * Compare the parents of each node for equality
+   * 
+   * <p>To be overridden in subclasses for specific NetworkTopology 
+   * implementations, as alternative to overriding the full 
+   * {@link #isOnSameRack(Node, Node)} method.
+   * 
+   * @param node1 the first node to compare
+   * @param node2 the second node to compare
+   * @return true if their parents are equal, false otherwise
+   * 
+   * @see #isOnSameRack(Node, Node)
+   */
+  protected boolean isSameParents(Node node1, Node node2) {
+    return node1.getParent()==node2.getParent();
+  }
     
-  final private static Random r = new Random();
+  final protected static Random r = new Random();
   /** randomly choose one node from <i>scope</i>
    * if scope starts with ~, choose one from the all nodes except for the
    * ones in <i>scope</i>; otherwise, choose one from <i>scope</i>
@@ -570,7 +664,25 @@ public class NetworkTopology {
     int leaveIndex = r.nextInt(numOfDatanodes);
     return innerNode.getLeaf(leaveIndex, node);
   }
-       
+
+  /** return leaves in <i>scope</i>
+   * @param scope a path string
+   * @return leaves nodes under specific scope
+   */
+  public List<Node> getLeaves(String scope) {
+    Node node = getNode(scope);
+    List<Node> leafNodes = new ArrayList<Node>();
+    if (!(node instanceof InnerNode)) {
+      leafNodes.add(node);
+    } else {
+      InnerNode innerNode = (InnerNode) node;
+      for (int i = 0; i < innerNode.getNumOfLeaves(); i++) {
+        leafNodes.add(innerNode.getLeaf(i, null));
+      }
+    }
+    return leafNodes;
+  }
+  
   /** return the number of leaves in <i>scope</i> but not in <i>excludedNodes</i>
    * if scope starts with ~, return the number of nodes that are not
    * in <i>scope</i> and <i>excludedNodes</i>; 
@@ -632,7 +744,7 @@ public class NetworkTopology {
   }
 
   /* swap two array items */
-  static private void swap(Node[] nodes, int i, int j) {
+  static protected void swap(Node[] nodes, int i, int j) {
     Node tempNode;
     tempNode = nodes[j];
     nodes[j] = nodes[i];

Modified: hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1408972&r1=1408971&r2=1408972&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
(original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
Tue Nov 13 21:36:55 2012
@@ -166,10 +166,10 @@ public class DatanodeInfo extends Datano
     this.xceiverCount = xceiverCount; 
   }
 
-  /** rack name **/
+  /** network location **/
   public synchronized String getNetworkLocation() {return location;}
     
-  /** Sets the rack name */
+  /** Sets the network location */
   public synchronized void setNetworkLocation(String location) {
     this.location = NodeBase.normalize(location);
   }

Modified: hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1408972&r1=1408971&r2=1408972&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Nov 13 21:36:55 2012
@@ -332,7 +332,7 @@ public class FSNamesystem implements FSC
   private Host2NodesMap host2DataNodeMap = new Host2NodesMap();
     
   // datanode networktoplogy
-  NetworkTopology clusterMap = new NetworkTopology();
+  NetworkTopology clusterMap;
   private DNSToSwitchMapping dnsToSwitchMapping;
   
   // for block replicas placement
@@ -484,6 +484,10 @@ public class FSNamesystem implements FSC
     this.defaultPermission = PermissionStatus.createImmutable(
         fsOwner.getShortUserName(), supergroup, new FsPermission(filePermission));
     
+    this.clusterMap = (NetworkTopology) ReflectionUtils.newInstance(
+        conf.getClass("net.topology.impl", NetworkTopology.class,
+            NetworkTopology.class), conf);
+
     this.replicator = BlockPlacementPolicy.getInstance(conf, this, clusterMap);
     
     this.defaultReplication = conf.getInt("dfs.replication", 3);
@@ -896,8 +900,20 @@ public class FSNamesystem implements FSC
     LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true);
     if (blocks != null) {
       //sort the blocks
-      DatanodeDescriptor client = host2DataNodeMap.getDatanodeByHost(
+      // In some deployment cases, cluster is with separation of task tracker 
+      // and datanode which means client machines will not always be recognized 
+      // as known data nodes, so here we should try to get node (but not 
+      // datanode only) for locality based sort.
+      Node client = host2DataNodeMap.getDatanodeByHost(
           clientMachine);
+      if (client == null) {
+        List<String> hosts = new ArrayList<String> (1);
+        hosts.add(clientMachine);
+        String rName = dnsToSwitchMapping.resolve(hosts).get(0);
+        if (rName != null)
+          client = new NodeBase(clientMachine, rName);
+      }   
+
       for (LocatedBlock b : blocks.getLocatedBlocks()) {
         clusterMap.pseudoSortByDistance(client, b.getLocations());
       }



Mime
View raw message