Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: core-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r712962 - in /hadoop/core/trunk: CHANGES.txt
 src/core/org/apache/hadoop/fs/BlockLocation.java
 src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
 src/test/org/apache/hadoop/hdfs/TestReplication.java
Date: Tue, 11 Nov 2008 06:28:21 -0000
To: core-commits@hadoop.apache.org
From: dhruba@apache.org
Message-Id: <20081111062821.89F4D238896F@eris.apache.org>

Author: dhruba
Date: Mon Nov 10 22:28:20 2008
New Revision: 712962

URL: http://svn.apache.org/viewvc?rev=712962&view=rev
Log:
HADOOP-4567. GetFileBlockLocations returns the NetworkTopology
information of the machines on where the blocks reside. (dhruba)


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Nov 10 22:28:20 2008
@@ -22,6 +22,9 @@
       NameNode(bindAddress, conf) is removed.
     (shv)
 
+    HADOOP-4567. GetFileBlockLocations returns the NetworkTopology
+    information of the machines where the blocks reside. (dhruba)
+
   NEW FEATURES
 
     HADOOP-4575. Add a proxy service for relaying HsftpFileSystem requests.

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java Mon Nov 10 22:28:20 2008
@@ -38,6 +38,7 @@
 
   private String[] hosts; //hostnames of datanodes
   private String[] names; //hostname:portNumber of datanodes
+  private String[] topologyPaths; // full path name in network topology
   private long offset;  //offset of the of the block in the file
   private long length;
 
@@ -65,6 +66,20 @@
     }
     this.offset = offset;
     this.length = length;
+    this.topologyPaths = new String[0];
+  }
+
+  /**
+   * Constructor with host, name, network topology, offset and length
+   */
+  public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
+                       long offset, long length) {
+    this(names, hosts, offset, length);
+    if (topologyPaths == null) {
+      this.topologyPaths = new String[0];
+    } else {
+      this.topologyPaths = topologyPaths;
+    }
   }
 
   /**
@@ -88,6 +103,18 @@
       return this.names;
     }
   }
+
+  /**
+   * Get the list of network topology paths for each of the hosts.
+   * The last component of the path is the host.
+   */
+  public String[] getTopologyPaths() throws IOException {
+    if ((topologyPaths == null) || (topologyPaths.length == 0)) {
+      return new String[0];
+    } else {
+      return this.topologyPaths;
+    }
+  }
   
   /**
    * Get the start offset of file associated with this block
@@ -140,6 +167,17 @@
   }
 
   /**
+   * Set the network topology paths of the hosts
+   */
+  public void setTopologyPaths(String[] topologyPaths) throws IOException {
+    if (topologyPaths == null) {
+      this.topologyPaths = new String[0];
+    } else {
+      this.topologyPaths = topologyPaths;
+    }
+  }
+
+  /**
    * Implement write of Writable
    */
   public void write(DataOutput out) throws IOException {
@@ -155,6 +193,11 @@
       Text host = new Text(hosts[i]);
       host.write(out);
     }
+    out.writeInt(topologyPaths.length);
+    for (int i=0; i < topologyPaths.length; i++) {
+      Text host = new Text(topologyPaths[i]);
+      host.write(out);
+    }
   }
   
   /**
@@ -176,6 +219,12 @@
       host.readFields(in);
       hosts[i] = host.toString();
     }
+    int numTops = in.readInt();
+    Text path = new Text();
+    for (int i = 0; i < numTops; i++) {
+      path.readFields(in);
+      topologyPaths[i] = path.toString();
+    }
   }
   
   public String toString() {

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Mon Nov 10 22:28:20 2008
@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.ipc.*;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.conf.*;
 import org.apache.hadoop.hdfs.DistributedFileSystem.DiskStatus;
 import org.apache.hadoop.hdfs.protocol.*;
@@ -309,11 +310,16 @@
       DatanodeInfo[] locations = blk.getLocations();
       String[] hosts = new String[locations.length];
       String[] names = new String[locations.length];
+      String[] racks = new String[locations.length];
       for (int hCnt = 0; hCnt < locations.length; hCnt++) {
         hosts[hCnt] = locations[hCnt].getHostName();
         names[hCnt] = locations[hCnt].getName();
+        NodeBase node = new NodeBase(names[hCnt], 
+                                     locations[hCnt].getNetworkLocation());
+        racks[hCnt] = node.toString();
       }
-      blkLocations[idx] = new BlockLocation(names, hosts, blk.getStartOffset(),
+      blkLocations[idx] = new BlockLocation(names, hosts, racks,
+                                            blk.getStartOffset(),
                                             blk.getBlockSize());
       idx++;
     }

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java Mon Nov 10 22:28:20 2008
@@ -35,6 +35,8 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.BlockLocation;
 
 /**
  * This class tests the replication of a DFS file.
@@ -74,6 +76,28 @@
     
     LocatedBlocks locations = namenode.getBlockLocations(name.toString(),0,
                                                          Long.MAX_VALUE);
+    FileStatus stat = fileSys.getFileStatus(name);
+    BlockLocation[] blockLocations = fileSys.getFileBlockLocations(stat,0L,
+                                                         Long.MAX_VALUE);
+    // verify that rack locations match
+    assertTrue(blockLocations.length == locations.locatedBlockCount());
+    for (int i = 0; i < blockLocations.length; i++) {
+      LocatedBlock blk = locations.get(i);
+      DatanodeInfo[] datanodes = blk.getLocations();
+      String[] topologyPaths = blockLocations[i].getTopologyPaths();
+      assertTrue(topologyPaths.length == datanodes.length);
+      for (int j = 0; j < topologyPaths.length; j++) {
+        boolean found = false;
+        for (int k = 0; k < racks.length; k++) {
+          if (topologyPaths[j].startsWith(racks[k])) {
+            found = true;
+            break;
+          }
+        }
+        assertTrue(found);
+      }
+    }
+
     boolean isOnSameRack = true, isNotOnSameRack = true;
     for (LocatedBlock blk : locations.getLocatedBlocks()) {
       DatanodeInfo[] datanodes = blk.getLocations();