hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r755222 - in /hadoop/core/branches/branch-0.20: ./ CHANGES.txt src/core/org/apache/hadoop/net/NetworkTopology.java src/mapred/org/apache/hadoop/mapred/FileInputFormat.java src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
Date Tue, 17 Mar 2009 12:37:40 GMT
Author: ddas
Date: Tue Mar 17 12:37:39 2009
New Revision: 755222

URL: http://svn.apache.org/viewvc?rev=755222&view=rev
Log:
HADOOP-5103. Committing the patch to 0.20 branch.

Modified:
    hadoop/core/branches/branch-0.20/   (props changed)
    hadoop/core/branches/branch-0.20/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
    hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java

Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 17 12:37:39 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Tue Mar 17 12:37:39 2009
@@ -735,6 +735,10 @@
     making sure that lease renewal thread exits before dfs client exits.
     (hairong)
 
+    HADOOP-5103. FileInputFormat now reuses the clusterMap network topology object
+    and that brings down the log messages in the JobClient to do with
+    NetworkTopology.add significantly. (Jothi Padmanabhan via ddas)
+
 Release 0.19.2 - Unreleased
 
   BUG FIXES

Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 17 12:37:39 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035

Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java Tue
Mar 17 12:37:39 2009
@@ -316,7 +316,6 @@
       throw new IllegalArgumentException(
         "Not allow to add an inner node: "+NodeBase.getPath(node));
     }
-    LOG.info("Adding a new node: "+NodeBase.getPath(node));
     netlock.writeLock().lock();
     try {
       Node rack = getNode(node.getNetworkLocation());
@@ -326,6 +325,7 @@
                                            + " at an illegal network location");
       }
       if (clusterMap.add(node)) {
+        LOG.info("Adding a new node: "+NodeBase.getPath(node));
         if (rack == null) {
           numOfRacks++;
         }

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
Tue Mar 17 12:37:39 2009
@@ -214,6 +214,7 @@
 
     // generate splits
     ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
+    NetworkTopology clusterMap = new NetworkTopology();
     for (FileStatus file: files) {
       Path path = file.getPath();
       FileSystem fs = path.getFileSystem(job);
@@ -226,7 +227,7 @@
         long bytesRemaining = length;
         while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
           String[] splitHosts = getSplitHosts(blkLocations, 
-              length-bytesRemaining, splitSize);
+              length-bytesRemaining, splitSize, clusterMap);
           splits.add(new FileSplit(path, length-bytesRemaining, splitSize, 
               splitHosts));
           bytesRemaining -= splitSize;
@@ -237,7 +238,7 @@
                      blkLocations[blkLocations.length-1].getHosts()));
         }
       } else if (length != 0) {
-        String[] splitHosts = getSplitHosts(blkLocations,0,length);
+        String[] splitHosts = getSplitHosts(blkLocations,0,length,clusterMap);
         splits.add(new FileSplit(path, 0, length, splitHosts));
       } else { 
         //Create empty hosts array for zero length files
@@ -417,7 +418,8 @@
    * @throws IOException
    */
   protected String[] getSplitHosts(BlockLocation[] blkLocations, 
-      long offset, long splitSize)   throws IOException {
+      long offset, long splitSize, NetworkTopology clusterMap)
+  throws IOException {
 
     int startIndex = getBlockIndex(blkLocations, offset);
 
@@ -442,7 +444,6 @@
     long bytesInLastBlock = bytesInThisBlock;
     int endIndex = index - 1;
     
-    NetworkTopology clusterMap = new NetworkTopology();
     Map <Node,NodeInfo> hostsMap = new IdentityHashMap<Node,NodeInfo>();
     Map <Node,NodeInfo> racksMap = new IdentityHashMap<Node,NodeInfo>();
     String [] allTopos = new String[0];
@@ -486,6 +487,11 @@
         if (node == null) {
           node = new NodeBase(topo);
           clusterMap.add(node);
+        }
+        
+        nodeInfo = hostsMap.get(node);
+        
+        if (nodeInfo == null) {
           nodeInfo = new NodeInfo(node);
           hostsMap.put(node,nodeInfo);
           parentNode = node.getParent();

Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
(original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
Tue Mar 17 12:37:39 2009
@@ -18,6 +18,8 @@
 package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.net.NetworkTopology;
+
 import junit.framework.TestCase;
 
 public class TestGetSplitHosts extends TestCase {
@@ -28,6 +30,7 @@
     int block1Size = 100, block2Size = 150, block3Size = 75;
     int fileSize = block1Size + block2Size + block3Size;
     int replicationFactor = 3;
+    NetworkTopology clusterMap = new NetworkTopology();
     
     BlockLocation[] bs = new BlockLocation[numBlocks];
     
@@ -72,7 +75,7 @@
     
     SequenceFileInputFormat< String, String> sif = 
       new SequenceFileInputFormat<String,String>();
-    String [] hosts = sif.getSplitHosts(bs, 0, fileSize);
+    String [] hosts = sif.getSplitHosts(bs, 0, fileSize, clusterMap);
 
     // Contributions By Racks are
     // Rack1   175       
@@ -93,7 +96,7 @@
     bs[2] = new BlockLocation(block3Names,block3Hosts,block1Size+block2Size,
                                block3Size);
 
-    hosts = sif.getSplitHosts(bs, 0, fileSize);
+    hosts = sif.getSplitHosts(bs, 0, fileSize, clusterMap);
     
     // host1 makes the highest contribution among all hosts
     // So, that should be returned before others



Mime
View raw message