Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CDBD96946 for ; Wed, 13 Jul 2011 23:25:03 +0000 (UTC) Received: (qmail 43914 invoked by uid 500); 13 Jul 2011 23:25:03 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 43848 invoked by uid 500); 13 Jul 2011 23:25:02 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 43840 invoked by uid 99); 13 Jul 2011 23:25:02 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 Jul 2011 23:25:02 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 Jul 2011 23:24:58 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0F40423888BD; Wed, 13 Jul 2011 23:24:37 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1146514 - in /hadoop/common/trunk/hdfs: ./ src/java/org/apache/hadoop/hdfs/server/blockmanagement/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/ src/test/hdfs/org/apache/hadoo... Date: Wed, 13 Jul 2011 23:24:36 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110713232437.0F40423888BD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Wed Jul 13 23:24:36 2011 New Revision: 1146514 URL: http://svn.apache.org/viewvc?rev=1146514&view=rev Log: HDFS-2140. Move Host2NodesMap to the blockmanagement package. Added: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java - copied, changed from r1146510, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java - copied, changed from r1146510, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHost2NodesMap.java Removed: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHost2NodesMap.java Modified: hadoop/common/trunk/hdfs/CHANGES.txt hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Modified: hadoop/common/trunk/hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/CHANGES.txt?rev=1146514&r1=1146513&r2=1146514&view=diff ============================================================================== --- hadoop/common/trunk/hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hdfs/CHANGES.txt Wed Jul 13 23:24:36 2011 @@ -552,6 +552,8 @@ Trunk (unreleased changes) HDFS-2131. Add new tests for the -overwrite/-f option in put and copyFromLocal by HADOOP-7361. (Uma Maheswara Rao G via szetszwo) + HDFS-2140. Move Host2NodesMap to the blockmanagement package. (szetszwo) + OPTIMIZATIONS HDFS-1458. Improve checkpoint performance by avoiding unnecessary image Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1146514&r1=1146513&r2=1146514&view=diff ============================================================================== --- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original) +++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Wed Jul 13 23:24:36 2011 @@ -221,6 +221,11 @@ public class BlockManager { datanodeManager.close(); } + /** @return the datanodeManager */ + public DatanodeManager getDatanodeManager() { + return datanodeManager; + } + public void metaSave(PrintWriter out) { // // Dump contents of neededReplication Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1146514&r1=1146513&r2=1146514&view=diff ============================================================================== --- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original) +++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Wed Jul 13 23:24:36 2011 @@ -17,12 +17,15 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.io.IOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.util.Daemon; @@ -35,6 +38,8 @@ public class DatanodeManager { static final Log LOG = LogFactory.getLog(DatanodeManager.class); final FSNamesystem namesystem; + + private final Host2NodesMap host2DatanodeMap = new Host2NodesMap(); DatanodeManager(final FSNamesystem namesystem) { this.namesystem = namesystem; @@ -54,4 +59,39 @@ public class DatanodeManager { void close() { if (decommissionthread != null) decommissionthread.interrupt(); } + + /** @return the datanode descriptor for the host. */ + public DatanodeDescriptor getDatanodeByHost(final String host) { + return host2DatanodeMap.getDatanodeByHost(host); + } + + /** Add a datanode. */ + public void addDatanode(final DatanodeDescriptor node) { + // To keep host2DatanodeMap consistent with datanodeMap, + // remove from host2DatanodeMap the datanodeDescriptor removed + // from datanodeMap before adding node to host2DatanodeMap. + synchronized (namesystem.datanodeMap) { + host2DatanodeMap.remove( + namesystem.datanodeMap.put(node.getStorageID(), node)); + } + host2DatanodeMap.add(node); + + if (LOG.isDebugEnabled()) { + LOG.debug(getClass().getSimpleName() + ".unprotectedAddDatanode: " + + "node " + node.getName() + " is added to datanodeMap."); + } + } + + /** Physically remove node from datanodeMap. */ + public void wipeDatanode(final DatanodeID node) throws IOException { + final String key = node.getStorageID(); + synchronized (namesystem.datanodeMap) { + host2DatanodeMap.remove(namesystem.datanodeMap.remove(key)); + } + if (LOG.isDebugEnabled()) { + LOG.debug(getClass().getSimpleName() + ".wipeDatanode(" + + node.getName() + "): storage " + key + + " is removed from datanodeMap."); + } + } } Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java (from r1146510, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java) URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java&r1=1146510&r2=1146514&rev=1146514&view=diff ============================================================================== --- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java (original) +++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java Wed Jul 13 23:24:36 2011 @@ -15,15 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; +package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.HashMap; import java.util.Random; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +/** A map from host names to datanode descriptors. */ +@InterfaceAudience.Private +@InterfaceStability.Evolving class Host2NodesMap { private HashMap map = new HashMap(); Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1146514&r1=1146513&r2=1146514&view=diff ============================================================================== --- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Jul 13 23:24:36 2011 @@ -317,7 +317,6 @@ public class FSNamesystem implements FSC ReplaceDatanodeOnFailure.DEFAULT; private volatile SafeModeInfo safeMode; // safe mode information - private Host2NodesMap host2DataNodeMap = new Host2NodesMap(); /** datanode network toplogy */ public NetworkTopology clusterMap = new NetworkTopology(); @@ -878,8 +877,8 @@ public class FSNamesystem implements FSC LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true); if (blocks != null) { //sort the blocks - DatanodeDescriptor client = host2DataNodeMap.getDatanodeByHost( - clientMachine); + final DatanodeDescriptor client = + blockManager.getDatanodeManager().getDatanodeByHost(clientMachine); for (LocatedBlock b : blocks.getLocatedBlocks()) { clusterMap.pseudoSortByDistance(client, b.getLocations()); @@ -1490,8 +1489,8 @@ public class FSNamesystem implements FSC } } - DatanodeDescriptor clientNode = - host2DataNodeMap.getDatanodeByHost(clientMachine); + final DatanodeDescriptor clientNode = + blockManager.getDatanodeManager().getDatanodeByHost(clientMachine); if (append && myFile != null) { // @@ -2842,7 +2841,8 @@ public class FSNamesystem implements FSC + " storage " + nodeReg.getStorageID()); DatanodeDescriptor nodeS = datanodeMap.get(nodeReg.getStorageID()); - DatanodeDescriptor nodeN = host2DataNodeMap.getDatanodeByName(nodeReg.getName()); + DatanodeDescriptor nodeN = + blockManager.getDatanodeManager().getDatanodeByHost(nodeReg.getName()); if (nodeN != null && nodeN != nodeS) { NameNode.LOG.info("BLOCK* NameSystem.registerDatanode: " @@ -2851,7 +2851,7 @@ public class FSNamesystem implements FSC // which is not served by anybody anymore. removeDatanode(nodeN); // physically remove node from datanodeMap - wipeDatanode(nodeN); + blockManager.getDatanodeManager().wipeDatanode(nodeN); nodeN = null; } @@ -2918,7 +2918,7 @@ public class FSNamesystem implements FSC DatanodeDescriptor nodeDescr = new DatanodeDescriptor(nodeReg, NetworkTopology.DEFAULT_RACK, hostName); resolveNetworkLocation(nodeDescr); - unprotectedAddDatanode(nodeDescr); + blockManager.getDatanodeManager().addDatanode(nodeDescr); clusterMap.add(nodeDescr); checkDecommissioning(nodeDescr, dnAddress); @@ -3356,44 +3356,6 @@ public class FSNamesystem implements FSC + nodeDescr.getName() + " is out of service now."); } } - - void unprotectedAddDatanode(DatanodeDescriptor nodeDescr) { - assert hasWriteLock(); - // To keep host2DataNodeMap consistent with datanodeMap, - // remove from host2DataNodeMap the datanodeDescriptor removed - // from datanodeMap before adding nodeDescr to host2DataNodeMap. - synchronized (datanodeMap) { - host2DataNodeMap.remove( - datanodeMap.put(nodeDescr.getStorageID(), nodeDescr)); - } - host2DataNodeMap.add(nodeDescr); - - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.unprotectedAddDatanode: " - + "node " + nodeDescr.getName() + " is added to datanodeMap."); - } - } - - /** - * Physically remove node from datanodeMap. - * - * @param nodeID node - * @throws IOException - */ - void wipeDatanode(DatanodeID nodeID) throws IOException { - assert hasWriteLock(); - String key = nodeID.getStorageID(); - synchronized (datanodeMap) { - host2DataNodeMap.remove(datanodeMap.remove(key)); - } - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.wipeDatanode: " - + nodeID.getName() + " storage " + key - + " is removed from datanodeMap."); - } - } FSImage getFSImage() { return dir.fsImage; Copied: hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java (from r1146510, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHost2NodesMap.java) URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java?p2=hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java&p1=hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHost2NodesMap.java&r1=1146510&r2=1146514&rev=1146514&view=diff ============================================================================== --- hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHost2NodesMap.java (original) +++ hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java Wed Jul 13 23:24:36 2011 @@ -16,12 +16,13 @@ * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; +package org.apache.hadoop.hdfs.server.blockmanagement; import junit.framework.TestCase; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.Host2NodesMap; public class TestHost2NodesMap extends TestCase { static private Host2NodesMap map = new Host2NodesMap();