Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A0CEF1088B for ; Wed, 15 Jan 2014 00:33:18 +0000 (UTC) Received: (qmail 21320 invoked by uid 500); 15 Jan 2014 00:33:13 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 21212 invoked by uid 500); 15 Jan 2014 00:33:12 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 21171 invoked by uid 99); 15 Jan 2014 00:33:10 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 15 Jan 2014 00:33:10 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 15 Jan 2014 00:33:09 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 2A96223889D5; Wed, 15 Jan 2014 00:32:49 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1558246 - in /hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ src/main/java/org/apache/hadoop/hdfs/server... Date: Wed, 15 Jan 2014 00:32:48 -0000 To: hdfs-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140115003249.2A96223889D5@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Wed Jan 15 00:32:47 2014 New Revision: 1558246 URL: http://svn.apache.org/r1558246 Log: YARN-321. Forwarding YARN-321 branch to latest trunk. Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java Propchange: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1557890-1558245 Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1558246&r1=1558245&r2=1558246&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jan 15 00:32:47 2014 @@ -749,6 +749,9 @@ Release 2.4.0 - UNRELEASED HDFS-5710. FSDirectory#getFullPathName should check inodes against null. (Uma Maheswara Rao G via jing9) + HDFS-5579. Under construction files make DataNode decommission take very long + hours. (zhaoyunjiong via jing9) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report @@ -921,6 +924,9 @@ Release 2.3.0 - UNRELEASED HDFS-5675. Add Mkdirs operation to NNThroughputBenchmark. (Plamen Jeliazkov via shv) + HDFS-5677. Need error checking for HA cluster configuration. + (Vincent Sheffer via cos) + OPTIMIZATIONS BUG FIXES @@ -2301,6 +2307,16 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4982. JournalNode should relogin from keytab before fetching logs from other JNs (todd) +Release 2.0.6-alpha - 08/22/2013 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + Release 2.0.5-alpha - 06/06/2013 INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1557890-1558245 Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1558246&r1=1558245&r2=1558246&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Wed Jan 15 00:32:47 2014 @@ -585,6 +585,12 @@ public class DFSUtil { String address = getConfValue(defaultValue, suffix, conf, keys); if (address != null) { InetSocketAddress isa = NetUtils.createSocketAddr(address); + if (isa.isUnresolved()) { + LOG.warn("Namenode for " + nsId + + " remains unresolved for ID " + nnId + + ". Check your hdfs-site.xml file to " + + "ensure namenodes are configured properly."); + } ret.put(nnId, isa); } } Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java?rev=1558246&r1=1558245&r2=1558246&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java Wed Jan 15 00:32:47 2014 @@ -31,7 +31,7 @@ public interface BlockCollection { /** * Get the last block of the collection. */ - public BlockInfo getLastBlock() throws IOException; + public BlockInfo getLastBlock(); /** * Get content summary. Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1558246&r1=1558245&r2=1558246&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Wed Jan 15 00:32:47 2014 @@ -1214,8 +1214,10 @@ public class BlockManager { // block should belong to a file bc = blocksMap.getBlockCollection(block); // abandoned block or block reopened for append - if(bc == null || bc.isUnderConstruction()) { - neededReplications.remove(block, priority); // remove from neededReplications + if (bc == null + || (bc.isUnderConstruction() && block.equals(bc.getLastBlock()))) { + // remove from neededReplications + neededReplications.remove(block, priority); continue; } @@ -1295,7 +1297,7 @@ public class BlockManager { // block should belong to a file bc = blocksMap.getBlockCollection(block); // abandoned block or block reopened for append - if(bc == null || bc.isUnderConstruction()) { + if(bc == null || (bc.isUnderConstruction() && block.equals(bc.getLastBlock()))) { neededReplications.remove(block, priority); // remove from neededReplications rw.targets = null; continue; @@ -2906,8 +2908,16 @@ assert storedBlock.findDatanode(dn) < 0 NumberReplicas num = countNodes(block); int curReplicas = num.liveReplicas(); int curExpectedReplicas = getReplication(block); + if (isNeededReplication(block, curExpectedReplicas, curReplicas)) { if (curExpectedReplicas > curReplicas) { + if (bc.isUnderConstruction()) { + if (block.equals(bc.getLastBlock()) && curReplicas > minReplication) { + continue; + } + underReplicatedInOpenFiles++; + } + // Log info about one block for this node which needs replication if (!status) { status = true; @@ -2924,9 +2934,6 @@ assert storedBlock.findDatanode(dn) < 0 if ((curReplicas == 0) && (num.decommissionedReplicas() > 0)) { decommissionOnlyReplicas++; } - if (bc.isUnderConstruction()) { - underReplicatedInOpenFiles++; - } } if (!neededReplications.contains(block) && pendingReplications.getNumReplicas(block) == 0) { Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1558246&r1=1558245&r2=1558246&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Wed Jan 15 00:32:47 2014 @@ -640,7 +640,7 @@ public class INodeFile extends INodeWith } @Override - public BlockInfo getLastBlock() throws IOException { + public BlockInfo getLastBlock() { return blocks == null || blocks.length == 0? null: blocks[blocks.length-1]; } Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java?rev=1558246&r1=1558245&r2=1558246&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java Wed Jan 15 00:32:47 2014 @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -779,4 +780,53 @@ public class TestDecommission { Thread.sleep(HEARTBEAT_INTERVAL * 1000); } } + + @Test(timeout=120000) + public void testDecommissionWithOpenfile() throws IOException, InterruptedException { + LOG.info("Starting test testDecommissionWithOpenfile"); + + //At most 4 nodes will be decommissioned + startCluster(1, 7, conf); + + FileSystem fileSys = cluster.getFileSystem(0); + FSNamesystem ns = cluster.getNamesystem(0); + + String openFile = "/testDecommissionWithOpenfile.dat"; + + writeFile(fileSys, new Path(openFile), (short)3); + // make sure the file was open for write + FSDataOutputStream fdos = fileSys.append(new Path(openFile)); + + LocatedBlocks lbs = NameNodeAdapter.getBlockLocations(cluster.getNameNode(0), openFile, 0, fileSize); + + DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations(); + DatanodeInfo[] dnInfos4FirstBlock = lbs.get(0).getLocations(); + + ArrayList nodes = new ArrayList(); + ArrayList dnInfos = new ArrayList(); + + for (DatanodeInfo datanodeInfo : dnInfos4FirstBlock) { + DatanodeInfo found = datanodeInfo; + for (DatanodeInfo dif: dnInfos4LastBlock) { + if (datanodeInfo.equals(dif)) { + found = null; + } + } + if (found != null) { + nodes.add(found.getXferAddr()); + dnInfos.add(found); + } + } + //decommission one of the 3 nodes which have last block + nodes.add(dnInfos4LastBlock[0].getXferAddr()); + dnInfos.add(dnInfos4LastBlock[0]); + + writeConfigFile(excludeFile, nodes); + refreshNodes(ns, conf); + for (DatanodeInfo dn : dnInfos) { + waitNodeState(dn, AdminStates.DECOMMISSIONED); + } + + fdos.close(); + } }