Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4957E9A0C for ; Thu, 17 May 2012 22:31:20 +0000 (UTC) Received: (qmail 35830 invoked by uid 500); 17 May 2012 22:31:20 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 35794 invoked by uid 500); 17 May 2012 22:31:20 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 35786 invoked by uid 99); 17 May 2012 22:31:20 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 17 May 2012 22:31:20 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 17 May 2012 22:31:15 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B65F423889E3; Thu, 17 May 2012 22:30:53 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1339898 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ src/main/java/org/apache/hadoop/hdfs/server/datanode/ src/main/java/org/apache/hadoop/hdfs/util/... Date: Thu, 17 May 2012 22:30:53 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120517223053.B65F423889E3@eris.apache.org> Author: todd Date: Thu May 17 22:30:52 2012 New Revision: 1339898 URL: http://svn.apache.org/viewvc?rev=1339898&view=rev Log: HDFS-3391. Fix InvalidateBlocks to compare blocks including their generation stamps. Contributed by Todd Lipcon. Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightLinkedSet.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu May 17 22:30:52 2012 @@ -73,6 +73,9 @@ Release 2.0.1-alpha - UNRELEASED HDFS-2800. Fix cancellation of checkpoints in the standby node to be more reliable. (todd) + HDFS-3391. Fix InvalidateBlocks to compare blocks including their + generation stamps. (todd) + Release 2.0.0-alpha - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java Thu May 17 22:30:52 2012 @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.bl import java.io.PrintWriter; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -56,10 +55,23 @@ class InvalidateBlocks { return numBlocks; } - /** Does this contain the block which is associated with the storage? */ + /** + * @return true if the given storage has the given block listed for + * invalidation. Blocks are compared including their generation stamps: + * if a block is pending invalidation but with a different generation stamp, + * returns false. + * @param storageID the storage to check + * @param the block to look for + * + */ synchronized boolean contains(final String storageID, final Block block) { - final Collection s = node2blocks.get(storageID); - return s != null && s.contains(block); + final LightWeightHashSet s = node2blocks.get(storageID); + if (s == null) { + return false; // no invalidate blocks for this storage ID + } + Block blockInSet = s.getElement(block); + return blockInSet != null && + block.getGenerationStamp() == blockInSet.getGenerationStamp(); } /** Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java Thu May 17 22:30:52 2012 @@ -592,7 +592,8 @@ class BPOfferService { processDistributedUpgradeCommand((UpgradeCommand)cmd); break; case DatanodeProtocol.DNA_RECOVERBLOCK: - dn.recoverBlocks(((BlockRecoveryCommand)cmd).getRecoveringBlocks()); + String who = "NameNode at " + actor.getNNSocketAddress(); + dn.recoverBlocks(who, ((BlockRecoveryCommand)cmd).getRecoveringBlocks()); break; case DatanodeProtocol.DNA_ACCESSKEYUPDATE: LOG.info("DatanodeCommand action: DNA_ACCESSKEYUPDATE"); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Thu May 17 22:30:52 2012 @@ -165,6 +165,7 @@ import org.apache.hadoop.util.StringUtil import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import com.google.common.annotations.VisibleForTesting; @@ -1710,13 +1711,16 @@ public class DataNode extends Configured secureMain(args, null); } - public Daemon recoverBlocks(final Collection blocks) { + public Daemon recoverBlocks( + final String who, + final Collection blocks) { + Daemon d = new Daemon(threadGroup, new Runnable() { /** Recover a list of blocks. It is run by the primary datanode. */ public void run() { for(RecoveringBlock b : blocks) { try { - logRecoverBlock("NameNode", b.getBlock(), b.getLocations()); + logRecoverBlock(who, b); recoverBlock(b); } catch (IOException e) { LOG.warn("recoverBlocks FAILED: " + b, e); @@ -1977,14 +1981,13 @@ public class DataNode extends Configured datanodes, storages); } - private static void logRecoverBlock(String who, - ExtendedBlock block, DatanodeID[] targets) { - StringBuilder msg = new StringBuilder(targets[0].toString()); - for (int i = 1; i < targets.length; i++) { - msg.append(", " + targets[i]); - } + private static void logRecoverBlock(String who, RecoveringBlock rb) { + ExtendedBlock block = rb.getBlock(); + DatanodeInfo[] targets = rb.getLocations(); + LOG.info(who + " calls recoverBlock(block=" + block - + ", targets=[" + msg + "])"); + + ", targets=[" + Joiner.on(", ").join(targets) + "]" + + ", newGenerationStamp=" + rb.getNewGenerationStamp() + ")"); } @Override // ClientDataNodeProtocol Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java Thu May 17 22:30:52 2012 @@ -174,31 +174,42 @@ public class LightWeightHashSet imple * @return true if element present, false otherwise. */ @SuppressWarnings("unchecked") + @Override public boolean contains(final Object key) { + return getElement((T)key) != null; + } + + /** + * Return the element in this set which is equal to + * the given key, if such an element exists. + * Otherwise returns null. + */ + public T getElement(final T key) { // validate key if (key == null) { throw new IllegalArgumentException("Null element is not supported."); } // find element - final int hashCode = ((T)key).hashCode(); + final int hashCode = key.hashCode(); final int index = getIndex(hashCode); - return containsElem(index, (T) key, hashCode); + return getContainedElem(index, key, hashCode); } /** - * Check if the set contains given element at given index. + * Check if the set contains given element at given index. If it + * does, return that element. * - * @return true if element present, false otherwise. + * @return the element, or null, if no element matches */ - protected boolean containsElem(int index, final T key, int hashCode) { + protected T getContainedElem(int index, final T key, int hashCode) { for (LinkedElement e = entries[index]; e != null; e = e.next) { // element found if (hashCode == e.hashCode && e.element.equals(key)) { - return true; + return e.element; } } // element not found - return false; + return null; } /** @@ -241,7 +252,7 @@ public class LightWeightHashSet imple final int hashCode = element.hashCode(); final int index = getIndex(hashCode); // return false if already present - if (containsElem(index, element, hashCode)) { + if (getContainedElem(index, element, hashCode) != null) { return false; } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightLinkedSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightLinkedSet.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightLinkedSet.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightLinkedSet.java Thu May 17 22:30:52 2012 @@ -88,7 +88,7 @@ public class LightWeightLinkedSet ext final int hashCode = element.hashCode(); final int index = getIndex(hashCode); // return false if already present - if (containsElem(index, element, hashCode)) { + if (getContainedElem(index, element, hashCode) != null) { return false; } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Thu May 17 22:30:52 2012 @@ -425,7 +425,7 @@ public class TestBlockRecovery { DataNode spyDN = spy(dn); doThrow(new RecoveryInProgressException("Replica recovery is in progress")). when(spyDN).initReplicaRecovery(any(RecoveringBlock.class)); - Daemon d = spyDN.recoverBlocks(initRecoveringBlocks()); + Daemon d = spyDN.recoverBlocks("fake NN", initRecoveringBlocks()); d.join(); verify(spyDN, never()).syncBlock( any(RecoveringBlock.class), anyListOf(BlockRecord.class)); @@ -445,7 +445,7 @@ public class TestBlockRecovery { DataNode spyDN = spy(dn); doThrow(new IOException()). when(spyDN).initReplicaRecovery(any(RecoveringBlock.class)); - Daemon d = spyDN.recoverBlocks(initRecoveringBlocks()); + Daemon d = spyDN.recoverBlocks("fake NN", initRecoveringBlocks()); d.join(); verify(spyDN, never()).syncBlock( any(RecoveringBlock.class), anyListOf(BlockRecord.class)); @@ -465,7 +465,7 @@ public class TestBlockRecovery { doReturn(new ReplicaRecoveryInfo(block.getBlockId(), 0, block.getGenerationStamp(), ReplicaState.FINALIZED)).when(spyDN). initReplicaRecovery(any(RecoveringBlock.class)); - Daemon d = spyDN.recoverBlocks(initRecoveringBlocks()); + Daemon d = spyDN.recoverBlocks("fake NN", initRecoveringBlocks()); d.join(); DatanodeProtocol dnP = dn.getActiveNamenodeForBP(POOL_ID); verify(dnP).commitBlockSynchronization( Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java?rev=1339898&r1=1339897&r2=1339898&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java Thu May 17 22:30:52 2012 @@ -421,5 +421,48 @@ public class TestLightWeightHashSet{ LOG.info("Test other - DONE"); } + + @Test + public void testGetElement() { + LightWeightHashSet objSet = new LightWeightHashSet(); + TestObject objA = new TestObject("object A"); + TestObject equalToObjA = new TestObject("object A"); + TestObject objB = new TestObject("object B"); + objSet.add(objA); + objSet.add(objB); + + assertSame(objA, objSet.getElement(objA)); + assertSame(objA, objSet.getElement(equalToObjA)); + assertSame(objB, objSet.getElement(objB)); + assertNull(objSet.getElement(new TestObject("not in set"))); + } + + /** + * Wrapper class which is used in + * {@link TestLightWeightHashSet#testGetElement()} + */ + private static class TestObject { + private final String value; + + public TestObject(String value) { + super(); + this.value = value; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) + return false; + TestObject other = (TestObject) obj; + return this.value.equals(other.value); + } + } } \ No newline at end of file