Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 233A418E44 for ; Sat, 9 May 2015 06:05:11 +0000 (UTC) Received: (qmail 44233 invoked by uid 500); 9 May 2015 06:05:10 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 44165 invoked by uid 500); 9 May 2015 06:05:10 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 44156 invoked by uid 99); 9 May 2015 06:05:10 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 09 May 2015 06:05:10 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B0C90E0979; Sat, 9 May 2015 06:05:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wheat9@apache.org To: common-commits@hadoop.apache.org Date: Sat, 09 May 2015 06:05:10 -0000 Message-Id: <0f99e6320bf54c279ead5363abe6526b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hadoop git commit: HDFS-6757. Simplify lease manager with INodeID. Contributed by Haohui Mai. Repository: hadoop Updated Branches: refs/heads/trunk 6471d18bc -> 00fe1ed3a http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java index 184cd87..255a607 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java @@ -213,11 +213,12 @@ public class TestDiskspaceQuotaUpdate { // ignore } + LeaseManager lm = cluster.getNamesystem().getLeaseManager(); // check that the file exists, isn't UC, and has no dangling lease INodeFile inode = fsdir.getINode(file.toString()).asFile(); Assert.assertNotNull(inode); Assert.assertFalse("should not be UC", inode.isUnderConstruction()); - Assert.assertNull("should not have a lease", cluster.getNamesystem().getLeaseManager().getLeaseByPath(file.toString())); + Assert.assertNull("should not have a lease", lm.getLease(inode)); // make sure the quota usage is unchanged final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getStorageSpace(); @@ -256,11 +257,11 @@ public class TestDiskspaceQuotaUpdate { } // check that the file exists, isn't UC, and has no dangling lease + LeaseManager lm = cluster.getNamesystem().getLeaseManager(); INodeFile inode = fsdir.getINode(file.toString()).asFile(); Assert.assertNotNull(inode); Assert.assertFalse("should not be UC", inode.isUnderConstruction()); - Assert.assertNull("should not have a lease", cluster.getNamesystem() - .getLeaseManager().getLeaseByPath(file.toString())); + Assert.assertNull("should not have a lease", lm.getLease(inode)); // make sure the quota usage is unchanged final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getStorageSpace(); @@ -296,11 +297,11 @@ public class TestDiskspaceQuotaUpdate { } // check that the file exists, isn't UC, and has no dangling lease + LeaseManager lm = cluster.getNamesystem().getLeaseManager(); INodeFile inode = fsdir.getINode(file.toString()).asFile(); Assert.assertNotNull(inode); Assert.assertFalse("should not be UC", inode.isUnderConstruction()); - Assert.assertNull("should not have a lease", cluster.getNamesystem() - .getLeaseManager().getLeaseByPath(file.toString())); + Assert.assertNull("should not have a lease", lm.getLease(inode)); // make sure the quota usage is unchanged final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getStorageSpace(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 7b9ea93..5653df5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -109,7 +109,7 @@ public class TestFSImage { assertEquals(1, blks.length); assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState()); // check lease manager - Lease lease = fsn.leaseManager.getLeaseByPath(file2.toString()); + Lease lease = fsn.leaseManager.getLease(file2Node); Assert.assertNotNull(lease); } finally { if (cluster != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index 3d0259e..8b0662c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -76,7 +76,7 @@ public class TestFSNamesystem { DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); LeaseManager leaseMan = fsn.getLeaseManager(); - leaseMan.addLease("client1", "importantFile"); + leaseMan.addLease("client1", fsn.getFSDirectory().allocateNewInodeId()); assertEquals(1, leaseMan.countLease()); fsn.clear(); leaseMan = fsn.getLeaseManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java index a19eb1d..4d0f994 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java @@ -70,7 +70,8 @@ public class TestGetBlockLocations { public Void answer(InvocationOnMock invocation) throws Throwable { INodesInPath iip = fsd.getINodesInPath(FILE_PATH, true); FSDirDeleteOp.delete(fsd, iip, new INode.BlocksMapUpdateInfo(), - new ArrayList(), now()); + new ArrayList(), new ArrayList(), + now()); invocation.callRealMethod(); return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java index 2f114a7..96907f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java @@ -21,39 +21,31 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; +import com.google.common.collect.Lists; import org.junit.Test; import org.mockito.Mockito; +import java.util.ArrayList; + +import static org.mockito.Mockito.*; public class TestLeaseManager { - final Configuration conf = new HdfsConfiguration(); - @Test - public void testRemoveLeaseWithPrefixPath() throws Exception { - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); - - LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem()); - lm.addLease("holder1", "/a/b"); - lm.addLease("holder2", "/a/c"); - assertNotNull(lm.getLeaseByPath("/a/b")); - assertNotNull(lm.getLeaseByPath("/a/c")); - - lm.removeLeaseWithPrefixPath("/a"); - - assertNull(lm.getLeaseByPath("/a/b")); - assertNull(lm.getLeaseByPath("/a/c")); - - lm.addLease("holder1", "/a/b"); - lm.addLease("holder2", "/a/c"); - - lm.removeLeaseWithPrefixPath("/a/"); + public void testRemoveLeases() throws Exception { + FSNamesystem fsn = mock(FSNamesystem.class); + LeaseManager lm = new LeaseManager(fsn); + ArrayList ids = Lists.newArrayList(INodeId.ROOT_INODE_ID + 1, + INodeId.ROOT_INODE_ID + 2, INodeId.ROOT_INODE_ID + 3, + INodeId.ROOT_INODE_ID + 4); + for (long id : ids) { + lm.addLease("foo", id); + } - assertNull(lm.getLeaseByPath("/a/b")); - assertNull(lm.getLeaseByPath("/a/c")); + assertEquals(4, lm.getINodeIdWithLeases().size()); + synchronized (lm) { + lm.removeLeases(ids); + } + assertEquals(0, lm.getINodeIdWithLeases().size()); } /** Check that even if LeaseManager.checkLease is not able to relinquish @@ -70,13 +62,13 @@ public class TestLeaseManager { LeaseManager lm = new LeaseManager(fsn); //Make sure the leases we are going to add exceed the hard limit - lm.setLeasePeriod(0,0); + lm.setLeasePeriod(0, 0); //Add some leases to the LeaseManager - lm.addLease("holder1", "src1"); - lm.addLease("holder2", "src2"); - lm.addLease("holder3", "src3"); - assertEquals(lm.getNumSortedLeases(), 3); + lm.addLease("holder1", INodeId.ROOT_INODE_ID + 1); + lm.addLease("holder2", INodeId.ROOT_INODE_ID + 2); + lm.addLease("holder3", INodeId.ROOT_INODE_ID + 3); + assertEquals(lm.countLease(), 3); //Initiate a call to checkLease. This should exit within the test timeout lm.checkLeases(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java index f43edfb..5be1cef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java @@ -615,7 +615,8 @@ public class TestSaveNamespace { cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); try { - cluster.getNamesystem().leaseManager.addLease("me", "/non-existent"); + cluster.getNamesystem().leaseManager.addLease("me", + INodeId.ROOT_INODE_ID + 1); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); cluster.getNameNodeRpc().saveNamespace(0, 0); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00fe1ed3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java index aba3bb3..e716d6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java @@ -40,11 +40,13 @@ import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; import org.apache.log4j.Level; import org.junit.After; @@ -279,4 +281,31 @@ public class TestINodeFileUnderConstructionWithSnapshot { assertEquals(BLOCKSIZE - 1, lastBlock.getBlockSize()); out.close(); } + + @Test + public void testLease() throws Exception { + try { + NameNodeAdapter.setLeasePeriod(fsn, 100, 200); + final Path foo = new Path(dir, "foo"); + final Path bar = new Path(foo, "bar"); + DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0); + HdfsDataOutputStream out = appendFileWithoutClosing(bar, 100); + out.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH)); + SnapshotTestHelper.createSnapshot(hdfs, dir, "s0"); + + hdfs.delete(foo, true); + Thread.sleep(1000); + try { + fsn.writeLock(); + NameNodeAdapter.getLeaseManager(fsn).runLeaseChecks(); + } finally { + fsn.writeUnlock(); + } + } finally { + NameNodeAdapter.setLeasePeriod( + fsn, + HdfsServerConstants.LEASE_SOFTLIMIT_PERIOD, + HdfsServerConstants.LEASE_HARDLIMIT_PERIOD); + } + } } \ No newline at end of file