From hdfs-commits-return-1978-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Aug 16 00:38:34 2011 Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A55CA7BAA for ; Tue, 16 Aug 2011 00:38:34 +0000 (UTC) Received: (qmail 4043 invoked by uid 500); 16 Aug 2011 00:38:34 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 4009 invoked by uid 500); 16 Aug 2011 00:38:34 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 3998 invoked by uid 99); 16 Aug 2011 00:38:34 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Aug 2011 00:38:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Aug 2011 00:38:16 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 127352388A6C; Tue, 16 Aug 2011 00:37:31 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1158072 [6/7] - in /hadoop/common/branches/HDFS-1623/hdfs: ./ ivy/ src/c++/libhdfs/ src/contrib/ src/contrib/fuse-dfs/ src/java/ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/ser... Date: Tue, 16 Aug 2011 00:37:25 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110816003731.127352388A6C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestHftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestHftpFileSystem.java Tue Aug 16 00:37:15 2011 @@ -19,14 +19,15 @@ package org.apache.hadoop.hdfs; import java.io.IOException; +import java.net.URISyntaxException; import java.net.URL; import java.net.HttpURLConnection; import java.util.Random; -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; +import org.junit.Test; +import org.junit.BeforeClass; +import org.junit.AfterClass; +import static org.junit.Assert.*; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; @@ -39,26 +40,48 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.util.ServletUtil; import org.apache.log4j.Level; -/** - * Unittest for HftpFileSystem. - * - */ -public class TestHftpFileSystem extends TestCase { +public class TestHftpFileSystem { private static final Random RAN = new Random(); - private static final Path TEST_FILE = new Path("/testfile+1"); private static Configuration config = null; private static MiniDFSCluster cluster = null; private static FileSystem hdfs = null; private static HftpFileSystem hftpFs = null; private static String blockPoolId = null; - - /** - * Setup hadoop mini-cluster for test. - */ - private static void oneTimeSetUp() throws IOException { + + private static Path[] TEST_PATHS = new Path[] { + // URI does not encode, Request#getPathInfo returns /foo + new Path("/foo;bar"), + + // URI does not encode, Request#getPathInfo returns verbatim + new Path("/foo+"), + new Path("/foo+bar/foo+bar"), + new Path("/foo=bar/foo=bar"), + new Path("/foo,bar/foo,bar"), + new Path("/foo@bar/foo@bar"), + new Path("/foo&bar/foo&bar"), + new Path("/foo$bar/foo$bar"), + new Path("/foo_bar/foo_bar"), + new Path("/foo~bar/foo~bar"), + new Path("/foo.bar/foo.bar"), + new Path("/foo../bar/foo../bar"), + new Path("/foo.../bar/foo.../bar"), + new Path("/foo'bar/foo'bar"), + new Path("/foo#bar/foo#bar"), + new Path("/foo!bar/foo!bar"), + // HDFS file names may not contain ":" + + // URI percent encodes, Request#getPathInfo decodes + new Path("/foo bar/foo bar"), + new Path("/foo?bar/foo?bar"), + new Path("/foo\">bar/foo\">bar"), + }; + + @BeforeClass + public static void setUp() throws IOException { ((Log4JLogger)HftpFileSystem.LOG).getLogger().setLevel(Level.ALL); final long seed = RAN.nextLong(); @@ -67,66 +90,73 @@ public class TestHftpFileSystem extends config = new Configuration(); config.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); - cluster = new MiniDFSCluster.Builder(config).numDataNodes(2).build(); hdfs = cluster.getFileSystem(); blockPoolId = cluster.getNamesystem().getBlockPoolId(); - final String hftpuri = + final String hftpUri = "hftp://" + config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); - hftpFs = (HftpFileSystem) new Path(hftpuri).getFileSystem(config); + hftpFs = (HftpFileSystem) new Path(hftpUri).getFileSystem(config); } - /** - * Shutdown the hadoop mini-cluster. - */ - private static void oneTimeTearDown() throws IOException { + @AfterClass + public static void tearDown() throws IOException { hdfs.close(); hftpFs.close(); cluster.shutdown(); } - - public TestHftpFileSystem(String name) { - super(name); - } /** - * For one time setup / teardown. + * Test file creation and access with file names that need encoding. */ - public static Test suite() { - TestSuite suite = new TestSuite(); - - suite.addTestSuite(TestHftpFileSystem.class); - - return new TestSetup(suite) { - @Override - protected void setUp() throws IOException { - oneTimeSetUp(); - } - - @Override - protected void tearDown() throws IOException { - oneTimeTearDown(); - } - }; + @Test + public void testFileNameEncoding() throws IOException, URISyntaxException { + for (Path p : TEST_PATHS) { + // Create and access the path (data and streamFile servlets) + FSDataOutputStream out = hdfs.create(p, true); + out.writeBytes("0123456789"); + out.close(); + FSDataInputStream in = hftpFs.open(p); + assertEquals('0', in.read()); + + // Check the file status matches the path. Hftp returns a FileStatus + // with the entire URI, extract the path part. + assertEquals(p, new Path(hftpFs.getFileStatus(p).getPath().toUri().getPath())); + + // Test list status (listPath servlet) + assertEquals(1, hftpFs.listStatus(p).length); + + // Test content summary (contentSummary servlet) + assertNotNull("No content summary", hftpFs.getContentSummary(p)); + + // Test checksums (fileChecksum and getFileChecksum servlets) + assertNotNull("No file checksum", hftpFs.getFileChecksum(p)); + } } - - public void testDataNodeRedirect() throws Exception { - if (hdfs.exists(TEST_FILE)) { - hdfs.delete(TEST_FILE, true); + + private void testDataNodeRedirect(Path path) throws IOException { + // Create the file + if (hdfs.exists(path)) { + hdfs.delete(path, true); } - FSDataOutputStream out = hdfs.create(TEST_FILE, (short) 1); + FSDataOutputStream out = hdfs.create(path, (short)1); out.writeBytes("0123456789"); out.close(); - + + // Get the path's block location so we can determine + // if we were redirected to the right DN. BlockLocation[] locations = - hdfs.getFileBlockLocations(TEST_FILE, 0, 10); - + hdfs.getFileBlockLocations(path, 0, 10); String locationName = locations[0].getNames()[0]; - URL u = hftpFs.getNamenodeFileURL(TEST_FILE); + + // Connect to the NN to get redirected + URL u = hftpFs.getNamenodeURL( + "/data" + ServletUtil.encodePath(path.toUri().getPath()), + "ugi=userx,groupy"); HttpURLConnection conn = (HttpURLConnection)u.openConnection(); HttpURLConnection.setFollowRedirects(true); conn.connect(); conn.getInputStream(); + boolean checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port @@ -138,19 +168,32 @@ public class TestHftpFileSystem extends assertEquals(dnR.getInfoPort(), conn.getURL().getPort()); } } - assertTrue("The test never checked that location of " + - "the block and hftp desitnation are the same", checked); + assertTrue("The test never checked that location of " + + "the block and hftp desitnation are the same", checked); + } + + /** + * Test that clients are redirected to the appropriate DN. + */ + @Test + public void testDataNodeRedirect() throws IOException { + for (Path p : TEST_PATHS) { + testDataNodeRedirect(p); + } } + /** * Tests getPos() functionality. */ - public void testGetPos() throws Exception { + @Test + public void testGetPos() throws IOException { + final Path testFile = new Path("/testfile+1"); // Write a test file. - FSDataOutputStream out = hdfs.create(TEST_FILE, true); + FSDataOutputStream out = hdfs.create(testFile, true); out.writeBytes("0123456789"); out.close(); - FSDataInputStream in = hftpFs.open(TEST_FILE); + FSDataInputStream in = hftpFs.open(testFile); // Test read(). for (int i = 0; i < 5; ++i) { @@ -175,17 +218,17 @@ public class TestHftpFileSystem extends assertEquals(10, in.getPos()); in.close(); } - + /** * Tests seek(). */ - public void testSeek() throws Exception { - // Write a test file. - FSDataOutputStream out = hdfs.create(TEST_FILE, true); + @Test + public void testSeek() throws IOException { + final Path testFile = new Path("/testfile+1"); + FSDataOutputStream out = hdfs.create(testFile, true); out.writeBytes("0123456789"); out.close(); - - FSDataInputStream in = hftpFs.open(TEST_FILE); + FSDataInputStream in = hftpFs.open(testFile); in.seek(7); assertEquals('7', in.read()); } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLease.java Tue Aug 16 00:37:15 2011 @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.UserGroupInformation; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,8 @@ import org.mockito.Mockito; public class TestLease { static boolean hasLease(MiniDFSCluster cluster, Path src) { - return cluster.getNamesystem().leaseManager.getLeaseByPath(src.toString()) != null; + return NameNodeAdapter.getLeaseManager(cluster.getNamesystem() + ).getLeaseByPath(src.toString()) != null; } final Path dir = new Path("/test/lease/"); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery.java Tue Aug 16 00:37:15 2011 @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.L import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.TestInterDatanodeProtocol; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; public class TestLeaseRecovery extends junit.framework.TestCase { static final int BLOCK_SIZE = 1024; @@ -133,7 +134,7 @@ public class TestLeaseRecovery extends j DFSTestUtil.waitReplication(dfs, filepath, (short)1); waitLeaseRecovery(cluster); // verify that we still cannot recover the lease - LeaseManager lm = cluster.getNamesystem().leaseManager; + LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem()); assertTrue("Found " + lm.countLease() + " lease, expected 1", lm.countLease() == 1); cluster.getNameNode().setSafeMode(FSConstants.SafeModeAction.SAFEMODE_LEAVE); } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java Tue Aug 16 00:37:15 2011 @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.ChecksumExce import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; /** * The test makes sure that NameNode detects presense blocks that do not have @@ -56,6 +57,7 @@ public class TestMissingBlocksAlert exte cluster = new MiniDFSCluster.Builder(conf).build(); cluster.waitActive(); + final BlockManager bm = cluster.getNamesystem().getBlockManager(); DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); @@ -86,8 +88,7 @@ public class TestMissingBlocksAlert exte } assertTrue(dfs.getMissingBlocksCount() == 1); assertEquals(4, dfs.getUnderReplicatedBlocksCount()); - assertEquals(3, - cluster.getNamesystem().getUnderReplicatedNotMissingBlocks()); + assertEquals(3, bm.getUnderReplicatedNotMissingBlocks()); // Now verify that it shows up on webui @@ -109,8 +110,7 @@ public class TestMissingBlocksAlert exte } assertEquals(2, dfs.getUnderReplicatedBlocksCount()); - assertEquals(2, - cluster.getNamesystem().getUnderReplicatedNotMissingBlocks()); + assertEquals(2, bm.getUnderReplicatedNotMissingBlocks()); // and make sure WARNING disappears // Now verify that it shows up on webui Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationToken.java Tue Aug 16 00:37:15 2011 @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -48,7 +49,8 @@ import org.junit.Test; public class TestDelegationToken { private MiniDFSCluster cluster; - Configuration config; + private DelegationTokenSecretManager dtSecretManager; + private Configuration config; private static final Log LOG = LogFactory.getLog(TestDelegationToken.class); @Before @@ -61,7 +63,9 @@ public class TestDelegationToken { FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0"); cluster = new MiniDFSCluster.Builder(config).build(); cluster.waitActive(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + dtSecretManager = NameNodeAdapter.getDtSecretManager( + cluster.getNamesystem()); + dtSecretManager.startThreads(); } @After @@ -73,8 +77,6 @@ public class TestDelegationToken { private Token generateDelegationToken( String owner, String renewer) { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem() - .getDelegationTokenSecretManager(); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text( owner), new Text(renewer), null); return new Token(dtId, dtSecretManager); @@ -82,8 +84,6 @@ public class TestDelegationToken { @Test public void testDelegationTokenSecretManager() throws Exception { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem() - .getDelegationTokenSecretManager(); Token token = generateDelegationToken( "SomeUser", "JobTracker"); // Fake renewer should not be able to renew @@ -122,8 +122,6 @@ public class TestDelegationToken { @Test public void testCancelDelegationToken() throws Exception { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem() - .getDelegationTokenSecretManager(); Token token = generateDelegationToken( "SomeUser", "JobTracker"); //Fake renewer should not be able to renew @@ -144,7 +142,6 @@ public class TestDelegationToken { @Test public void testDelegationTokenDFSApi() throws Exception { - DelegationTokenSecretManager dtSecretManager = cluster.getNamesystem().getDelegationTokenSecretManager(); DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); Token token = dfs.getDelegationToken("JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java Tue Aug 16 00:37:15 2011 @@ -31,19 +31,20 @@ import java.util.Enumeration; import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.Text; -import org.apache.commons.logging.*; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.TestDoAsEffectiveUser; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -98,7 +99,7 @@ public class TestDelegationTokenForProxy FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0"); cluster = new MiniDFSCluster.Builder(config).build(); cluster.waitActive(); - cluster.getNamesystem().getDelegationTokenSecretManager().startThreads(); + NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads(); ProxyUsers.refreshSuperUserGroupsConfiguration(config); } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Tue Aug 16 00:37:15 2011 @@ -25,10 +25,13 @@ import java.util.Set; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.util.Daemon; public class BlockManagerTestUtil { + public static void setNodeReplicationLimit(final BlockManager blockManager, + final int limit) { + blockManager.maxReplicationStreams = limit; + } /** @return the datanode descriptor for the given the given storageID. */ public static DatanodeDescriptor getDatanode(final FSNamesystem ns, Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java Tue Aug 16 00:37:15 2011 @@ -45,8 +45,8 @@ public class TestComputeInvalidateWork e final FSNamesystem namesystem = cluster.getNamesystem(); final BlockManager bm = namesystem.getBlockManager(); final int blockInvalidateLimit = bm.getDatanodeManager().blockInvalidateLimit; - DatanodeDescriptor[] nodes = - namesystem.heartbeats.toArray(new DatanodeDescriptor[NUM_OF_DATANODES]); + final DatanodeDescriptor[] nodes = bm.getDatanodeManager( + ).getHeartbeatManager().getDatanodes(); assertEquals(nodes.length, NUM_OF_DATANODES); namesystem.writeLock(); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java Tue Aug 16 00:37:15 2011 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.io.IOException; import java.util.ArrayList; import junit.framework.TestCase; @@ -52,6 +51,8 @@ public class TestHeartbeatHandling exten try { cluster.waitActive(); final FSNamesystem namesystem = cluster.getNamesystem(); + final HeartbeatManager hm = namesystem.getBlockManager( + ).getDatanodeManager().getHeartbeatManager(); final String poolId = namesystem.getBlockPoolId(); final DatanodeRegistration nodeReg = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); @@ -69,12 +70,12 @@ public class TestHeartbeatHandling exten try { namesystem.writeLock(); - synchronized (namesystem.heartbeats) { + synchronized(hm) { for (int i=0; i WAIT_FOR_DEATH; // Eventually the NN should report two volume failures - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); /* @@ -177,10 +178,10 @@ public class TestDataNodeVolumeFailureRe ArrayList live = new ArrayList(); ArrayList dead = new ArrayList(); - ns.DFSNodesStatus(live, dead); + dm.fetchDatanodes(live, dead, false); live.clear(); dead.clear(); - ns.DFSNodesStatus(live, dead); + dm.fetchDatanodes(live, dead, false); assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures()); @@ -189,8 +190,8 @@ public class TestDataNodeVolumeFailureRe * total capacity should be down by three volumes (assuming the host * did not grow or shrink the data volume while the test was running). */ - dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, + dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 3, origCapacity - (3*dnCapacity), WAIT_FOR_HEARTBEATS); /* @@ -212,7 +213,7 @@ public class TestDataNodeVolumeFailureRe getMetrics(dns.get(2).getMetrics().name())); // The NN considers the DN dead - DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 2, 1, 2, origCapacity - (4*dnCapacity), WAIT_FOR_HEARTBEATS); /* @@ -236,7 +237,7 @@ public class TestDataNodeVolumeFailureRe * and that the volume failure count should be reported as zero by * both the metrics and the NN. */ - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS); } @@ -251,9 +252,10 @@ public class TestDataNodeVolumeFailureRe cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); - FSNamesystem ns = cluster.getNamesystem(); - long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); - long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); // Fail the first volume on both datanodes (we have to keep the // third healthy so one node in the pipeline will not fail). @@ -267,13 +269,13 @@ public class TestDataNodeVolumeFailureRe DFSTestUtil.waitReplication(fs, file1, (short)2); // The NN reports two volumes failures - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); // After restarting the NN it still see the two failures cluster.restartNameNode(0); cluster.waitActive(); - DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, + DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); } } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java Tue Aug 16 00:37:15 2011 @@ -17,29 +17,32 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + import java.io.File; import java.io.IOException; +import java.util.concurrent.TimeoutException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.DFSConfigKeys; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.log4j.Level; - import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; -import static org.junit.Assume.assumeTrue; /** * Test the ability of a DN to tolerate volume failures. @@ -154,9 +157,10 @@ public class TestDataNodeVolumeFailureTo conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 0); cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); - FSNamesystem ns = cluster.getNamesystem(); - long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns); - long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0); // Fail a volume on the 2nd DN File dn2Vol1 = new File(dataDir, "data"+(2*1+1)); @@ -168,7 +172,7 @@ public class TestDataNodeVolumeFailureTo DFSTestUtil.waitReplication(fs, file1, (short)2); // Check that this single failure caused a DN to die. - DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 0, + DFSTestUtil.waitForDatanodeStatus(dm, 2, 1, 0, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); // If we restore the volume we should still only be able to get @@ -187,7 +191,7 @@ public class TestDataNodeVolumeFailureTo */ private void restartDatanodes(int volTolerated, boolean manageDfsDirs) throws IOException { - //Make sure no datanode is running + // Make sure no datanode is running cluster.shutdownDataNodes(); conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, volTolerated); cluster.startDataNodes(conf, 1, manageDfsDirs, null, null); @@ -224,7 +228,7 @@ public class TestDataNodeVolumeFailureTo */ private void testVolumeConfig(int volumesTolerated, int volumesFailed, boolean expectedBPServiceState, boolean manageDfsDirs) - throws IOException, InterruptedException { + throws IOException, InterruptedException, TimeoutException { assumeTrue(!System.getProperty("os.name").startsWith("Windows")); final int dnIndex = 0; // Fail the current directory since invalid storage directory perms @@ -259,4 +263,30 @@ public class TestDataNodeVolumeFailureTo assertEquals("Couldn't chmod local vol", 0, FileUtil.chmod(dir.toString(), "000")); } + + /** + * Test that a volume that is considered failed on startup is seen as + * a failed volume by the NN. + */ + @Test + public void testFailedVolumeOnStartupIsCounted() throws Exception { + assumeTrue(!System.getProperty("os.name").startsWith("Windows")); + final DatanodeManager dm = cluster.getNamesystem().getBlockManager( + ).getDatanodeManager(); + long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm); + File dir = new File(MiniDFSCluster.getStorageDir(0, 0), "current"); + + try { + prepareDirToFail(dir); + restartDatanodes(1, false); + // The cluster is up.. + assertEquals(true, cluster.getDataNodes().get(0) + .isBPServiceAlive(cluster.getNamesystem().getBlockPoolId())); + // but there has been a single volume failure + DFSTestUtil.waitForDatanodeStatus(dm, 1, 0, 1, + origCapacity / 2, WAIT_FOR_HEARTBEATS); + } finally { + FileUtil.chmod(dir.toString(), "755"); + } + } } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java Tue Aug 16 00:37:15 2011 @@ -28,7 +28,6 @@ import java.net.URLEncoder; import javax.servlet.http.HttpServletRequest; import javax.servlet.jsp.JspWriter; -import org.apache.commons.httpclient.util.URIUtil; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; @@ -36,6 +35,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.util.ServletUtil; import org.junit.Test; import org.mockito.Mockito; @@ -71,7 +71,7 @@ public class TestDatanodeJsp { if (!doTail) { assertTrue("page should show link to download file", viewFilePage - .contains("/streamFile" + URIUtil.encodePath(testPath.toString()) + + .contains("/streamFile" + ServletUtil.encodePath(testPath.toString()) + "?nnaddr=localhost:" + nnIpcAddress.getPort())); } } @@ -82,15 +82,22 @@ public class TestDatanodeJsp { try { cluster = new MiniDFSCluster.Builder(CONF).build(); cluster.waitActive(); - - testViewingFile(cluster, "/test-file", false); - testViewingFile(cluster, "/tmp/test-file", false); - testViewingFile(cluster, "/tmp/test-file%with goofy&characters", false); - - testViewingFile(cluster, "/test-file", true); - testViewingFile(cluster, "/tmp/test-file", true); - testViewingFile(cluster, "/tmp/test-file%with goofy&characters", true); - + String paths[] = { + "/test-file", + "/tmp/test-file", + "/tmp/test-file%with goofy&characters", + "/foo bar/foo bar", + "/foo+bar/foo+bar", + "/foo;bar/foo;bar", + "/foo=bar/foo=bar", + "/foo,bar/foo,bar", + "/foo?bar/foo?bar", + "/foo\">bar/foo\">bar" + }; + for (String p : paths) { + testViewingFile(cluster, p, false); + testViewingFile(cluster, p, true); + } } finally { if (cluster != null) { cluster.shutdown(); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Tue Aug 16 00:37:15 2011 @@ -24,25 +24,27 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import org.apache.commons.logging.Log; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.IOUtils; import org.mockito.Mockito; import com.google.common.base.Joiner; -import com.google.common.collect.ComparisonChain; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -110,6 +112,40 @@ public abstract class FSImageTestUtil { return sd; } + /** + * Make a mock storage directory that returns some set of file contents. + * @param type type of storage dir + * @param previousExists should we mock that the previous/ dir exists? + * @param fileNames the names of files contained in current/ + */ + static StorageDirectory mockStorageDirectory( + StorageDirType type, + boolean previousExists, + String... fileNames) { + StorageDirectory sd = mock(StorageDirectory.class); + + doReturn(type).when(sd).getStorageDirType(); + + // Version file should always exist + doReturn(mockFile(true)).when(sd).getVersionFile(); + + // Previous dir optionally exists + doReturn(mockFile(previousExists)) + .when(sd).getPreviousDir(); + + // Return a mock 'current' directory which has the given paths + File[] files = new File[fileNames.length]; + for (int i = 0; i < fileNames.length; i++) { + files[i] = new File(fileNames[i]); + } + + File mockDir = Mockito.spy(new File("/dir/current")); + doReturn(files).when(mockDir).listFiles(); + doReturn(mockDir).when(sd).getCurrentDir(); + + return sd; + } + static File mockFile(boolean exists) { File mockFile = mock(File.class); doReturn(exists).when(mockFile).exists(); @@ -154,9 +190,9 @@ public abstract class FSImageTestUtil { for (File dir : dirs) { FSImageTransactionalStorageInspector inspector = inspectStorageDirectory(dir, NameNodeDirType.IMAGE); - FoundFSImage latestImage = inspector.getLatestImage(); + FSImageFile latestImage = inspector.getLatestImage(); assertNotNull("No image in " + dir, latestImage); - long thisTxId = latestImage.getTxId(); + long thisTxId = latestImage.getCheckpointTxId(); if (imageTxId != -1 && thisTxId != imageTxId) { fail("Storage directory " + dir + " does not have the same " + "last image index " + imageTxId + " as another"); @@ -283,7 +319,7 @@ public abstract class FSImageTestUtil { new FSImageTransactionalStorageInspector(); inspector.inspectDirectory(sd); - FoundFSImage latestImage = inspector.getLatestImage(); + FSImageFile latestImage = inspector.getLatestImage(); return (latestImage == null) ? null : latestImage.getFile(); } @@ -316,23 +352,15 @@ public abstract class FSImageTestUtil { * @return the latest edits log, finalized or otherwise, from the given * storage directory. */ - public static FoundEditLog findLatestEditsLog(StorageDirectory sd) + public static EditLogFile findLatestEditsLog(StorageDirectory sd) throws IOException { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); inspector.inspectDirectory(sd); - List foundEditLogs = Lists.newArrayList( - inspector.getFoundEditLogs()); - return Collections.max(foundEditLogs, new Comparator() { - @Override - public int compare(FoundEditLog a, FoundEditLog b) { - return ComparisonChain.start() - .compare(a.getStartTxId(), b.getStartTxId()) - .compare(a.getLastTxId(), b.getLastTxId()) - .result(); - } - }); + List foundEditLogs = Lists.newArrayList( + inspector.getEditLogFiles()); + return Collections.max(foundEditLogs, EditLogFile.COMPARE_BY_START_TXID); } /** @@ -371,5 +399,16 @@ public abstract class FSImageTestUtil { assertNotNull(image); } - + public static void logStorageContents(Log LOG, NNStorage storage) { + LOG.info("current storages and corresponding sizes:"); + for (StorageDirectory sd : storage.dirIterable(null)) { + File curDir = sd.getCurrentDir(); + LOG.info("In directory " + curDir); + File[] files = curDir.listFiles(); + Arrays.sort(files); + for (File f : files) { + LOG.info(" file " + f.getAbsolutePath() + "; len = " + f.length()); + } + } + } } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Tue Aug 16 00:37:15 2011 @@ -1128,7 +1128,8 @@ public class NNThroughputBenchmark { // decommission data-nodes decommissionNodes(); // set node replication limit - namesystem.setNodeReplicationLimit(nodeReplicationLimit); + BlockManagerTestUtil.setNodeReplicationLimit(namesystem.getBlockManager(), + nodeReplicationLimit); } private void decommissionNodes() throws IOException { @@ -1171,9 +1172,7 @@ public class NNThroughputBenchmark { void printResults() { String blockDistribution = ""; String delim = "("; - int totalReplicas = 0; for(int idx=0; idx < blockReportObject.getNumDatanodes(); idx++) { - totalReplicas += blockReportObject.datanodes[idx].nrBlocks; blockDistribution += delim + blockReportObject.datanodes[idx].nrBlocks; delim = ", "; } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Tue Aug 16 00:37:15 2011 @@ -19,10 +19,12 @@ package org.apache.hadoop.hdfs.server.na import java.io.IOException; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.ipc.Server; /** @@ -52,7 +54,33 @@ public class NameNodeAdapter { public static Server getRpcServer(NameNode namenode) { return namenode.server; } + + public static DelegationTokenSecretManager getDtSecretManager( + final FSNamesystem ns) { + return ns.getDelegationTokenSecretManager(); + } + + public static DatanodeCommand[] sendHeartBeat(DatanodeRegistration nodeReg, + DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException { + return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(), + dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(), 0, 0, 0); + } + + public static boolean setReplication(final FSNamesystem ns, + final String src, final short replication) throws IOException { + return ns.setReplication(src, replication); + } + public static LeaseManager getLeaseManager(final FSNamesystem ns) { + return ns.leaseManager; + } + + /** Set the softLimit and hardLimit of client lease periods. */ + public static void setLeasePeriod(final FSNamesystem namesystem, long soft, long hard) { + getLeaseManager(namesystem).setLeasePeriod(soft, hard); + namesystem.lmthread.interrupt(); + } + public static String getLeaseHolderForPath(NameNode namenode, String path) { return namenode.getNamesystem().leaseManager.getLeaseByPath(path).getHolder(); } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Tue Aug 16 00:37:15 2011 @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; @@ -163,8 +163,8 @@ public class TestBackupNode extends Test // When shutting down the BN, it shouldn't finalize logs that are // still open on the NN - FoundEditLog editsLog = FSImageTestUtil.findLatestEditsLog(sd); - assertEquals(editsLog.getStartTxId(), + EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd); + assertEquals(editsLog.getFirstTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId()); assertTrue("Should not have finalized " + editsLog, editsLog.isInProgress()); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java Tue Aug 16 00:37:15 2011 @@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; @@ -82,7 +82,7 @@ public class TestCheckPointForSecurityTo // verify that the edits file is NOT empty NameNode nn = cluster.getNameNode(); for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) { - FoundEditLog log = FSImageTestUtil.findLatestEditsLog(sd); + EditLogFile log = FSImageTestUtil.findLatestEditsLog(sd); assertTrue(log.isInProgress()); assertEquals("In-progress log " + log + " should have 5 transactions", 5, log.validateLog().numTransactions); @@ -97,7 +97,7 @@ public class TestCheckPointForSecurityTo } // verify that the edits file is empty except for the START txn for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) { - FoundEditLog log = FSImageTestUtil.findLatestEditsLog(sd); + EditLogFile log = FSImageTestUtil.findLatestEditsLog(sd); assertTrue(log.isInProgress()); assertEquals("In-progress log " + log + " should only have START txn", 1, log.validateLog().numTransactions); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Tue Aug 16 00:37:15 2011 @@ -1450,7 +1450,7 @@ public class TestCheckpoint extends Test // Make a finalized log on the server side. nn.rollEditLog(); - RemoteEditLogManifest manifest = nn.getEditLogManifest(0); + RemoteEditLogManifest manifest = nn.getEditLogManifest(1); RemoteEditLog log = manifest.getLogs().get(0); NNStorage dstImage = Mockito.mock(NNStorage.class); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java Tue Aug 16 00:37:15 2011 @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Random; import org.apache.hadoop.conf.Configuration; @@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -146,7 +148,7 @@ public class TestDecommissioningStatus { /* * Decommissions the node at the given index */ - private String decommissionNode(FSNamesystem namesystem, Configuration conf, + private String decommissionNode(FSNamesystem namesystem, DFSClient client, FileSystem localFileSys, int nodeIndex) throws IOException { DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); @@ -158,7 +160,6 @@ public class TestDecommissioningStatus { ArrayList nodes = new ArrayList(decommissionedNodes); nodes.add(nodename); writeConfigFile(localFileSys, excludeFile, nodes); - namesystem.refreshNodes(conf); return nodename; } @@ -199,13 +200,13 @@ public class TestDecommissioningStatus { Thread.sleep(5000); FSNamesystem fsn = cluster.getNamesystem(); + final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager(); for (int iteration = 0; iteration < numDatanodes; iteration++) { - String downnode = decommissionNode(fsn, conf, client, localFileSys, - iteration); + String downnode = decommissionNode(fsn, client, localFileSys, iteration); + dm.refreshNodes(conf); decommissionedNodes.add(downnode); Thread.sleep(5000); - ArrayList decommissioningNodes = fsn - .getDecommissioningNodes(); + final List decommissioningNodes = dm.getDecommissioningNodes(); if (iteration == 0) { assertEquals(decommissioningNodes.size(), 1); DatanodeDescriptor decommNode = decommissioningNodes.get(0); @@ -222,7 +223,7 @@ public class TestDecommissioningStatus { // This will remove the datanodes from decommissioning list and // make them available again. writeConfigFile(localFileSys, excludeFile, null); - fsn.refreshNodes(conf); + dm.refreshNodes(conf); st1.close(); cleanupFile(fileSys, file1); cleanupFile(fileSys, file2); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Tue Aug 16 00:37:15 2011 @@ -22,9 +22,15 @@ import java.io.*; import java.net.URI; import java.util.Collection; import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Arrays; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -49,6 +55,10 @@ import org.apache.log4j.Level; import org.aspectj.util.FileUtil; import org.mockito.Mockito; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import static org.apache.hadoop.test.MetricsAsserts.*; @@ -676,28 +686,44 @@ public class TestEditLog extends TestCas private static class EditLogByteInputStream extends EditLogInputStream { private InputStream input; private long len; + private int version; + private FSEditLogOp.Reader reader = null; + private FSEditLogLoader.PositionTrackingInputStream tracker = null; - public EditLogByteInputStream(byte[] data) { + public EditLogByteInputStream(byte[] data) throws IOException { len = data.length; input = new ByteArrayInputStream(data); - } - public int available() throws IOException { - return input.available(); - } - - public int read() throws IOException { - return input.read(); + BufferedInputStream bin = new BufferedInputStream(input); + DataInputStream in = new DataInputStream(bin); + version = EditLogFileInputStream.readLogVersion(in); + tracker = new FSEditLogLoader.PositionTrackingInputStream(in); + in = new DataInputStream(tracker); + + reader = new FSEditLogOp.Reader(in, version); } + @Override public long length() throws IOException { return len; } - - public int read(byte[] b, int off, int len) throws IOException { - return input.read(b, off, len); + + @Override + public long getPosition() { + return tracker.getPos(); + } + + @Override + public FSEditLogOp readOp() throws IOException { + return reader.readOp(); + } + + @Override + public int getVersion() throws IOException { + return version; } + @Override public void close() throws IOException { input.close(); } @@ -729,4 +755,103 @@ public class TestEditLog extends TestCas log.close(); } } + + /** + * Tests the getEditLogManifest function using mock storage for a number + * of different situations. + */ + @Test + public void testEditLogManifestMocks() throws IOException { + NNStorage storage; + FSEditLog log; + // Simple case - different directories have the same + // set of logs, with an in-progress one at end + storage = mockStorageWithEdits( + "[1,100]|[101,200]|[201,]", + "[1,100]|[101,200]|[201,]"); + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200]]", + log.getEditLogManifest(1).toString()); + assertEquals("[[101,200]]", + log.getEditLogManifest(101).toString()); + + // Another simple case, different directories have different + // sets of files + storage = mockStorageWithEdits( + "[1,100]|[101,200]", + "[1,100]|[201,300]|[301,400]"); // nothing starting at 101 + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200], [201,300], [301,400]]", + log.getEditLogManifest(1).toString()); + + // Case where one directory has an earlier finalized log, followed + // by a gap. The returned manifest should start after the gap. + storage = mockStorageWithEdits( + "[1,100]|[301,400]", // gap from 101 to 300 + "[301,400]|[401,500]"); + log = new FSEditLog(storage); + assertEquals("[[301,400], [401,500]]", + log.getEditLogManifest(1).toString()); + + // Case where different directories have different length logs + // starting at the same txid - should pick the longer one + storage = mockStorageWithEdits( + "[1,100]|[101,150]", // short log at 101 + "[1,50]|[101,200]"); // short log at 1 + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200]]", + log.getEditLogManifest(1).toString()); + assertEquals("[[101,200]]", + log.getEditLogManifest(101).toString()); + + // Case where the first storage has an inprogress while + // the second has finalised that file (i.e. the first failed + // recently) + storage = mockStorageWithEdits( + "[1,100]|[101,]", + "[1,100]|[101,200]"); + log = new FSEditLog(storage); + assertEquals("[[1,100], [101,200]]", + log.getEditLogManifest(1).toString()); + assertEquals("[[101,200]]", + log.getEditLogManifest(101).toString()); + } + + /** + * Create a mock NNStorage object with several directories, each directory + * holding edit logs according to a specification. Each directory + * is specified by a pipe-separated string. For example: + * [1,100]|[101,200] specifies a directory which + * includes two finalized segments, one from 1-100, and one from 101-200. + * The syntax [1,] specifies an in-progress log starting at + * txid 1. + */ + private NNStorage mockStorageWithEdits(String... editsDirSpecs) { + List sds = Lists.newArrayList(); + for (String dirSpec : editsDirSpecs) { + List files = Lists.newArrayList(); + String[] logSpecs = dirSpec.split("\\|"); + for (String logSpec : logSpecs) { + Matcher m = Pattern.compile("\\[(\\d+),(\\d+)?\\]").matcher(logSpec); + assertTrue("bad spec: " + logSpec, m.matches()); + if (m.group(2) == null) { + files.add(NNStorage.getInProgressEditsFileName( + Long.valueOf(m.group(1)))); + } else { + files.add(NNStorage.getFinalizedEditsFileName( + Long.valueOf(m.group(1)), + Long.valueOf(m.group(2)))); + } + } + sds.add(FSImageTestUtil.mockStorageDirectory( + NameNodeDirType.EDITS, false, + files.toArray(new String[0]))); + } + + NNStorage storage = Mockito.mock(NNStorage.class); + Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); + return storage; + } + + } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java Tue Aug 16 00:37:15 2011 @@ -61,7 +61,7 @@ public class TestEditLogFileOutputStream .getStorage().getStorageDir(0); File editLog = NNStorage.getInProgressEditsFile(sd, 1); - EditLogValidation validation = FSEditLogLoader.validateEditLog(editLog); + EditLogValidation validation = EditLogFileInputStream.validateEditLog(editLog); assertEquals("Edit log should contain a header as valid length", HEADER_LEN, validation.validLength); assertEquals(1, validation.numTransactions); @@ -73,7 +73,7 @@ public class TestEditLogFileOutputStream new FsPermission((short)777)); long oldLength = validation.validLength; - validation = FSEditLogLoader.validateEditLog(editLog); + validation = EditLogFileInputStream.validateEditLog(editLog); assertTrue("Edit log should have more valid data after writing a txn " + "(was: " + oldLength + " now: " + validation.validLength + ")", validation.validLength > oldLength); Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Tue Aug 16 00:37:15 2011 @@ -186,7 +186,7 @@ public class TestFSEditLogLoader { // Make sure that uncorrupted log has the expected length and number // of transactions. - EditLogValidation validation = FSEditLogLoader.validateEditLog(logFile); + EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals(NUM_TXNS + 2, validation.numTransactions); assertEquals(validLength, validation.validLength); @@ -202,7 +202,7 @@ public class TestFSEditLogLoader { // Restore backup, truncate the file exactly before the txn Files.copy(logFileBak, logFile); truncateFile(logFile, txOffset); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when truncating to length " + txOffset, txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -211,7 +211,7 @@ public class TestFSEditLogLoader { // also isn't valid Files.copy(logFileBak, logFile); truncateFile(logFile, txOffset + 1); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when truncating to length " + (txOffset + 1), txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -219,7 +219,7 @@ public class TestFSEditLogLoader { // Restore backup, corrupt the txn opcode Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when corrupting txn opcode at " + txOffset, txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -227,7 +227,7 @@ public class TestFSEditLogLoader { // Restore backup, corrupt a byte a few bytes into the txn Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset+5); - validation = FSEditLogLoader.validateEditLog(logFile); + validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when corrupting txn data at " + (txOffset+5), txid - 1, validation.numTransactions); assertEquals(txOffset, validation.validLength); @@ -240,7 +240,7 @@ public class TestFSEditLogLoader { for (long offset = 0; offset < validLength; offset++) { Files.copy(logFileBak, logFile); corruptByteInFile(logFile, offset); - EditLogValidation val = FSEditLogLoader.validateEditLog(logFile); + EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile); assertTrue(val.numTransactions >= prevNumValid); prevNumValid = val.numTransactions; } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Tue Aug 16 00:37:15 2011 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.na import static org.junit.Assert.*; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import java.io.File; @@ -29,15 +28,14 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.TransactionalLoadPlan; import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.LogGroup; import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.LoadPlan; @@ -56,7 +54,7 @@ public class TestFSImageStorageInspector FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockDir = mockDirectory( + StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), @@ -72,7 +70,7 @@ public class TestFSImageStorageInspector assertEquals(2, inspector.foundImages.size()); assertTrue(inspector.foundEditLogs.get(1).isInProgress()); - FoundFSImage latestImage = inspector.getLatestImage(); + FSImageFile latestImage = inspector.getLatestImage(); assertEquals(456, latestImage.txId); assertSame(mockDir, latestImage.sd); assertTrue(inspector.isUpgradeFinalized()); @@ -95,7 +93,7 @@ public class TestFSImageStorageInspector FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockDir = mockDirectory( + StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), @@ -123,7 +121,7 @@ public class TestFSImageStorageInspector FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockDir = mockDirectory( + StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE_AND_EDITS, false, "/foo/current/" + getImageFileName(123), @@ -196,14 +194,14 @@ public class TestFSImageStorageInspector inspector.inspectDirectory( mockDirectoryWithEditLogs("/foo3/current/" + getInProgressEditsFileName(123))); - inspector.inspectDirectory(mockDirectory( + inspector.inspectDirectory(FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE, false, "/foo4/current/" + getImageFileName(122))); LogGroup lg = inspector.logGroups.get(123L); assertEquals(3, lg.logs.size()); - FoundEditLog inProgressLog = lg.logs.get(2); + EditLogFile inProgressLog = lg.logs.get(2); assertTrue(inProgressLog.isInProgress()); LoadPlan plan = inspector.createLoadPlan(); @@ -282,7 +280,7 @@ public class TestFSImageStorageInspector assertTrue(lg.logs.get(2).isCorrupt()); // Calling recover should move it aside - FoundEditLog badLog = lg.logs.get(2); + EditLogFile badLog = lg.logs.get(2); Mockito.doNothing().when(badLog).moveAsideCorruptFile(); Mockito.doNothing().when(lg.logs.get(0)).finalizeLog(); Mockito.doNothing().when(lg.logs.get(1)).finalizeLog(); @@ -303,12 +301,12 @@ public class TestFSImageStorageInspector String path, int numValidTransactions) throws IOException { for (LogGroup lg : inspector.logGroups.values()) { - List logs = lg.logs; + List logs = lg.logs; for (int i = 0; i < logs.size(); i++) { - FoundEditLog log = logs.get(i); - if (log.file.getPath().equals(path)) { + EditLogFile log = logs.get(i); + if (log.getFile().getPath().equals(path)) { // mock out its validation - FoundEditLog spyLog = spy(log); + EditLogFile spyLog = spy(log); doReturn(new FSEditLogLoader.EditLogValidation(-1, numValidTransactions)) .when(spyLog).validateLog(); logs.set(i, spyLog); @@ -327,15 +325,15 @@ public class TestFSImageStorageInspector FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector(); - StorageDirectory mockImageDir = mockDirectory( + StorageDirectory mockImageDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE, false, "/foo/current/" + getImageFileName(123)); - StorageDirectory mockImageDir2 = mockDirectory( + StorageDirectory mockImageDir2 = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.IMAGE, false, "/foo2/current/" + getImageFileName(456)); - StorageDirectory mockEditsDir = mockDirectory( + StorageDirectory mockEditsDir = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.EDITS, false, "/foo3/current/" + getFinalizedEditsFileName(123, 456), @@ -356,7 +354,7 @@ public class TestFSImageStorageInspector // Check plan TransactionalLoadPlan plan = (TransactionalLoadPlan)inspector.createLoadPlan(); - FoundFSImage pickedImage = plan.image; + FSImageFile pickedImage = plan.image; assertEquals(456, pickedImage.txId); assertSame(mockImageDir2, pickedImage.sd); assertEquals(new File("/foo2/current/" + getImageFileName(456)), @@ -364,43 +362,8 @@ public class TestFSImageStorageInspector assertArrayEquals(new File[] { new File("/foo3/current/" + getInProgressEditsFileName(457)) }, plan.getEditsFiles().toArray(new File[0])); - - // Check log manifest - assertEquals("[[123,456]]", inspector.getEditLogManifest(123).toString()); - assertEquals("[[123,456]]", inspector.getEditLogManifest(456).toString()); - assertEquals("[]", inspector.getEditLogManifest(457).toString()); } - @Test - public void testLogManifest() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo1/current/" - + getFinalizedEditsFileName(1,1), - "/foo1/current/" - + getFinalizedEditsFileName(2,200))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo2/current/" - + getInProgressEditsFileName(1), - "/foo2/current/" - + getFinalizedEditsFileName(201, 400))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo3/current/" - + getFinalizedEditsFileName(1, 1), - "/foo3/current/" - + getFinalizedEditsFileName(2,200))); - - assertEquals("[[1,1], [2,200], [201,400]]", - inspector.getEditLogManifest(1).toString()); - assertEquals("[[2,200], [201,400]]", - inspector.getEditLogManifest(2).toString()); - assertEquals("[[2,200], [201,400]]", - inspector.getEditLogManifest(10).toString()); - assertEquals("[[201,400]]", - inspector.getEditLogManifest(201).toString()); - } - /** * Test case where an in-progress log is in an earlier name directory * than a finalized log. Previously, getEditLogManifest wouldn't @@ -426,46 +389,9 @@ public class TestFSImageStorageInspector + getFinalizedEditsFileName(2626,2627), "/foo2/current/" + getFinalizedEditsFileName(2628,2629))); - - assertEquals("[[2622,2623], [2624,2625], [2626,2627], [2628,2629]]", - inspector.getEditLogManifest(2621).toString()); } - private StorageDirectory mockDirectoryWithEditLogs(String... fileNames) { - return mockDirectory(NameNodeDirType.EDITS, false, fileNames); - } - - /** - * Make a mock storage directory that returns some set of file contents. - * @param type type of storage dir - * @param previousExists should we mock that the previous/ dir exists? - * @param fileNames the names of files contained in current/ - */ - static StorageDirectory mockDirectory( - StorageDirType type, - boolean previousExists, - String... fileNames) { - StorageDirectory sd = mock(StorageDirectory.class); - - doReturn(type).when(sd).getStorageDirType(); - - // Version file should always exist - doReturn(FSImageTestUtil.mockFile(true)).when(sd).getVersionFile(); - - // Previous dir optionally exists - doReturn(FSImageTestUtil.mockFile(previousExists)) - .when(sd).getPreviousDir(); - - // Return a mock 'current' directory which has the given paths - File[] files = new File[fileNames.length]; - for (int i = 0; i < fileNames.length; i++) { - files[i] = new File(fileNames[i]); - } - - File mockDir = Mockito.spy(new File("/dir/current")); - doReturn(files).when(mockDir).listFiles(); - doReturn(mockDir).when(sd).getCurrentDir(); - - return sd; + static StorageDirectory mockDirectoryWithEditLogs(String... fileNames) { + return FSImageTestUtil.mockStorageDirectory(NameNodeDirType.EDITS, false, fileNames); } } Modified: hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1158072&r1=1158071&r2=1158072&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java (original) +++ hadoop/common/branches/HDFS-1623/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java Tue Aug 16 00:37:15 2011 @@ -24,8 +24,8 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName; import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName; @@ -168,14 +168,14 @@ public class TestNNStorageRetentionManag StoragePurger mockPurger = Mockito.mock(NNStorageRetentionManager.StoragePurger.class); - ArgumentCaptor imagesPurgedCaptor = - ArgumentCaptor.forClass(FoundFSImage.class); - ArgumentCaptor logsPurgedCaptor = - ArgumentCaptor.forClass(FoundEditLog.class); + ArgumentCaptor imagesPurgedCaptor = + ArgumentCaptor.forClass(FSImageFile.class); + ArgumentCaptor logsPurgedCaptor = + ArgumentCaptor.forClass(EditLogFile.class); // Ask the manager to purge files we don't need any more new NNStorageRetentionManager(conf, - tc.mockStorage(), tc.mockEditLog(), mockPurger) + tc.mockStorage(), tc.mockEditLog(mockPurger), mockPurger) .purgeOldStorage(); // Verify that it asked the purger to remove the correct files @@ -186,7 +186,7 @@ public class TestNNStorageRetentionManag // Check images Set purgedPaths = Sets.newHashSet(); - for (FoundFSImage purged : imagesPurgedCaptor.getAllValues()) { + for (FSImageFile purged : imagesPurgedCaptor.getAllValues()) { purgedPaths.add(purged.getFile().toString()); } Assert.assertEquals(Joiner.on(",").join(tc.expectedPurgedImages), @@ -194,7 +194,7 @@ public class TestNNStorageRetentionManag // Check images purgedPaths.clear(); - for (FoundEditLog purged : logsPurgedCaptor.getAllValues()) { + for (EditLogFile purged : logsPurgedCaptor.getAllValues()) { purgedPaths.add(purged.getFile().toString()); } Assert.assertEquals(Joiner.on(",").join(tc.expectedPurgedLogs), @@ -216,7 +216,7 @@ public class TestNNStorageRetentionManag } StorageDirectory mockStorageDir() { - return TestFSImageStorageInspector.mockDirectory( + return FSImageTestUtil.mockStorageDirectory( type, false, files.toArray(new String[0])); } @@ -256,13 +256,14 @@ public class TestNNStorageRetentionManag return mockStorageForDirs(sds.toArray(new StorageDirectory[0])); } - public FSEditLog mockEditLog() { + public FSEditLog mockEditLog(StoragePurger purger) { final List jms = Lists.newArrayList(); for (FakeRoot root : dirRoots.values()) { if (!root.type.isOfType(NameNodeDirType.EDITS)) continue; FileJournalManager fjm = new FileJournalManager( root.mockStorageDir()); + fjm.purger = purger; jms.add(fjm); } @@ -272,17 +273,15 @@ public class TestNNStorageRetentionManag @Override public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); - assert args.length == 2; + assert args.length == 1; long txId = (Long) args[0]; - StoragePurger purger = (StoragePurger) args[1]; for (JournalManager jm : jms) { - jm.purgeLogsOlderThan(txId, purger); + jm.purgeLogsOlderThan(txId); } return null; } - }).when(mockLog).purgeLogsOlderThan( - Mockito.anyLong(), (StoragePurger) Mockito.anyObject()); + }).when(mockLog).purgeLogsOlderThan(Mockito.anyLong()); return mockLog; } }