Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4A7C87CB8 for ; Thu, 14 Jul 2011 19:11:10 +0000 (UTC) Received: (qmail 16940 invoked by uid 500); 14 Jul 2011 19:11:10 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 16876 invoked by uid 500); 14 Jul 2011 19:11:09 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 16863 invoked by uid 99); 14 Jul 2011 19:11:09 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Jul 2011 19:11:09 +0000 X-ASF-Spam-Status: No, hits=-1998.0 required=5.0 tests=ALL_TRUSTED,FB_GET_MEDS X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Jul 2011 19:11:07 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 5124523888CE; Thu, 14 Jul 2011 19:10:47 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1146854 - in /hadoop/common/branches/HDFS-1073/hdfs: ./ src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ src/test/hdfs/org/apache/hadoop/hdfs/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/h... Date: Thu, 14 Jul 2011 19:10:47 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110714191047.5124523888CE@eris.apache.org> Author: todd Date: Thu Jul 14 19:10:46 2011 New Revision: 1146854 URL: http://svn.apache.org/viewvc?rev=1146854&view=rev Log: HDFS-2101. Fix remaining unit tests for new storage filenames. Contributed by Todd Lipcon. Modified: hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSFinalize.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Modified: hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt (original) +++ hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt Thu Jul 14 19:10:46 2011 @@ -68,3 +68,4 @@ HDFS-2010. Fix NameNode to exit if all e via todd) HDFS-2123. Checkpoint interval should be based on txn count, not size. (todd) HDFS-1979. Fix backupnode for new edits/image layout. (todd) +HDFS-2101. Fix remaining unit tests for new storage filenames. (todd) Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Thu Jul 14 19:10:46 2011 @@ -123,7 +123,7 @@ class ImageLoaderCurrent implements Imag protected final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23, - -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36 }; + -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38}; private int imageVersion = 0; /* (non-Javadoc) Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSFinalize.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSFinalize.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSFinalize.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSFinalize.java Thu Jul 14 19:10:46 2011 @@ -19,6 +19,9 @@ package org.apache.hadoop.hdfs; import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.List; + import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -26,6 +29,11 @@ import org.apache.hadoop.conf.Configurat import static org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType.NAME_NODE; import static org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType.DATA_NODE; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.test.GenericTestUtils; + +import com.google.common.collect.Lists; +import com.google.common.io.Files; /** * This test ensures the appropriate response from the system when @@ -57,14 +65,17 @@ public class TestDFSFinalize extends Tes * because its removal is asynchronous therefore we have no reliable * way to know when it will happen. */ - void checkResult(String[] nameNodeDirs, String[] dataNodeDirs) throws IOException { + static void checkResult(String[] nameNodeDirs, String[] dataNodeDirs) throws Exception { + List dirs = Lists.newArrayList(); for (int i = 0; i < nameNodeDirs.length; i++) { - assertTrue(new File(nameNodeDirs[i],"current").isDirectory()); - assertTrue(new File(nameNodeDirs[i],"current/VERSION").isFile()); - assertTrue(new File(nameNodeDirs[i],"current/edits").isFile()); - assertTrue(new File(nameNodeDirs[i],"current/fsimage").isFile()); - assertTrue(new File(nameNodeDirs[i],"current/fstime").isFile()); + File curDir = new File(nameNodeDirs[i], "current"); + dirs.add(curDir); + FSImageTestUtil.assertReasonableNameCurrentDir(curDir); } + + FSImageTestUtil.assertParallelFilesAreIdentical( + dirs, Collections.emptySet()); + for (int i = 0; i < dataNodeDirs.length; i++) { assertEquals( UpgradeUtilities.checksumContents( Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java Thu Jul 14 19:10:46 2011 @@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.ser import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.List; import junit.framework.TestCase; @@ -32,8 +34,11 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.util.StringUtils; +import com.google.common.collect.Lists; + /** * This test ensures the appropriate response (successful or failure) from * the system when the system is rolled back under various storage state and @@ -61,26 +66,26 @@ public class TestDFSRollback extends Tes * Verify that the new current directory is the old previous. * It is assumed that the server has recovered and rolled back. */ - void checkResult(NodeType nodeType, String[] baseDirs) throws IOException { - switch (nodeType) { - case NAME_NODE: - for (int i = 0; i < baseDirs.length; i++) { - assertTrue(new File(baseDirs[i],"current").isDirectory()); - assertTrue(new File(baseDirs[i],"current/VERSION").isFile()); - assertTrue(new File(baseDirs[i],"current/edits").isFile()); - assertTrue(new File(baseDirs[i],"current/fsimage").isFile()); - assertTrue(new File(baseDirs[i],"current/fstime").isFile()); - } - break; - case DATA_NODE: - for (int i = 0; i < baseDirs.length; i++) { + void checkResult(NodeType nodeType, String[] baseDirs) throws Exception { + List curDirs = Lists.newArrayList(); + for (String baseDir : baseDirs) { + File curDir = new File(baseDir, "current"); + curDirs.add(curDir); + switch (nodeType) { + case NAME_NODE: + FSImageTestUtil.assertReasonableNameCurrentDir(curDir); + break; + case DATA_NODE: assertEquals( - UpgradeUtilities.checksumContents( - nodeType, new File(baseDirs[i],"current")), - UpgradeUtilities.checksumMasterDataNodeContents()); + UpgradeUtilities.checksumContents(nodeType, curDir), + UpgradeUtilities.checksumMasterDataNodeContents()); + break; } - break; } + + FSImageTestUtil.assertParallelFilesAreIdentical( + curDirs, Collections.emptySet()); + for (int i = 0; i < baseDirs.length; i++) { assertFalse(new File(baseDirs[i],"previous").isDirectory()); } @@ -241,21 +246,17 @@ public class TestDFSRollback extends Tes log("NameNode rollback with no edits file", numDirs); UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); baseDirs = UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); - for (File f : baseDirs) { - FileUtil.fullyDelete(new File(f,"edits")); - } + deleteMatchingFiles(baseDirs, "edits.*"); startNameNodeShouldFail(StartupOption.ROLLBACK, - "Edits file is not found"); + "but there are no logs to load"); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("NameNode rollback with no image file", numDirs); UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); baseDirs = UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); - for (File f : baseDirs) { - FileUtil.fullyDelete(new File(f,"fsimage")); - } + deleteMatchingFiles(baseDirs, "fsimage_.*"); startNameNodeShouldFail(StartupOption.ROLLBACK, - "Image file is not found"); + "No valid image files found"); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("NameNode rollback with corrupt version file", numDirs); @@ -284,6 +285,16 @@ public class TestDFSRollback extends Tes } // end numDir loop } + private void deleteMatchingFiles(File[] baseDirs, String regex) { + for (File baseDir : baseDirs) { + for (File f : baseDir.listFiles()) { + if (f.getName().matches(regex)) { + f.delete(); + } + } + } + } + protected void tearDown() throws Exception { LOG.info("Shutting down MiniDFSCluster"); if (cluster != null) cluster.shutdown(); Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java Thu Jul 14 19:10:46 2011 @@ -417,7 +417,7 @@ public class UpgradeUtilities { storage = new NNStorage(conf, Collections.emptyList(), Collections.emptyList()); - + storage.setStorageInfo(version); StorageDirectory sd = storage.new StorageDirectory(parent[i].getParentFile()); sd.write(versionFile); versionFiles[i] = versionFile; Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Thu Jul 14 19:10:46 2011 @@ -171,7 +171,12 @@ public abstract class FSImageTestUtil { } for (List sameNameList : groupedByName.values()) { - assertFileContentsSame(sameNameList.toArray(new File[0])); + if (sameNameList.get(0).isDirectory()) { + // recurse + assertParallelFilesAreIdentical(sameNameList, ignoredFileNames); + } else { + assertFileContentsSame(sameNameList.toArray(new File[0])); + } } } @@ -334,5 +339,14 @@ public abstract class FSImageTestUtil { } } + public static void assertReasonableNameCurrentDir(File curDir) + throws IOException { + assertTrue(curDir.isDirectory()); + assertTrue(new File(curDir, "VERSION").isFile()); + assertTrue(new File(curDir, "seen_txid").isFile()); + File image = findNewestImageFile(curDir.toString()); + assertNotNull(image); + } + } Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java Thu Jul 14 19:10:46 2011 @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.junit.Assert.*; import junit.framework.Assert; import java.io.*; import java.net.URI; @@ -29,6 +30,8 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; @@ -47,20 +50,6 @@ public class TestCheckPointForSecurityTo short replication = 3; MiniDFSCluster cluster = null; - NameNode startNameNode( Configuration conf, - String imageDirs, - String editsDirs, - StartupOption start) throws IOException { - conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:0"); - conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); - conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, imageDirs); - conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsDirs); - String[] args = new String[]{start.getName()}; - NameNode nn = NameNode.createNameNode(args, conf); - Assert.assertTrue(nn.isInSafeMode()); - return nn; - } - private void cancelToken(Token token) throws IOException { cluster.getNamesystem().cancelDelegationToken(token); @@ -95,10 +84,12 @@ public class TestCheckPointForSecurityTo String[] args = new String[]{"-saveNamespace"}; // verify that the edits file is NOT empty - Collection editsDirs = cluster.getNameEditsDirs(0); - for(URI uri : editsDirs) { - File ed = new File(uri.getPath()); - Assert.assertTrue(new File(ed, "current/edits").length() > Integer.SIZE/Byte.SIZE); + NameNode nn = cluster.getNameNode(); + for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) { + FoundEditLog log = FSImageTestUtil.findLatestEditsLog(sd); + assertTrue(log.isInProgress()); + assertEquals("In-progress log " + log + " should have 5 transactions", + 5, log.validateLog().numTransactions); } // Saving image in safe mode should succeed @@ -108,10 +99,12 @@ public class TestCheckPointForSecurityTo } catch(Exception e) { throw new IOException(e.getMessage()); } - // verify that the edits file is empty - for(URI uri : editsDirs) { - File ed = new File(uri.getPath()); - Assert.assertTrue(new File(ed, "current/edits").length() == Integer.SIZE/Byte.SIZE); + // verify that the edits file is empty except for the START txn + for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) { + FoundEditLog log = FSImageTestUtil.findLatestEditsLog(sd); + assertTrue(log.isInProgress()); + assertEquals("In-progress log " + log + " should only have START txn", + 1, log.validateLog().numTransactions); } // restart cluster Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1146854&r1=1146853&r2=1146854&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Thu Jul 14 19:10:46 2011 @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; /** @@ -128,11 +129,10 @@ public class TestOfflineImageViewer exte cluster.getNameNode().saveNamespace(); // Determine location of fsimage file - URI [] files = cluster.getNameDirs(0).toArray(new URI[0]); - orig = new File(files[0].getPath(), "current/fsimage"); - - if (!orig.exists()) { - fail("Didn't generate or can't find fsimage."); + orig = FSImageTestUtil.findLatestImageFile( + cluster.getNameNode().getFSImage().getStorage().getStorageDir(0)); + if (orig == null) { + fail("Didn't generate or can't find fsimage"); } } finally { if(cluster != null)