Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3A616EF93 for ; Tue, 12 Feb 2013 00:50:23 +0000 (UTC) Received: (qmail 78965 invoked by uid 500); 12 Feb 2013 00:50:23 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 78927 invoked by uid 500); 12 Feb 2013 00:50:23 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 78915 invoked by uid 99); 12 Feb 2013 00:50:22 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Feb 2013 00:50:22 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Feb 2013 00:50:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 14FA32388978; Tue, 12 Feb 2013 00:50:02 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1445006 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/ Date: Tue, 12 Feb 2013 00:50:01 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130212005002.14FA32388978@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Tue Feb 12 00:50:00 2013 New Revision: 1445006 URL: http://svn.apache.org/r1445006 Log: HDFS-4342. Directories configured in dfs.namenode.edits.dir.required but not in dfs.namenode.edits.dir are silently ignored. Contributed by Arpit Agarwal Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1445006&r1=1445005&r2=1445006&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Feb 12 00:50:00 2013 @@ -317,6 +317,10 @@ Release 2.0.4-beta - UNRELEASED HDFS-4471. Namenode WebUI file browsing does not work with wildcard addresses configured. (Andrew Wang via atm) + HDFS-4342. Directories configured in dfs.namenode.edits.dir.required + but not in dfs.namenode.edits.dir are silently ignored. (Arpit Agarwal + via szetszwo) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1445006&r1=1445005&r2=1445006&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Feb 12 00:50:00 2013 @@ -235,6 +235,7 @@ public class DFSConfigKeys extends Commo public static final String DFS_NAMENODE_SHARED_EDITS_DIR_KEY = "dfs.namenode.shared.edits.dir"; public static final String DFS_NAMENODE_EDITS_PLUGIN_PREFIX = "dfs.namenode.edits.journal-plugin"; public static final String DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY = "dfs.namenode.edits.dir.required"; + public static final String DFS_NAMENODE_EDITS_DIR_DEFAULT = "file:///tmp/hadoop/dfs/name"; public static final String DFS_CLIENT_READ_PREFETCH_SIZE_KEY = "dfs.client.read.prefetch.size"; public static final String DFS_CLIENT_RETRY_WINDOW_BASE= "dfs.client.retry.window.base"; public static final String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id"; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1445006&r1=1445005&r2=1445006&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Feb 12 00:50:00 2013 @@ -127,6 +127,7 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.ServiceFailedException; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -422,51 +423,73 @@ public class FSNamesystem implements Nam } /** - - /** - * Instantiates an FSNamesystem loaded from the image and edits - * directories specified in the passed Configuration. - * - * @param conf the Configuration which specifies the storage directories - * from which to load - * @return an FSNamesystem which contains the loaded namespace - * @throws IOException if loading fails + * Check the supplied configuration for correctness. + * @param conf Supplies the configuration to validate. + * @throws IOException if the configuration could not be queried. + * @throws IllegalArgumentException if the configuration is invalid. */ - public static FSNamesystem loadFromDisk(Configuration conf) + private static void checkConfiguration(Configuration conf) throws IOException { - Collection namespaceDirs = FSNamesystem.getNamespaceDirs(conf); - List namespaceEditsDirs = - FSNamesystem.getNamespaceEditsDirs(conf); - return loadFromDisk(conf, namespaceDirs, namespaceEditsDirs); - } - /** - * Instantiates an FSNamesystem loaded from the image and edits - * directories passed. - * - * @param conf the Configuration which specifies the storage directories - * from which to load - * @param namespaceDirs directories to load the fsimages - * @param namespaceEditsDirs directories to load the edits from - * @return an FSNamesystem which contains the loaded namespace - * @throws IOException if loading fails - */ - public static FSNamesystem loadFromDisk(Configuration conf, - Collection namespaceDirs, List namespaceEditsDirs) - throws IOException { + final Collection namespaceDirs = + FSNamesystem.getNamespaceDirs(conf); + final Collection editsDirs = + FSNamesystem.getNamespaceEditsDirs(conf); + final Collection requiredEditsDirs = + FSNamesystem.getRequiredNamespaceEditsDirs(conf); + final Collection sharedEditsDirs = + FSNamesystem.getSharedEditsDirs(conf); + + for (URI u : requiredEditsDirs) { + if (u.toString().compareTo( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_DEFAULT) == 0) { + continue; + } + + // Each required directory must also be in editsDirs or in + // sharedEditsDirs. + if (!editsDirs.contains(u) && + !sharedEditsDirs.contains(u)) { + throw new IllegalArgumentException( + "Required edits directory " + u.toString() + " not present in " + + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY + ". " + + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY + "=" + + editsDirs.toString() + "; " + + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY + "=" + + requiredEditsDirs.toString() + ". " + + DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "=" + + sharedEditsDirs.toString() + "."); + } + } if (namespaceDirs.size() == 1) { LOG.warn("Only one image storage directory (" + DFS_NAMENODE_NAME_DIR_KEY + ") configured. Beware of dataloss" + " due to lack of redundant storage directories!"); } - if (namespaceEditsDirs.size() == 1) { + if (editsDirs.size() == 1) { LOG.warn("Only one namespace edits storage directory (" + DFS_NAMENODE_EDITS_DIR_KEY + ") configured. Beware of dataloss" + " due to lack of redundant storage directories!"); } + } + + /** + * Instantiates an FSNamesystem loaded from the image and edits + * directories specified in the passed Configuration. + * + * @param conf the Configuration which specifies the storage directories + * from which to load + * @return an FSNamesystem which contains the loaded namespace + * @throws IOException if loading fails + */ + public static FSNamesystem loadFromDisk(Configuration conf) + throws IOException { - FSImage fsImage = new FSImage(conf, namespaceDirs, namespaceEditsDirs); + checkConfiguration(conf); + FSImage fsImage = new FSImage(conf, + FSNamesystem.getNamespaceDirs(conf), + FSNamesystem.getNamespaceEditsDirs(conf)); FSNamesystem namesystem = new FSNamesystem(conf, fsImage); StartupOption startOpt = NameNode.getStartupOption(conf); if (startOpt == StartupOption.RECOVER) { @@ -913,7 +936,8 @@ public class FSNamesystem implements Nam "\n\t\t- use Backup Node as a persistent and up-to-date storage " + "of the file system meta-data."); } else if (dirNames.isEmpty()) { - dirNames = Collections.singletonList("file:///tmp/hadoop/dfs/name"); + dirNames = Collections.singletonList( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_DEFAULT); } return Util.stringCollectionAsURIs(dirNames); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1445006&r1=1445005&r2=1445006&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Feb 12 00:50:00 2013 @@ -78,6 +78,7 @@ import org.apache.hadoop.util.ServicePlu import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -781,6 +782,26 @@ public class NameNode { } /** + * Clone the supplied configuration but remove the shared edits dirs. + * + * @param conf Supplies the original configuration. + * @return Cloned configuration without the shared edit dirs. + * @throws IOException on failure to generate the configuration. + */ + private static Configuration getConfigurationWithoutSharedEdits( + Configuration conf) + throws IOException { + List editsDirs = FSNamesystem.getNamespaceEditsDirs(conf, false); + String editsDirsString = Joiner.on(",").join(editsDirs); + + Configuration confWithoutShared = new Configuration(conf); + confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); + confWithoutShared.setStrings(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + editsDirsString); + return confWithoutShared; + } + + /** * Format a new shared edits dir and copy in enough edit log segments so that * the standby NN can start up. * @@ -809,11 +830,8 @@ public class NameNode { NNStorage existingStorage = null; try { - Configuration confWithoutShared = new Configuration(conf); - confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); - FSNamesystem fsns = FSNamesystem.loadFromDisk(confWithoutShared, - FSNamesystem.getNamespaceDirs(conf), - FSNamesystem.getNamespaceEditsDirs(conf, false)); + FSNamesystem fsns = + FSNamesystem.loadFromDisk(getConfigurationWithoutSharedEdits(conf)); existingStorage = fsns.getFSImage().getStorage(); NamespaceInfo nsInfo = existingStorage.getNamespaceInfo(); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java?rev=1445006&r1=1445005&r2=1445006&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java Tue Feb 12 00:50:00 2013 @@ -309,6 +309,88 @@ public class TestNameEditsConfigs { } /** + * Test edits.dir.required configuration options. + * 1. Directory present in dfs.namenode.edits.dir.required but not in + * dfs.namenode.edits.dir. Expected to fail. + * 2. Directory present in both dfs.namenode.edits.dir.required and + * dfs.namenode.edits.dir. Expected to succeed. + * 3. Directory present only in dfs.namenode.edits.dir. Expected to + * succeed. + */ + @Test + public void testNameEditsRequiredConfigs() throws IOException { + MiniDFSCluster cluster = null; + File nameAndEditsDir = new File(base_dir, "name_and_edits"); + File nameAndEditsDir2 = new File(base_dir, "name_and_edits2"); + + // 1 + // Bad configuration. Add a directory to dfs.namenode.edits.dir.required + // without adding it to dfs.namenode.edits.dir. + try { + Configuration conf = new HdfsConfiguration(); + conf.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, + nameAndEditsDir2.toURI().toString()); + conf.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + nameAndEditsDir.toURI().toString()); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_DATA_NODES) + .manageNameDfsDirs(false) + .build(); + fail("Successfully started cluster but should not have been able to."); + } catch (IllegalArgumentException iae) { // expect to fail + LOG.info("EXPECTED: cluster start failed due to bad configuration" + iae); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + cluster = null; + } + + // 2 + // Good configuration. Add a directory to both dfs.namenode.edits.dir.required + // and dfs.namenode.edits.dir. + try { + Configuration conf = new HdfsConfiguration(); + conf.setStrings( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + nameAndEditsDir.toURI().toString(), + nameAndEditsDir2.toURI().toString()); + conf.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, + nameAndEditsDir2.toURI().toString()); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_DATA_NODES) + .manageNameDfsDirs(false) + .build(); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + + // 3 + // Good configuration. Adds a directory to dfs.namenode.edits.dir but not to + // dfs.namenode.edits.dir.required. + try { + Configuration conf = new HdfsConfiguration(); + conf.setStrings( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + nameAndEditsDir.toURI().toString(), + nameAndEditsDir2.toURI().toString()); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_DATA_NODES) + .manageNameDfsDirs(false) + .build(); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + /** * Test various configuration options of dfs.namenode.name.dir and dfs.namenode.edits.dir * This test tries to simulate failure scenarios. * 1. Start cluster with shared name and edits dir