Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D03D492BF for ; Mon, 17 Oct 2011 20:24:20 +0000 (UTC) Received: (qmail 10542 invoked by uid 500); 17 Oct 2011 20:24:20 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 10496 invoked by uid 500); 17 Oct 2011 20:24:19 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 10488 invoked by uid 99); 17 Oct 2011 20:24:19 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Oct 2011 20:24:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Oct 2011 20:24:17 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E5BA023888FD; Mon, 17 Oct 2011 20:23:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1185354 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/ Date: Mon, 17 Oct 2011 20:23:57 -0000 To: hdfs-commits@hadoop.apache.org From: jitendra@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111017202357.E5BA023888FD@eris.apache.org> Author: jitendra Date: Mon Oct 17 20:23:57 2011 New Revision: 1185354 URL: http://svn.apache.org/viewvc?rev=1185354&view=rev Log: HDFS-2188. Make FSEditLog create its journals from a list of URIs rather than NNStorage. Contributed by Ivan Kelly. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1185354&r1=1185353&r2=1185354&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Oct 17 20:23:57 2011 @@ -129,6 +129,9 @@ Trunk (unreleased changes) file or creating a file without specifying the replication parameter. (szetszwo) + HDFS-2188. Make FSEditLog create its journals from a list of URIs rather + than NNStorage. (Ivan Kelly via jitendra) + Release 0.23.0 - Unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1185354&r1=1185353&r2=1185354&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Mon Oct 17 20:23:57 2011 @@ -18,10 +18,11 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.Util.now; - +import java.net.URI; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; @@ -42,9 +43,11 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.conf.Configuration; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; /** * FSEditLog maintains a log of the namespace modifications. @@ -122,23 +125,68 @@ public class FSEditLog { } }; + final private Collection editsDirs; + + /** + * Construct FSEditLog with default configuration, taking editDirs from NNStorage + * @param storage Storage object used by namenode + */ + @VisibleForTesting FSEditLog(NNStorage storage) { + this(new Configuration(), storage, Collections.emptyList()); + } + + /** + * Constructor for FSEditLog. Add underlying journals are constructed, but + * no streams are opened until open() is called. + * + * @param conf The namenode configuration + * @param storage Storage object used by namenode + * @param editsDirs List of journals to use + */ + FSEditLog(Configuration conf, NNStorage storage, Collection editsDirs) { isSyncRunning = false; this.storage = storage; metrics = NameNode.getNameNodeMetrics(); lastPrintTime = now(); + + if (editsDirs.isEmpty()) { + // if this is the case, no edit dirs have been explictly configured + // image dirs are to be used for edits too + try { + editsDirs = Lists.newArrayList(storage.getEditsDirectories()); + } catch (IOException ioe) { + // cannot get list from storage, so the empty editsDirs + // will be assigned. an error will be thrown on first use + // of the editlog, as no journals will exist + } + this.editsDirs = editsDirs; + } else { + this.editsDirs = Lists.newArrayList(editsDirs); + } this.journalSet = new JournalSet(); - for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { - journalSet.add(new FileJournalManager(sd)); + for (URI u : this.editsDirs) { + StorageDirectory sd = storage.getStorageDirectory(u); + if (sd != null) { + journalSet.add(new FileJournalManager(sd)); + } } - + if (journalSet.isEmpty()) { LOG.error("No edits directories configured!"); } state = State.BETWEEN_LOG_SEGMENTS; } - + + /** + * Get the list of URIs the editlog is using for storage + * @return collection of URIs in use by the edit log + */ + Collection getEditURIs() { + return editsDirs; + } + /** * Initialize the output stream for logging, opening the first * log segment. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1185354&r1=1185353&r2=1185354&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Mon Oct 17 20:23:57 2011 @@ -120,7 +120,7 @@ public class FSImage implements Closeabl storage.setRestoreFailedStorage(true); } - this.editLog = new FSEditLog(storage); + this.editLog = new FSEditLog(conf, storage, editsDirs); archivalManager = new NNStorageRetentionManager(conf, storage, editLog); } @@ -150,8 +150,7 @@ public class FSImage implements Closeabl "NameNode formatting should be performed before reading the image"; Collection imageDirs = storage.getImageDirectories(); - Collection editsDirs = storage.getEditsDirectories(); - + Collection editsDirs = editLog.getEditURIs(); // none of the data dirs exist if((imageDirs.size() == 0 || editsDirs.size() == 0) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1185354&r1=1185353&r2=1185354&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Mon Oct 17 20:23:57 2011 @@ -59,6 +59,7 @@ import org.apache.hadoop.net.DNS; import com.google.common.base.Preconditions; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; /** * NNStorage is responsible for management of the StorageDirectories used by @@ -154,7 +155,9 @@ public class NNStorage extends Storage i storageDirs = new CopyOnWriteArrayList(); - setStorageDirectories(imageDirs, editsDirs); + // this may modify the editsDirs, so copy before passing in + setStorageDirectories(imageDirs, + Lists.newArrayList(editsDirs)); } @Override // Storage @@ -299,6 +302,27 @@ public class NNStorage extends Storage i } /** + * Return the storage directory corresponding to the passed URI + * @param uri URI of a storage directory + * @return The matching storage directory or null if none found + */ + StorageDirectory getStorageDirectory(URI uri) { + try { + uri = Util.fileAsURI(new File(uri)); + Iterator it = dirIterator(); + for (; it.hasNext(); ) { + StorageDirectory sd = it.next(); + if (Util.fileAsURI(sd.getRoot()).equals(uri)) { + return sd; + } + } + } catch (IOException ioe) { + LOG.warn("Error converting file to URI", ioe); + } + return null; + } + + /** * Checks the consistency of a URI, in particular if the scheme * is specified and is supported by a concrete implementation * @param u URI whose consistency is being checked. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1185354&r1=1185353&r2=1185354&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Mon Oct 17 20:23:57 2011 @@ -42,10 +42,13 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.conf.Configuration; import org.mockito.Mockito; +import org.mockito.Matchers; import com.google.common.base.Joiner; import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Files; @@ -106,7 +109,7 @@ public abstract class FSImageTestUtil { Mockito.doReturn(type) .when(sd).getStorageDirType(); Mockito.doReturn(currentDir).when(sd).getCurrentDir(); - + Mockito.doReturn(currentDir).when(sd).getRoot(); Mockito.doReturn(mockFile(true)).when(sd).getVersionFile(); Mockito.doReturn(mockFile(false)).when(sd).getPreviousDir(); return sd; @@ -128,7 +131,8 @@ public abstract class FSImageTestUtil { // Version file should always exist doReturn(mockFile(true)).when(sd).getVersionFile(); - + doReturn(mockFile(true)).when(sd).getRoot(); + // Previous dir optionally exists doReturn(mockFile(previousExists)) .when(sd).getPreviousDir(); @@ -143,6 +147,7 @@ public abstract class FSImageTestUtil { doReturn(files).when(mockDir).listFiles(); doReturn(mockDir).when(sd).getCurrentDir(); + return sd; } @@ -170,11 +175,16 @@ public abstract class FSImageTestUtil { assertTrue(logDir.mkdirs() || logDir.exists()); Files.deleteDirectoryContents(logDir); NNStorage storage = Mockito.mock(NNStorage.class); - List sds = Lists.newArrayList( - FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS)); + StorageDirectory sd + = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS); + List sds = Lists.newArrayList(sd); Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); + Mockito.doReturn(sd).when(storage) + .getStorageDirectory(Matchers.anyObject()); - return new FSEditLog(storage); + return new FSEditLog(new Configuration(), + storage, + ImmutableList.of(logDir.toURI())); } /** Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1185354&r1=1185353&r2=1185354&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Mon Oct 17 20:23:57 2011 @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; @@ -861,8 +862,11 @@ public class TestEditLog extends TestCas * The syntax [1,] specifies an in-progress log starting at * txid 1. */ - private NNStorage mockStorageWithEdits(String... editsDirSpecs) { + private NNStorage mockStorageWithEdits(String... editsDirSpecs) throws IOException { List sds = Lists.newArrayList(); + List uris = Lists.newArrayList(); + + NNStorage storage = Mockito.mock(NNStorage.class); for (String dirSpec : editsDirSpecs) { List files = Lists.newArrayList(); String[] logSpecs = dirSpec.split("\\|"); @@ -878,13 +882,17 @@ public class TestEditLog extends TestCas Long.valueOf(m.group(2)))); } } - sds.add(FSImageTestUtil.mockStorageDirectory( + StorageDirectory sd = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.EDITS, false, - files.toArray(new String[0]))); - } - - NNStorage storage = Mockito.mock(NNStorage.class); + files.toArray(new String[0])); + sds.add(sd); + URI u = URI.create("file:///storage"+ Math.random()); + Mockito.doReturn(sd).when(storage).getStorageDirectory(u); + uris.add(u); + } + Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); + Mockito.doReturn(uris).when(storage).getEditsDirectories(); return storage; }