Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C4297DFC4 for ; Wed, 22 May 2013 19:43:22 +0000 (UTC) Received: (qmail 61880 invoked by uid 500); 22 May 2013 19:43:23 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 61828 invoked by uid 500); 22 May 2013 19:43:23 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 61815 invoked by uid 99); 22 May 2013 19:43:23 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 May 2013 19:43:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 May 2013 19:43:16 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C926323889FA; Wed, 22 May 2013 19:42:54 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1485375 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src: contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/ main/java/org/apache/hadoop/hdfs/qjournal/client/ main/java/org/apache/hadoop/hdfs/qjou... Date: Wed, 22 May 2013 19:42:53 -0000 To: hdfs-commits@hadoop.apache.org From: atm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130522194254.C926323889FA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: atm Date: Wed May 22 19:42:52 2013 New Revision: 1485375 URL: http://svn.apache.org/r1485375 Log: HDFS-4298. StorageRetentionManager spews warnings when used with QJM. Contributed by Aaron T. Myers. Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Wed May 22 19:42:52 2013 @@ -500,9 +500,15 @@ public class BookKeeperJournalManager im } } - @Override public void selectInputStreams(Collection streams, long fromTxId, boolean inProgressOk) throws IOException { + selectInputStreams(streams, fromTxId, inProgressOk, true); + } + + @Override + public void selectInputStreams(Collection streams, + long fromTxId, boolean inProgressOk, boolean forReading) + throws IOException { List currentLedgerList = getLedgerList(fromTxId, inProgressOk); try { Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java Wed May 22 19:42:52 2013 @@ -109,7 +109,7 @@ interface AsyncLogger { * Fetch the list of edit logs available on the remote node. */ public ListenableFuture getEditLogManifest( - long fromTxnId); + long fromTxnId, boolean forReading); /** * Prepare recovery. See the HDFS-3077 design document for details. Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java Wed May 22 19:42:52 2013 @@ -263,13 +263,13 @@ class AsyncLoggerSet { } public QuorumCall - getEditLogManifest(long fromTxnId) { + getEditLogManifest(long fromTxnId, boolean forReading) { Map> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture future = - logger.getEditLogManifest(fromTxnId); + logger.getEditLogManifest(fromTxnId, forReading); calls.put(logger, future); } return QuorumCall.create(calls); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java Wed May 22 19:42:52 2013 @@ -519,12 +519,12 @@ public class IPCLoggerChannel implements @Override public ListenableFuture getEditLogManifest( - final long fromTxnId) { + final long fromTxnId, final boolean forReading) { return executor.submit(new Callable() { @Override public RemoteEditLogManifest call() throws IOException { GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest( - journalId, fromTxnId); + journalId, fromTxnId, forReading); // Update the http port, since we need this to build URLs to any of the // returned logs. httpPort = ret.getHttpPort(); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java Wed May 22 19:42:52 2013 @@ -445,13 +445,18 @@ public class QuorumJournalManager implem public void close() throws IOException { loggers.close(); } + + public void selectInputStreams(Collection streams, + long fromTxnId, boolean inProgressOk) throws IOException { + selectInputStreams(streams, fromTxnId, inProgressOk, true); + } @Override public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk) throws IOException { + long fromTxnId, boolean inProgressOk, boolean forReading) throws IOException { QuorumCall q = - loggers.getEditLogManifest(fromTxnId); + loggers.getEditLogManifest(fromTxnId, forReading); Map resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java Wed May 22 19:42:52 2013 @@ -123,10 +123,12 @@ public interface QJournalProtocol { /** * @param jid the journal from which to enumerate edits * @param sinceTxId the first transaction which the client cares about + * @param forReading whether or not the caller intends to read from the edit + * logs * @return a list of edit log segments since the given transaction ID. */ public GetEditLogManifestResponseProto getEditLogManifest( - String jid, long sinceTxId) throws IOException; + String jid, long sinceTxId, boolean forReading) throws IOException; /** * Begin the recovery process for a given segment. See the HDFS-3077 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java Wed May 22 19:42:52 2013 @@ -202,7 +202,8 @@ public class QJournalProtocolServerSideT try { return impl.getEditLogManifest( request.getJid().getIdentifier(), - request.getSinceTxId()); + request.getSinceTxId(), + request.getForReading()); } catch (IOException e) { throw new ServiceException(e); } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java Wed May 22 19:42:52 2013 @@ -228,12 +228,13 @@ public class QJournalProtocolTranslatorP @Override public GetEditLogManifestResponseProto getEditLogManifest(String jid, - long sinceTxId) throws IOException { + long sinceTxId, boolean forReading) throws IOException { try { return rpcProxy.getEditLogManifest(NULL_CONTROLLER, GetEditLogManifestRequestProto.newBuilder() .setJid(convertJournalId(jid)) .setSinceTxId(sinceTxId) + .setForReading(forReading) .build()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java Wed May 22 19:42:52 2013 @@ -627,14 +627,14 @@ class Journal implements Closeable { /** * @see QJournalProtocol#getEditLogManifest(String, long) */ - public RemoteEditLogManifest getEditLogManifest(long sinceTxId) - throws IOException { + public RemoteEditLogManifest getEditLogManifest(long sinceTxId, + boolean forReading) throws IOException { // No need to checkRequest() here - anyone may ask for the list // of segments. checkFormatted(); RemoteEditLogManifest manifest = new RemoteEditLogManifest( - fjm.getRemoteEditLogs(sinceTxId)); + fjm.getRemoteEditLogs(sinceTxId, forReading)); return manifest; } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java Wed May 22 19:42:52 2013 @@ -175,10 +175,10 @@ class JournalNodeRpcServer implements QJ @Override public GetEditLogManifestResponseProto getEditLogManifest(String jid, - long sinceTxId) throws IOException { + long sinceTxId, boolean forReading) throws IOException { RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid) - .getEditLogManifest(sinceTxId); + .getEditLogManifest(sinceTxId, forReading); return GetEditLogManifestResponseProto.newBuilder() .setManifest(PBHelper.convert(manifest)) Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java Wed May 22 19:42:52 2013 @@ -77,7 +77,7 @@ class BackupJournalManager implements Jo @Override public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk) { + long fromTxnId, boolean inProgressOk, boolean forReading) { // This JournalManager is never used for input. Therefore it cannot // return any transactions } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Wed May 22 19:42:52 2013 @@ -252,7 +252,7 @@ public class FSEditLog implements LogsPu // Safety check: we should never start a segment if there are // newer txids readable. List streams = new ArrayList(); - journalSet.selectInputStreams(streams, segmentTxId, true); + journalSet.selectInputStreams(streams, segmentTxId, true, true); if (!streams.isEmpty()) { String error = String.format("Cannot start writing at txid %s " + "when there is a stream available for read: %s", @@ -879,7 +879,7 @@ public class FSEditLog implements LogsPu */ public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId) throws IOException { - return journalSet.getEditLogManifest(fromTxId); + return journalSet.getEditLogManifest(fromTxId, true); } /** @@ -1129,8 +1129,8 @@ public class FSEditLog implements LogsPu } public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk) throws IOException { - journalSet.selectInputStreams(streams, fromTxId, inProgressOk); + long fromTxId, boolean inProgressOk, boolean forReading) throws IOException { + journalSet.selectInputStreams(streams, fromTxId, inProgressOk, forReading); } public Collection selectInputStreams( @@ -1149,7 +1149,7 @@ public class FSEditLog implements LogsPu long fromTxId, long toAtLeastTxId, MetaRecoveryContext recovery, boolean inProgressOk) throws IOException { List streams = new ArrayList(); - selectInputStreams(streams, fromTxId, inProgressOk); + selectInputStreams(streams, fromTxId, inProgressOk, true); try { checkForGaps(streams, fromTxId, toAtLeastTxId, inProgressOk); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Wed May 22 19:42:52 2013 @@ -164,10 +164,13 @@ public class FileJournalManager implemen /** * Find all editlog segments starting at or above the given txid. * @param fromTxId the txnid which to start looking + * @param forReading whether or not the caller intends to read from the edit + * logs * @return a list of remote edit logs * @throws IOException if edit logs cannot be listed. */ - public List getRemoteEditLogs(long firstTxId) throws IOException { + public List getRemoteEditLogs(long firstTxId, + boolean forReading) throws IOException { File currentDir = sd.getCurrentDir(); List allLogFiles = matchEditLogs(currentDir); List ret = Lists.newArrayListWithCapacity( @@ -177,11 +180,15 @@ public class FileJournalManager implemen if (elf.hasCorruptHeader() || elf.isInProgress()) continue; if (elf.getFirstTxId() >= firstTxId) { ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId)); - } else if ((firstTxId > elf.getFirstTxId()) && - (firstTxId <= elf.getLastTxId())) { - // Note that this behavior is different from getLogFiles below. - throw new IllegalStateException("Asked for firstTxId " + firstTxId - + " which is in the middle of file " + elf.file); + } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) { + // If the firstTxId is in the middle of an edit log segment + if (forReading) { + // Note that this behavior is different from getLogFiles below. + throw new IllegalStateException("Asked for firstTxId " + firstTxId + + " which is in the middle of file " + elf.file); + } else { + ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId)); + } } } @@ -242,7 +249,7 @@ public class FileJournalManager implemen @Override synchronized public void selectInputStreams( Collection streams, long fromTxId, - boolean inProgressOk) throws IOException { + boolean inProgressOk, boolean forReading) throws IOException { List elfs = matchEditLogs(sd.getCurrentDir()); LOG.debug(this + ": selecting input streams starting at " + fromTxId + (inProgressOk ? " (inProgress ok) " : " (excluding inProgress) ") + Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java Wed May 22 19:42:52 2013 @@ -233,10 +233,12 @@ public class JournalSet implements Journ * may not be sorted-- this is up to the caller. * @param fromTxId The transaction ID to start looking for streams at * @param inProgressOk Should we consider unfinalized streams? + * @param forReading Whether or not the caller intends to read from + * the returned streams. */ @Override public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk) throws IOException { + long fromTxId, boolean inProgressOk, boolean forReading) throws IOException { final PriorityQueue allStreams = new PriorityQueue(64, EDIT_LOG_INPUT_STREAM_COMPARATOR); @@ -246,7 +248,8 @@ public class JournalSet implements Journ continue; } try { - jas.getManager().selectInputStreams(allStreams, fromTxId, inProgressOk); + jas.getManager().selectInputStreams(allStreams, fromTxId, inProgressOk, + forReading); } catch (IOException ioe) { LOG.warn("Unable to determine input streams from " + jas.getManager() + ". Skipping.", ioe); @@ -585,14 +588,15 @@ public class JournalSet implements Journ * @param fromTxId Starting transaction id to read the logs. * @return RemoteEditLogManifest object. */ - public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId) { + public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId, + boolean forReading) { // Collect RemoteEditLogs available from each FileJournalManager List allLogs = Lists.newArrayList(); for (JournalAndStream j : journals) { if (j.getManager() instanceof FileJournalManager) { FileJournalManager fjm = (FileJournalManager)j.getManager(); try { - allLogs.addAll(fjm.getRemoteEditLogs(fromTxId)); + allLogs.addAll(fjm.getRemoteEditLogs(fromTxId, forReading)); } catch (Throwable t) { LOG.warn("Cannot list edit logs in " + fjm, t); } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java Wed May 22 19:42:52 2013 @@ -42,12 +42,13 @@ interface LogsPurgeable { * * @param fromTxId the first transaction id we want to read * @param inProgressOk whether or not in-progress streams should be returned + * @param forReading whether or not the caller intends to read from the edit logs * * @return a list of streams * @throws IOException if the underlying storage has an error or is otherwise * inaccessible */ void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk) throws IOException; + long fromTxId, boolean inProgressOk, boolean forReading) throws IOException; } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java Wed May 22 19:42:52 2013 @@ -108,7 +108,7 @@ public class NNStorageRetentionManager { long purgeLogsFrom = Math.max(0, minimumRequiredTxId - numExtraEditsToRetain); ArrayList editLogs = new ArrayList(); - purgeableLogs.selectInputStreams(editLogs, purgeLogsFrom, false); + purgeableLogs.selectInputStreams(editLogs, purgeLogsFrom, false, false); Collections.sort(editLogs, new Comparator() { @Override public int compare(EditLogInputStream a, EditLogInputStream b) { Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Wed May 22 19:42:52 2013 @@ -820,7 +820,7 @@ public class SecondaryNameNode implement @Override public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk) { + long fromTxId, boolean inProgressOk, boolean forReading) { Iterator iter = storage.dirIterator(); while (iter.hasNext()) { StorageDirectory dir = iter.next(); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto Wed May 22 19:42:52 2013 @@ -169,6 +169,8 @@ message NewEpochResponseProto { message GetEditLogManifestRequestProto { required JournalIdProto jid = 1; required uint64 sinceTxId = 2; // Transaction ID + // Whether or not the client will be reading from the returned streams. + optional bool forReading = 3 [default = true]; } message GetEditLogManifestResponseProto { Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java Wed May 22 19:42:52 2013 @@ -31,6 +31,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; @@ -900,6 +901,26 @@ public class TestQuorumJournalManager { "QJM to \\[127.0.0.1:\\d+, 127.0.0.1:\\d+, 127.0.0.1:\\d+\\]"); } + @Test + public void testSelectInputStreamsNotOnBoundary() throws Exception { + final int txIdsPerSegment = 10; + for (int txid = 1; txid <= 5 * txIdsPerSegment; txid += txIdsPerSegment) { + writeSegment(cluster, qjm, txid, txIdsPerSegment, true); + } + File curDir = cluster.getCurrentDir(0, JID); + GenericTestUtils.assertGlobEquals(curDir, "edits_.*", + NNStorage.getFinalizedEditsFileName(1, 10), + NNStorage.getFinalizedEditsFileName(11, 20), + NNStorage.getFinalizedEditsFileName(21, 30), + NNStorage.getFinalizedEditsFileName(31, 40), + NNStorage.getFinalizedEditsFileName(41, 50)); + + ArrayList streams = new ArrayList(); + qjm.selectInputStreams(streams, 25, false, false); + + verifyEdits(streams, 25, 50); + } + private QuorumJournalManager createSpyingQJM() throws IOException, URISyntaxException { Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Wed May 22 19:42:52 2013 @@ -74,7 +74,7 @@ public class TestFileJournalManager { final PriorityQueue allStreams = new PriorityQueue(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); - jm.selectInputStreams(allStreams, fromTxId, inProgressOk); + jm.selectInputStreams(allStreams, fromTxId, inProgressOk, true); EditLogInputStream elis = null; try { while ((elis = allStreams.poll()) != null) { @@ -393,7 +393,7 @@ public class TestFileJournalManager { final PriorityQueue allStreams = new PriorityQueue(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); - jm.selectInputStreams(allStreams, txId, inProgressOk); + jm.selectInputStreams(allStreams, txId, inProgressOk, true); EditLogInputStream elis = null, ret; try { while ((elis = allStreams.poll()) != null) { @@ -463,6 +463,6 @@ public class TestFileJournalManager { private static String getLogsAsString( FileJournalManager fjm, long firstTxId) throws IOException { - return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId)); + return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId, true)); } } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Wed May 22 19:42:52 2013 @@ -174,7 +174,7 @@ public class TestGenericJournalConf { @Override public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk) { + long fromTxnId, boolean inProgressOk, boolean forReading) { } @Override Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1485375&r1=1485374&r2=1485375&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java Wed May 22 19:42:52 2013 @@ -355,11 +355,12 @@ public class TestNNStorageRetentionManag public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); journalSet.selectInputStreams((Collection)args[0], - (long)((Long)args[1]), (boolean)((Boolean)args[2])); + (long)((Long)args[1]), (boolean)((Boolean)args[2]), + (boolean)((Boolean)args[3])); return null; } }).when(mockLog).selectInputStreams(Mockito.anyCollection(), - Mockito.anyLong(), Mockito.anyBoolean()); + Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyBoolean()); return mockLog; } }