From hdfs-commits-return-1304-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Fri Apr 01 03:51:06 2011 Return-Path: Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: (qmail 13985 invoked from network); 1 Apr 2011 03:51:05 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 1 Apr 2011 03:51:05 -0000 Received: (qmail 99341 invoked by uid 500); 1 Apr 2011 03:51:05 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 99312 invoked by uid 500); 1 Apr 2011 03:51:05 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 99304 invoked by uid 99); 1 Apr 2011 03:51:04 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Apr 2011 03:51:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Apr 2011 03:51:00 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D23A023888E4; Fri, 1 Apr 2011 03:50:38 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1087574 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/server/protocol/ src/test/hdfs/org/apache/hadoop/hdfs/server/n... Date: Fri, 01 Apr 2011 03:50:38 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110401035038.D23A023888E4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: todd Date: Fri Apr 1 03:50:38 2011 New Revision: 1087574 URL: http://svn.apache.org/viewvc?rev=1087574&view=rev Log: HDFS-1794. Add code to list which edit logs are available on a remote NN. Contributed by Todd Lipcon. Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/common/Storage.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original) +++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Fri Apr 1 03:50:38 2011 @@ -11,3 +11,4 @@ HDFS-1538. Refactor more startup and ima HDFS-1729. Add code to detect valid length of an edits file. (todd) HDFS-1793. Add code to inspect a storage directory with txid-based filenames (todd) +HDFS-1794. Add code to list which edit logs are available on a remote NN (todd) Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/common/Storage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Fri Apr 1 03:50:38 2011 @@ -179,6 +179,16 @@ public abstract class Storage extends St return new DirIterator(dirType); } + public Iterable dirIterable(final StorageDirType dirType) { + return new Iterable() { + @Override + public Iterator iterator() { + return dirIterator(dirType); + } + }; + } + + /** * generate storage list (debug line) */ Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri Apr 1 03:50:38 2011 @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.JournalStream.JournalType; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; @@ -811,6 +812,21 @@ public class FSEditLog implements NNStor } /** + * Return a manifest of what finalized edit logs are available + */ + public RemoteEditLogManifest getEditLogManifest(long sinceTxId) + throws IOException { + FSImageTransactionalStorageInspector inspector = + new FSImageTransactionalStorageInspector(); + + for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { + inspector.inspectDirectory(sd); + } + + return inspector.getEditLogManifest(sinceTxId); + } + + /** * Closes the current edit log and opens edits.new. * @return the transaction id that will be used as the first transaction * in the new log Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java Fri Apr 1 03:50:38 2011 @@ -37,6 +37,8 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import com.google.common.collect.Lists; @@ -219,6 +221,22 @@ class FSImageTransactionalStorageInspect public boolean needToSave() { return false; // TODO do we need to do this ever? } + + + RemoteEditLogManifest getEditLogManifest(long sinceTxId) { + List logs = Lists.newArrayList(); + for (LogGroup g : logGroups.values()) { + if (!g.hasFinalized) continue; + + FoundEditLog fel = g.getBestNonCorruptLog(); + if (fel.getLastTxId() < sinceTxId) continue; + + logs.add(new RemoteEditLog(fel.getStartTxId(), + fel.getLastTxId())); + } + + return new RemoteEditLogManifest(logs); + } /** * A group of logs that all start at the same txid. Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Apr 1 03:50:38 2011 @@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -4337,6 +4338,10 @@ public class FSNamesystem implements FSC writeUnlock(); } } + + public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException { + return getEditLog().getEditLogManifest(sinceTxId); + } NamenodeCommand startCheckpoint( NamenodeRegistration bnReg, // backup node Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Apr 1 03:50:38 2011 @@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.NodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet; @@ -1120,6 +1121,12 @@ public class NameNode implements Namenod public void rollFsImage(CheckpointSignature sig) throws IOException { namesystem.rollFSImage(sig); } + + @Override + public RemoteEditLogManifest getEditLogManifest(long sinceTxId) + throws IOException { + return namesystem.getEditLogManifest(sinceTxId); + } public void finalizeUpgrade() throws IOException { namesystem.finalizeUpgrade(); Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java Fri Apr 1 03:50:38 2011 @@ -177,6 +177,15 @@ public interface NamenodeProtocol extend */ public void endCheckpoint(NamenodeRegistration registration, CheckpointSignature sig) throws IOException; + + + /** + * Return a structure containing details about all edit logs + * available to be fetched from the NameNode. + * @param sinceTxId return only logs that contain transactions >= sinceTxId + */ + public RemoteEditLogManifest getEditLogManifest(long sinceTxId) + throws IOException; /** * Get the size of the active name-node journal (edit log) in bytes. Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java?rev=1087574&view=auto ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java (added) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java Fri Apr 1 03:50:38 2011 @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.protocol; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; + +public class RemoteEditLog implements Writable { + private long startTxId = -1; + private long endTxId = -1; + + public RemoteEditLog() { + } + + public RemoteEditLog(long startTxId, long endTxId) { + this.startTxId = startTxId; + this.endTxId = endTxId; + } + + public long getStartTxId() { + return startTxId; + } + + public long getEndTxId() { + return endTxId; + } + + @Override + public String toString() { + return "[" + startTxId + "," + endTxId + "]"; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeLong(startTxId); + out.writeLong(endTxId); + } + + @Override + public void readFields(DataInput in) throws IOException { + startTxId = in.readLong(); + endTxId = in.readLong(); + } + +} Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java?rev=1087574&view=auto ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java (added) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java Fri Apr 1 03:50:38 2011 @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.protocol; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.io.Writable; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * An enumeration of logs available on a remote NameNode. + */ +public class RemoteEditLogManifest implements Writable { + + private List logs; + + public RemoteEditLogManifest() { + } + + public RemoteEditLogManifest(List logs) { + this.logs = logs; + checkState(); + } + + + /** + * Check that the logs are contiguous and non-overlapping + * sequences of transactions, in sorted order + * @throws IllegalStateException if incorrect + */ + private void checkState() { + Preconditions.checkNotNull(logs); + + RemoteEditLog prev = null; + for (RemoteEditLog log : logs) { + if (prev != null) { + if (log.getStartTxId() != prev.getEndTxId() + 1) { + throw new IllegalStateException("Invalid log manifest:" + this); + } + } + + prev = log; + } + } + + public List getLogs() { + return Collections.unmodifiableList(logs); + } + + + + @Override + public String toString() { + return "[" + Joiner.on(", ").join(logs) + "]"; + } + + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(logs.size()); + for (RemoteEditLog log : logs) { + log.write(out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int numLogs = in.readInt(); + logs = Lists.newArrayList(); + for (int i = 0; i < numLogs; i++) { + RemoteEditLog log = new RemoteEditLog(); + log.readFields(in); + logs.add(log); + } + checkState(); + } +} Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1087574&r1=1087573&r2=1087574&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Fri Apr 1 03:50:38 2011 @@ -318,6 +318,34 @@ public class TestFSImageStorageInspector new File("/foo3/current/edits_inprogress_457") }, plan.getEditsFiles().toArray(new File[0])); + // Check log manifest + assertEquals("[[123,456]]", inspector.getEditLogManifest(123).toString()); + assertEquals("[[123,456]]", inspector.getEditLogManifest(456).toString()); + assertEquals("[]", inspector.getEditLogManifest(457).toString()); + } + + @Test + public void testLogManifest() throws IOException { + FSImageTransactionalStorageInspector inspector = + new FSImageTransactionalStorageInspector(); + inspector.inspectDirectory( + mockDirectoryWithEditLogs("/foo1/current/edits_1-1", + "/foo1/current/edits_2-200")); + inspector.inspectDirectory( + mockDirectoryWithEditLogs("/foo2/current/edits_1-inprogress", + "/foo2/current/edits_201-400")); + inspector.inspectDirectory( + mockDirectoryWithEditLogs("/foo3/current/edits_1-1", + "/foo3/current/edts_2-200")); + + assertEquals("[[1,1], [2,200], [201,400]]", + inspector.getEditLogManifest(1).toString()); + assertEquals("[[2,200], [201,400]]", + inspector.getEditLogManifest(2).toString()); + assertEquals("[[2,200], [201,400]]", + inspector.getEditLogManifest(10).toString()); + assertEquals("[[201,400]]", + inspector.getEditLogManifest(201).toString()); } private StorageDirectory mockDirectoryWithEditLogs(String... fileNames) {