Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 83A4F10919 for ; Wed, 18 Dec 2013 02:46:26 +0000 (UTC) Received: (qmail 96160 invoked by uid 500); 18 Dec 2013 02:46:26 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 96122 invoked by uid 500); 18 Dec 2013 02:46:26 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 96114 invoked by uid 99); 18 Dec 2013 02:46:26 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Dec 2013 02:46:26 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Dec 2013 02:46:22 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id EFE922388993; Wed, 18 Dec 2013 02:46:00 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1551812 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/ Date: Wed, 18 Dec 2013 02:46:00 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131218024600.EFE922388993@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Wed Dec 18 02:46:00 2013 New Revision: 1551812 URL: http://svn.apache.org/r1551812 Log: HDFS-5674. Editlog code cleanup: remove @SuppressWarnings("deprecation") in FSEditLogOp; change FSEditLogOpCodes.fromByte(..) to be more efficient; and change Some fields in FSEditLog to final. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1551812&r1=1551811&r2=1551812&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Dec 18 02:46:00 2013 @@ -750,6 +750,10 @@ Release 2.4.0 - UNRELEASED HDFS-5629. Support HTTPS in JournalNode and SecondaryNameNode. (Haohui Mai via jing9) + HDFS-5674. Editlog code cleanup: remove @SuppressWarnings("deprecation") in + FSEditLogOp; change FSEditLogOpCodes.fromByte(..) to be more efficient; and + change Some fields in FSEditLog to final. (szetszwo) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1551812&r1=1551811&r2=1551812&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Wed Dec 18 02:46:00 2013 @@ -160,10 +160,10 @@ public class FSEditLog implements LogsPu private long totalTimeTransactions; // total time for all transactions private NameNodeMetrics metrics; - private NNStorage storage; - private Configuration conf; + private final NNStorage storage; + private final Configuration conf; - private List editsDirs; + private final List editsDirs; private ThreadLocal cache = new ThreadLocal() { @@ -176,7 +176,7 @@ public class FSEditLog implements LogsPu /** * The edit directories that are shared between primary and secondary. */ - private List sharedEditsDirs; + private final List sharedEditsDirs; private static class TransactionId { public long txid; @@ -203,10 +203,6 @@ public class FSEditLog implements LogsPu * @param editsDirs List of journals to use */ FSEditLog(Configuration conf, NNStorage storage, List editsDirs) { - init(conf, storage, editsDirs); - } - - private void init(Configuration conf, NNStorage storage, List editsDirs) { isSyncRunning = false; this.conf = conf; this.storage = storage; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1551812&r1=1551811&r2=1551812&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java Wed Dec 18 02:46:00 2013 @@ -118,11 +118,10 @@ import com.google.common.base.Preconditi @InterfaceStability.Unstable public abstract class FSEditLogOp { public final FSEditLogOpCodes opCode; - long txid; + long txid = HdfsConstants.INVALID_TXID; byte[] rpcClientId = RpcConstants.DUMMY_CLIENT_ID; int rpcCallId = RpcConstants.INVALID_CALL_ID; - @SuppressWarnings("deprecation") final public static class OpInstanceCache { private EnumMap inst = new EnumMap(FSEditLogOpCodes.class); @@ -147,13 +146,10 @@ public abstract class FSEditLogOp { inst.put(OP_REASSIGN_LEASE, new ReassignLeaseOp()); inst.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp()); inst.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp()); - inst.put(OP_CANCEL_DELEGATION_TOKEN, - new CancelDelegationTokenOp()); + inst.put(OP_CANCEL_DELEGATION_TOKEN, new CancelDelegationTokenOp()); inst.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp()); - inst.put(OP_START_LOG_SEGMENT, - new LogSegmentOp(OP_START_LOG_SEGMENT)); - inst.put(OP_END_LOG_SEGMENT, - new LogSegmentOp(OP_END_LOG_SEGMENT)); + inst.put(OP_START_LOG_SEGMENT, new LogSegmentOp(OP_START_LOG_SEGMENT)); + inst.put(OP_END_LOG_SEGMENT, new LogSegmentOp(OP_END_LOG_SEGMENT)); inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp()); inst.put(OP_ALLOW_SNAPSHOT, new AllowSnapshotOp()); @@ -163,12 +159,10 @@ public abstract class FSEditLogOp { inst.put(OP_RENAME_SNAPSHOT, new RenameSnapshotOp()); inst.put(OP_SET_GENSTAMP_V2, new SetGenstampV2Op()); inst.put(OP_ALLOCATE_BLOCK_ID, new AllocateBlockIdOp()); - inst.put(OP_ADD_CACHE_DIRECTIVE, - new AddCacheDirectiveInfoOp()); - inst.put(OP_MODIFY_CACHE_DIRECTIVE, - new ModifyCacheDirectiveInfoOp()); - inst.put(OP_REMOVE_CACHE_DIRECTIVE, - new RemoveCacheDirectiveInfoOp()); + + inst.put(OP_ADD_CACHE_DIRECTIVE, new AddCacheDirectiveInfoOp()); + inst.put(OP_MODIFY_CACHE_DIRECTIVE, new ModifyCacheDirectiveInfoOp()); + inst.put(OP_REMOVE_CACHE_DIRECTIVE, new RemoveCacheDirectiveInfoOp()); inst.put(OP_ADD_CACHE_POOL, new AddCachePoolOp()); inst.put(OP_MODIFY_CACHE_POOL, new ModifyCachePoolOp()); inst.put(OP_REMOVE_CACHE_POOL, new RemoveCachePoolOp()); @@ -185,7 +179,6 @@ public abstract class FSEditLogOp { */ private FSEditLogOp(FSEditLogOpCodes opCode) { this.opCode = opCode; - this.txid = HdfsConstants.INVALID_TXID; } public long getTransactionId() { @@ -3332,9 +3325,7 @@ public abstract class FSEditLogOp { * @param in The stream to read from. * @param logVersion The version of the data coming from the stream. */ - @SuppressWarnings("deprecation") - public Reader(DataInputStream in, StreamLimiter limiter, - int logVersion) { + public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) { this.logVersion = logVersion; if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) { this.checksum = new PureJavaCrc32(); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1551812&r1=1551811&r2=1551812&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Wed Dec 18 02:46:00 2013 @@ -17,9 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.util.Map; -import java.util.HashMap; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -30,7 +27,6 @@ import org.apache.hadoop.classification. @InterfaceStability.Unstable public enum FSEditLogOpCodes { // last op code in file - OP_INVALID ((byte) -1), OP_ADD ((byte) 0), OP_RENAME_OLD ((byte) 1), // deprecated operation OP_DELETE ((byte) 2), @@ -69,9 +65,12 @@ public enum FSEditLogOpCodes { OP_ADD_CACHE_POOL ((byte) 35), OP_MODIFY_CACHE_POOL ((byte) 36), OP_REMOVE_CACHE_POOL ((byte) 37), - OP_MODIFY_CACHE_DIRECTIVE ((byte) 38); + OP_MODIFY_CACHE_DIRECTIVE ((byte) 38), + + // Note that fromByte(..) depends on OP_INVALID being at the last position. + OP_INVALID ((byte) -1); - private byte opCode; + private final byte opCode; /** * Constructor @@ -91,14 +90,7 @@ public enum FSEditLogOpCodes { return opCode; } - private static final Map byteToEnum = - new HashMap(); - - static { - // initialize byte to enum map - for(FSEditLogOpCodes opCode : values()) - byteToEnum.put(opCode.getOpCode(), opCode); - } + private static final FSEditLogOpCodes[] VALUES = FSEditLogOpCodes.values(); /** * Converts byte to FSEditLogOpCodes enum value @@ -107,6 +99,12 @@ public enum FSEditLogOpCodes { * @return enum with byte value of opCode */ public static FSEditLogOpCodes fromByte(byte opCode) { - return byteToEnum.get(opCode); + if (opCode == -1) { + return OP_INVALID; + } + if (opCode >= 0 && opCode < OP_INVALID.ordinal()) { + return VALUES[opCode]; + } + return null; } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1551812&r1=1551811&r2=1551812&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Wed Dec 18 02:46:00 2013 @@ -31,6 +31,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; +import java.util.HashMap; import java.util.Map; import java.util.SortedMap; @@ -383,4 +384,33 @@ public class TestFSEditLogLoader { assertTrue(!validation.hasCorruptHeader()); assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId()); } + + private static final Map byteToEnum = + new HashMap(); + static { + for(FSEditLogOpCodes opCode : FSEditLogOpCodes.values()) { + byteToEnum.put(opCode.getOpCode(), opCode); + } + } + + private static FSEditLogOpCodes fromByte(byte opCode) { + return byteToEnum.get(opCode); + } + + @Test + public void testFSEditLogOpCodes() throws IOException { + //try all codes + for(FSEditLogOpCodes c : FSEditLogOpCodes.values()) { + final byte code = c.getOpCode(); + assertEquals("c=" + c + ", code=" + code, + c, FSEditLogOpCodes.fromByte(code)); + } + + //try all byte values + for(int b = 0; b < (1 << Byte.SIZE); b++) { + final byte code = (byte)b; + assertEquals("b=" + b + ", code=" + code, + fromByte(code), FSEditLogOpCodes.fromByte(code)); + } + } }