Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0771D9A90 for ; Thu, 5 Apr 2012 20:17:06 +0000 (UTC) Received: (qmail 56493 invoked by uid 500); 5 Apr 2012 20:17:05 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 56449 invoked by uid 500); 5 Apr 2012 20:17:05 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 56440 invoked by uid 99); 5 Apr 2012 20:17:05 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Apr 2012 20:17:05 +0000 X-ASF-Spam-Status: No, hits=-1996.5 required=5.0 tests=ALL_TRUSTED,URIBL_BLACK X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Apr 2012 20:16:45 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E21622388B46; Thu, 5 Apr 2012 20:16:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1310048 [2/3] - in /hadoop/common/branches/HDFS-3092/hadoop-hdfs-project: dev-support/ hadoop-hdfs-httpfs/ hadoop-hdfs/ hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ hadoop-hdfs/src/main/java/org/apache/hadoo... Date: Thu, 05 Apr 2012 20:16:20 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120405201622.E21622388B46@eris.apache.org> Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Apr 5 20:16:15 2012 @@ -25,15 +25,17 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_STANDBY_CHECKPOINTS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_STANDBY_CHECKPOINTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT; @@ -49,15 +51,13 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_STANDBY_CHECKPOINTS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_STANDBY_CHECKPOINTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERSIST_BLOCKS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERSIST_BLOCKS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERSIST_BLOCKS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT; @@ -150,9 +150,9 @@ import org.apache.hadoop.hdfs.server.com import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; @@ -164,6 +164,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer; import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState; import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; +import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; @@ -260,30 +261,28 @@ public class FSNamesystem implements Nam static final int DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED = 100; static int BLOCK_DELETION_INCREMENT = 1000; - private boolean isPermissionEnabled; - private boolean persistBlocks; - private UserGroupInformation fsOwner; - private String supergroup; - private boolean standbyShouldCheckpoint; + private final boolean isPermissionEnabled; + private final boolean persistBlocks; + private final UserGroupInformation fsOwner; + private final String supergroup; + private final boolean standbyShouldCheckpoint; // Scan interval is not configurable. private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS); - private DelegationTokenSecretManager dtSecretManager; - private boolean alwaysUseDelegationTokensForTests; + private final DelegationTokenSecretManager dtSecretManager; + private final boolean alwaysUseDelegationTokensForTests; - // - // Stores the correct file name hierarchy - // + /** The namespace tree. */ FSDirectory dir; - private BlockManager blockManager; - private DatanodeStatistics datanodeStatistics; + private final BlockManager blockManager; + private final DatanodeStatistics datanodeStatistics; // Block pool ID used by this namenode private String blockPoolId; - LeaseManager leaseManager = new LeaseManager(this); + final LeaseManager leaseManager = new LeaseManager(this); Daemon smmthread = null; // SafeModeMonitor thread @@ -291,23 +290,23 @@ public class FSNamesystem implements Nam private volatile boolean hasResourcesAvailable = false; private volatile boolean fsRunning = true; - long systemStart = 0; + + /** The start time of the namesystem. */ + private final long startTime = now(); - //resourceRecheckInterval is how often namenode checks for the disk space availability - private long resourceRecheckInterval; + /** The interval of namenode checking for the disk space availability */ + private final long resourceRecheckInterval; // The actual resource checker instance. NameNodeResourceChecker nnResourceChecker; - private FsServerDefaults serverDefaults; - - private boolean supportAppends; - private ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure = - ReplaceDatanodeOnFailure.DEFAULT; + private final FsServerDefaults serverDefaults; + private final boolean supportAppends; + private final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure; private volatile SafeModeInfo safeMode; // safe mode information - private long maxFsObjects = 0; // maximum number of fs objects + private final long maxFsObjects; // maximum number of fs objects /** * The global generation stamp for this file system. @@ -315,10 +314,10 @@ public class FSNamesystem implements Nam private final GenerationStamp generationStamp = new GenerationStamp(); // precision of access times. - private long accessTimePrecision = 0; + private final long accessTimePrecision; - // lock to protect FSNamesystem. - private ReentrantReadWriteLock fsLock; + /** Lock to protect FSNamesystem. */ + private ReentrantReadWriteLock fsLock = new ReentrantReadWriteLock(true); /** * Used when this NN is in standby state to read from the shared edit log. @@ -336,9 +335,7 @@ public class FSNamesystem implements Nam */ private HAContext haContext; - private boolean haEnabled; - - private final Configuration conf; + private final boolean haEnabled; /** * Instantiates an FSNamesystem loaded from the image and edits @@ -350,10 +347,27 @@ public class FSNamesystem implements Nam * @throws IOException if loading fails */ public static FSNamesystem loadFromDisk(Configuration conf) - throws IOException { + throws IOException { Collection namespaceDirs = FSNamesystem.getNamespaceDirs(conf); List namespaceEditsDirs = FSNamesystem.getNamespaceEditsDirs(conf); + return loadFromDisk(conf, namespaceDirs, namespaceEditsDirs); + } + + /** + * Instantiates an FSNamesystem loaded from the image and edits + * directories passed. + * + * @param conf the Configuration which specifies the storage directories + * from which to load + * @param namespaceDirs directories to load the fsimages + * @param namespaceEditsDirs directories to load the edits from + * @return an FSNamesystem which contains the loaded namespace + * @throws IOException if loading fails + */ + public static FSNamesystem loadFromDisk(Configuration conf, + Collection namespaceDirs, List namespaceEditsDirs) + throws IOException { if (namespaceDirs.size() == 1) { LOG.warn("Only one " + DFS_NAMENODE_NAME_DIR_KEY @@ -374,8 +388,10 @@ public class FSNamesystem implements Nam HAUtil.isHAEnabled(conf, nameserviceId)); long timeTakenToLoadFSImage = now() - loadStart; LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs"); - NameNode.getNameNodeMetrics().setFsImageLoadTime( - (int) timeTakenToLoadFSImage); + NameNodeMetrics nnMetrics = NameNode.getNameNodeMetrics(); + if (nnMetrics != null) { + nnMetrics.setFsImageLoadTime((int) timeTakenToLoadFSImage); + } return namesystem; } @@ -390,9 +406,71 @@ public class FSNamesystem implements Nam * @throws IOException on bad configuration */ FSNamesystem(Configuration conf, FSImage fsImage) throws IOException { - this.conf = conf; try { - initialize(conf, fsImage); + resourceRecheckInterval = conf.getLong( + DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, + DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT); + + this.blockManager = new BlockManager(this, this, conf); + this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics(); + + this.fsOwner = UserGroupInformation.getCurrentUser(); + this.supergroup = conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY, + DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT); + this.isPermissionEnabled = conf.getBoolean(DFS_PERMISSIONS_ENABLED_KEY, + DFS_PERMISSIONS_ENABLED_DEFAULT); + LOG.info("fsOwner = " + fsOwner); + LOG.info("supergroup = " + supergroup); + LOG.info("isPermissionEnabled = " + isPermissionEnabled); + + final boolean persistBlocks = conf.getBoolean(DFS_PERSIST_BLOCKS_KEY, + DFS_PERSIST_BLOCKS_DEFAULT); + // block allocation has to be persisted in HA using a shared edits directory + // so that the standby has up-to-date namespace information + String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf); + this.haEnabled = HAUtil.isHAEnabled(conf, nameserviceId); + this.persistBlocks = persistBlocks || (haEnabled && HAUtil.usesSharedEditsDir(conf)); + + // Sanity check the HA-related config. + if (nameserviceId != null) { + LOG.info("Determined nameservice ID: " + nameserviceId); + } + LOG.info("HA Enabled: " + haEnabled); + if (!haEnabled && HAUtil.usesSharedEditsDir(conf)) { + LOG.warn("Configured NNs:\n" + DFSUtil.nnAddressesAsString(conf)); + throw new IOException("Invalid configuration: a shared edits dir " + + "must not be specified if HA is not enabled."); + } + + this.serverDefaults = new FsServerDefaults( + conf.getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT), + conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT), + conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT), + (short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT), + conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)); + + this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, + DFS_NAMENODE_MAX_OBJECTS_DEFAULT); + + this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 0); + this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT); + LOG.info("Append Enabled: " + haEnabled); + + this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); + + this.standbyShouldCheckpoint = conf.getBoolean( + DFS_HA_STANDBY_CHECKPOINTS_KEY, DFS_HA_STANDBY_CHECKPOINTS_DEFAULT); + + // For testing purposes, allow the DT secret manager to be started regardless + // of whether security is enabled. + alwaysUseDelegationTokensForTests = conf.getBoolean( + DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, + DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT); + + this.dtSecretManager = createDelegationTokenSecretManager(conf); + this.dir = new FSDirectory(fsImage, this, conf); + this.safeMode = new SafeModeInfo(conf); + } catch(IOException e) { LOG.error(getClass().getSimpleName() + " initialization failed.", e); close(); @@ -400,24 +478,6 @@ public class FSNamesystem implements Nam } } - /** - * Initialize FSNamesystem. - */ - private void initialize(Configuration conf, FSImage fsImage) - throws IOException { - resourceRecheckInterval = conf.getLong( - DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, - DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT); - this.systemStart = now(); - this.blockManager = new BlockManager(this, this, conf); - this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics(); - this.fsLock = new ReentrantReadWriteLock(true); // fair locking - setConfigurationParameters(conf); - dtSecretManager = createDelegationTokenSecretManager(conf); - this.dir = new FSDirectory(fsImage, this, conf); - this.safeMode = new SafeModeInfo(conf); - } - void loadFSImage(StartupOption startOpt, FSImage fsImage, boolean haEnabled) throws IOException { // format before starting up if requested @@ -601,13 +661,13 @@ public class FSNamesystem implements Nam } /** Start services required in standby state */ - void startStandbyServices() { + void startStandbyServices(final Configuration conf) { LOG.info("Starting services required for standby state"); if (!dir.fsImage.editLog.isOpenForRead()) { // During startup, we're already open for read. dir.fsImage.editLog.initSharedJournalsForRead(); } - editLogTailer = new EditLogTailer(this); + editLogTailer = new EditLogTailer(this, conf); editLogTailer.start(); if (standbyShouldCheckpoint) { standbyCheckpointer = new StandbyCheckpointer(conf, this); @@ -768,10 +828,6 @@ public class FSNamesystem implements Nam DFS_NAMENODE_SHARED_EDITS_DIR_KEY); return Util.stringCollectionAsURIs(dirNames); } - - public Configuration getConf() { - return conf; - } @Override public void readLock() { @@ -806,69 +862,6 @@ public class FSNamesystem implements Nam return hasReadLock() || hasWriteLock(); } - - /** - * Initializes some of the members from configuration - */ - private void setConfigurationParameters(Configuration conf) - throws IOException { - fsOwner = UserGroupInformation.getCurrentUser(); - - LOG.info("fsOwner=" + fsOwner); - - this.supergroup = conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY, - DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT); - this.isPermissionEnabled = conf.getBoolean(DFS_PERMISSIONS_ENABLED_KEY, - DFS_PERMISSIONS_ENABLED_DEFAULT); - LOG.info("supergroup=" + supergroup); - LOG.info("isPermissionEnabled=" + isPermissionEnabled); - - this.persistBlocks = conf.getBoolean(DFS_PERSIST_BLOCKS_KEY, - DFS_PERSIST_BLOCKS_DEFAULT); - // block allocation has to be persisted in HA using a shared edits directory - // so that the standby has up-to-date namespace information - String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf); - this.haEnabled = HAUtil.isHAEnabled(conf, nameserviceId); - this.persistBlocks |= haEnabled && HAUtil.usesSharedEditsDir(conf); - - // Sanity check the HA-related config. - if (nameserviceId != null) { - LOG.info("Determined nameservice ID: " + nameserviceId); - } - LOG.info("HA Enabled: " + haEnabled); - if (!haEnabled && HAUtil.usesSharedEditsDir(conf)) { - LOG.warn("Configured NNs:\n" + DFSUtil.nnAddressesAsString(conf)); - throw new IOException("Invalid configuration: a shared edits dir " + - "must not be specified if HA is not enabled."); - } - - this.serverDefaults = new FsServerDefaults( - conf.getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT), - conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT), - conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT), - (short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT), - conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)); - - this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, - DFS_NAMENODE_MAX_OBJECTS_DEFAULT); - - this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 0); - this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, - DFS_SUPPORT_APPEND_DEFAULT); - - this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); - - this.standbyShouldCheckpoint = conf.getBoolean( - DFS_HA_STANDBY_CHECKPOINTS_KEY, - DFS_HA_STANDBY_CHECKPOINTS_DEFAULT); - - // For testing purposes, allow the DT secret manager to be started regardless - // of whether security is enabled. - alwaysUseDelegationTokensForTests = - conf.getBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, - DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT); - } - NamespaceInfo getNamespaceInfo() { readLock(); try { @@ -2761,7 +2754,7 @@ public class FSNamesystem implements Nam } private Lease reassignLease(Lease lease, String src, String newHolder, - INodeFileUnderConstruction pendingFile) throws IOException { + INodeFileUnderConstruction pendingFile) { assert hasWriteLock(); if(newHolder == null) return lease; @@ -3329,7 +3322,7 @@ public class FSNamesystem implements Nam } Date getStartTime() { - return new Date(systemStart); + return new Date(startTime); } void finalizeUpgrade() throws IOException { @@ -3506,7 +3499,7 @@ public class FSNamesystem implements Nam if (!isPopulatingReplQueues() && !isInStandbyState()) { initializeReplQueues(); } - long timeInSafemode = now() - systemStart; + long timeInSafemode = now() - startTime; NameNode.stateChangeLog.info("STATE* Leaving safe mode after " + timeInSafemode/1000 + " secs."); NameNode.getNameNodeMetrics().setSafeModeTime((int) timeInSafemode); @@ -4876,7 +4869,7 @@ public class FSNamesystem implements Nam * * @param key new delegation key. */ - public void logUpdateMasterKey(DelegationKey key) throws IOException { + public void logUpdateMasterKey(DelegationKey key) { assert !isInSafeMode() : "this should never be called while in safemode, since we stop " + @@ -4889,7 +4882,7 @@ public class FSNamesystem implements Nam } private void logReassignLease(String leaseHolder, String src, - String newHolder) throws IOException { + String newHolder) { writeLock(); try { getEditLog().logReassignLease(leaseHolder, src, newHolder); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Apr 5 20:16:15 2012 @@ -58,6 +58,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetUtils; @@ -70,6 +71,9 @@ import org.apache.hadoop.tools.GetUserMa import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; + /********************************************************** * NameNode serves as both directory namespace manager and * "inode table" for the Hadoop DFS. There is a single NameNode @@ -729,6 +733,67 @@ public class NameNode { + "to true in order to format this filesystem"); } } + + @VisibleForTesting + public static boolean initializeSharedEdits(Configuration conf) { + return initializeSharedEdits(conf, true); + } + + @VisibleForTesting + public static boolean initializeSharedEdits(Configuration conf, + boolean force) { + return initializeSharedEdits(conf, force, false); + } + + /** + * Format a new shared edits dir. + * + * @param conf configuration + * @param force format regardless of whether or not the shared edits dir exists + * @param interactive prompt the user when a dir exists + * @return true if the command aborts, false otherwise + */ + private static boolean initializeSharedEdits(Configuration conf, + boolean force, boolean interactive) { + NNStorage existingStorage = null; + try { + FSNamesystem fsns = FSNamesystem.loadFromDisk(conf, + FSNamesystem.getNamespaceDirs(conf), + FSNamesystem.getNamespaceEditsDirs(conf, false)); + + existingStorage = fsns.getFSImage().getStorage(); + + Collection sharedEditsDirs = FSNamesystem.getSharedEditsDirs(conf); + if (!confirmFormat(sharedEditsDirs, force, interactive)) { + return true; // aborted + } + NNStorage newSharedStorage = new NNStorage(conf, + Lists.newArrayList(), + sharedEditsDirs); + + newSharedStorage.format(new NamespaceInfo( + existingStorage.getNamespaceID(), + existingStorage.getClusterID(), + existingStorage.getBlockPoolID(), + existingStorage.getCTime(), + existingStorage.getDistributedUpgradeVersion())); + } catch (Exception e) { + LOG.error("Could not format shared edits dir", e); + return true; // aborted + } finally { + // Have to unlock storage explicitly for the case when we're running in a + // unit test, which runs in the same JVM as NNs. + if (existingStorage != null) { + try { + existingStorage.unlockAll(); + } catch (IOException ioe) { + LOG.warn("Could not unlock storage directories", ioe); + return true; // aborted + } + } + } + return false; // did not abort + } private static boolean finalize(Configuration conf, boolean isConfirmationNeeded @@ -763,7 +828,8 @@ public class NameNode { StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + StartupOption.IMPORT.getName() + "] | [" + - StartupOption.BOOTSTRAPSTANDBY.getName() + "]"); + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + + StartupOption.INITIALIZESHAREDEDITS.getName() + "]"); } private static StartupOption parseArguments(String args[]) { @@ -804,6 +870,9 @@ public class NameNode { } else if (StartupOption.BOOTSTRAPSTANDBY.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.BOOTSTRAPSTANDBY; return startOpt; + } else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) { + startOpt = StartupOption.INITIALIZESHAREDEDITS; + return startOpt; } else { return null; } @@ -868,29 +937,39 @@ public class NameNode { } switch (startOpt) { - case FORMAT: + case FORMAT: { boolean aborted = format(conf, false); System.exit(aborted ? 1 : 0); return null; // avoid javac warning - case GENCLUSTERID: + } + case GENCLUSTERID: { System.err.println("Generating new cluster id:"); System.out.println(NNStorage.newClusterID()); System.exit(0); return null; - case FINALIZE: - aborted = finalize(conf, true); + } + case FINALIZE: { + boolean aborted = finalize(conf, true); System.exit(aborted ? 1 : 0); return null; // avoid javac warning - case BOOTSTRAPSTANDBY: + } + case BOOTSTRAPSTANDBY: { String toolArgs[] = Arrays.copyOfRange(argv, 1, argv.length); int rc = BootstrapStandby.run(toolArgs, conf); System.exit(rc); return null; // avoid warning + } + case INITIALIZESHAREDEDITS: { + boolean aborted = initializeSharedEdits(conf, false, true); + System.exit(aborted ? 1 : 0); + return null; // avoid warning + } case BACKUP: - case CHECKPOINT: + case CHECKPOINT: { NamenodeRole role = startOpt.toNodeRole(); DefaultMetricsSystem.initialize(role.toString().replace(" ", "")); return new BackupNode(conf, role); + } default: DefaultMetricsSystem.initialize("NameNode"); return new NameNode(conf); @@ -1061,7 +1140,7 @@ public class NameNode { @Override public void startStandbyServices() throws IOException { - namesystem.startStandbyServices(); + namesystem.startStandbyServices(conf); } @Override Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java Thu Apr 5 20:16:15 2012 @@ -61,6 +61,7 @@ public class EditLogTailer { private final EditLogTailerThread tailerThread; + private final Configuration conf; private final FSNamesystem namesystem; private FSEditLog editLog; @@ -98,13 +99,12 @@ public class EditLogTailer { */ private long sleepTimeMs; - public EditLogTailer(FSNamesystem namesystem) { + public EditLogTailer(FSNamesystem namesystem, Configuration conf) { this.tailerThread = new EditLogTailerThread(); + this.conf = conf; this.namesystem = namesystem; this.editLog = namesystem.getEditLog(); - - Configuration conf = namesystem.getConf(); lastLoadTimestamp = now(); logRollPeriodMs = conf.getInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, @@ -129,14 +129,12 @@ public class EditLogTailer { } private InetSocketAddress getActiveNodeAddress() { - Configuration conf = namesystem.getConf(); Configuration activeConf = HAUtil.getConfForOtherNode(conf); return NameNode.getServiceAddress(activeConf, true); } private NamenodeProtocol getActiveNodeProxy() throws IOException { if (cachedActiveProxy == null) { - Configuration conf = namesystem.getConf(); NamenodeProtocolPB proxy = RPC.waitForProxy(NamenodeProtocolPB.class, RPC.getProtocolVersion(NamenodeProtocolPB.class), activeAddr, conf); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java Thu Apr 5 20:16:15 2012 @@ -50,14 +50,19 @@ public class NamespaceInfo extends Stora super(); buildVersion = null; } - - public NamespaceInfo(int nsID, String clusterID, String bpID, - long cT, int duVersion) { + + public NamespaceInfo(int nsID, String clusterID, String bpID, + long cT, int duVersion, String buildVersion) { super(HdfsConstants.LAYOUT_VERSION, nsID, clusterID, cT); blockPoolID = bpID; - buildVersion = Storage.getBuildVersion(); + this.buildVersion = buildVersion; this.distributedUpgradeVersion = duVersion; } + + public NamespaceInfo(int nsID, String clusterID, String bpID, + long cT, int duVersion) { + this(nsID, clusterID, bpID, cT, duVersion, Storage.getBuildVersion()); + } public String getBuildVersion() { return buildVersion; Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Thu Apr 5 20:16:15 2012 @@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet; +import org.apache.hadoop.hdfs.web.URLUtils; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; @@ -224,8 +225,7 @@ public class DelegationTokenFetcher { URL remoteURL = new URL(url.toString()); SecurityUtil.fetchServiceTicket(remoteURL); - URLConnection connection = remoteURL.openConnection(); - + URLConnection connection = URLUtils.openConnection(remoteURL); InputStream in = connection.getInputStream(); Credentials ts = new Credentials(); dis = new DataInputStream(in); @@ -265,7 +265,7 @@ public class DelegationTokenFetcher { try { URL url = new URL(buf.toString()); SecurityUtil.fetchServiceTicket(url); - connection = (HttpURLConnection) url.openConnection(); + connection = (HttpURLConnection)URLUtils.openConnection(url); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new IOException("Error renewing token: " + connection.getResponseMessage()); @@ -359,7 +359,7 @@ public class DelegationTokenFetcher { try { URL url = new URL(buf.toString()); SecurityUtil.fetchServiceTicket(url); - connection = (HttpURLConnection) url.openConnection(); + connection = (HttpURLConnection)URLUtils.openConnection(url); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new IOException("Error cancelling token: " + connection.getResponseMessage()); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java Thu Apr 5 20:16:15 2012 @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.tools; import java.net.InetSocketAddress; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.ha.BadFencingConfigurationException; @@ -28,6 +29,8 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetUtils; +import com.google.common.base.Preconditions; + /** * One of the NN NameNodes acting as the target of an administrative command * (e.g. failover). @@ -35,14 +38,36 @@ import org.apache.hadoop.net.NetUtils; @InterfaceAudience.Private public class NNHAServiceTarget extends HAServiceTarget { + // Keys added to the fencing script environment + private static final String NAMESERVICE_ID_KEY = "nameserviceid"; + private static final String NAMENODE_ID_KEY = "namenodeid"; + private final InetSocketAddress addr; private NodeFencer fencer; private BadFencingConfigurationException fenceConfigError; + private final String nnId; + private final String nsId; public NNHAServiceTarget(HdfsConfiguration conf, String nsId, String nnId) { + Preconditions.checkNotNull(nnId); + + if (nsId == null) { + nsId = DFSUtil.getOnlyNameServiceIdOrNull(conf); + if (nsId == null) { + throw new IllegalArgumentException( + "Unable to determine the nameservice id."); + } + } + assert nsId != null; + + // Make a copy of the conf, and override configs based on the + // target node -- not the node we happen to be running on. + HdfsConfiguration targetConf = new HdfsConfiguration(conf); + NameNode.initializeGenericKeys(targetConf, nsId, nnId); + String serviceAddr = - DFSUtil.getNamenodeServiceAddr(conf, nsId, nnId); + DFSUtil.getNamenodeServiceAddr(targetConf, nsId, nnId); if (serviceAddr == null) { throw new IllegalArgumentException( "Unable to determine service address for namenode '" + nnId + "'"); @@ -50,10 +75,12 @@ public class NNHAServiceTarget extends H this.addr = NetUtils.createSocketAddr(serviceAddr, NameNode.DEFAULT_PORT); try { - this.fencer = NodeFencer.create(conf); + this.fencer = NodeFencer.create(targetConf); } catch (BadFencingConfigurationException e) { this.fenceConfigError = e; } + this.nnId = nnId; + this.nsId = nsId; } /** @@ -81,4 +108,19 @@ public class NNHAServiceTarget extends H return "NameNode at " + addr; } + public String getNameServiceId() { + return this.nsId; + } + + public String getNameNodeId() { + return this.nnId; + } + + @Override + protected void addFencingParameters(Map ret) { + super.addFencingParameters(ret); + + ret.put(NAMESERVICE_ID_KEY, getNameServiceId()); + ret.put(NAMENODE_ID_KEY, getNameNodeId()); + } } Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java Thu Apr 5 20:16:15 2012 @@ -17,104 +17,51 @@ */ package org.apache.hadoop.hdfs.tools.offlineEditsViewer; -import java.io.FileOutputStream; -import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; /** * BinaryEditsVisitor implements a binary EditsVisitor */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class BinaryEditsVisitor extends EditsVisitor { - final private DataOutputStream out; +public class BinaryEditsVisitor implements OfflineEditsVisitor { + final private EditLogFileOutputStream elfos; /** - * Create a processor that writes to a given file and - * reads using a given Tokenizer + * Create a processor that writes to a given file * * @param filename Name of file to write output to - * @param tokenizer Input tokenizer */ - public BinaryEditsVisitor(String filename, Tokenizer tokenizer) - throws IOException { - - this(filename, tokenizer, false); - } - - /** - * Create a processor that writes to a given file and reads using - * a given Tokenizer, may also print to screen - * - * @param filename Name of file to write output to - * @param tokenizer Input tokenizer - * @param printToScreen Mirror output to screen? (ignored for binary) - */ - public BinaryEditsVisitor(String filename, - Tokenizer tokenizer, - boolean printToScreen) throws IOException { - - super(tokenizer); - out = new DataOutputStream(new FileOutputStream(filename)); + public BinaryEditsVisitor(String outputName) throws IOException { + this.elfos = new EditLogFileOutputStream(new File(outputName), 0); + elfos.create(); } /** * Start the visitor (initialization) */ @Override - void start() throws IOException { - // nothing to do for binary format + public void start(int version) throws IOException { } /** * Finish the visitor */ @Override - void finish() throws IOException { - close(); - } - - /** - * Finish the visitor and indicate an error - */ - @Override - void finishAbnormally() throws IOException { - System.err.println("Error processing EditLog file. Exiting."); - close(); - } - - /** - * Close output stream and prevent further writing - */ - private void close() throws IOException { - out.close(); - } - - /** - * Visit a enclosing element (element that has other elements in it) - */ - @Override - void visitEnclosingElement(Tokenizer.Token value) throws IOException { - // nothing to do for binary format + public void close(Throwable error) throws IOException { + elfos.setReadyToFlush(); + elfos.flushAndSync(); + elfos.close(); } - /** - * End of eclosing element - */ - @Override - void leaveEnclosingElement() throws IOException { - // nothing to do for binary format - } - - /** - * Visit a Token - */ @Override - Tokenizer.Token visit(Tokenizer.Token value) throws IOException { - value.toBinary(out); - return value; + public void visitOp(FSEditLogOp op) throws IOException { + elfos.write(op); } -} +} \ No newline at end of file Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java Thu Apr 5 20:16:15 2012 @@ -18,12 +18,16 @@ package org.apache.hadoop.hdfs.tools.offlineEditsViewer; import java.io.EOFException; +import java.io.File; import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; +import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; +import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -33,6 +37,7 @@ import org.apache.commons.cli.OptionBuil import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; +import org.xml.sax.SAXParseException; /** * This class implements an offline edits viewer, tool that @@ -42,29 +47,9 @@ import org.apache.commons.cli.PosixParse @InterfaceStability.Unstable public class OfflineEditsViewer extends Configured implements Tool { - private EditsLoader editsLoader; private final static String defaultProcessor = "xml"; /** - * Set editsLoader - * - * @param editsLoader EditsLoader - */ - private void setEditsLoader(EditsLoader editsLoader) { - this.editsLoader = editsLoader; - } - - /** - * Process EditLog file. - * - * @param visitor use this visitor to process the file - */ - public void go(EditsVisitor visitor) throws IOException { - setEditsLoader(EditsLoader.LoaderFactory.getLoader(visitor)); - editsLoader.loadEdits(); - } - - /** * Print help. */ private void printHelp() { @@ -90,6 +75,9 @@ public class OfflineEditsViewer extends " format), stats (prints statistics about\n" + " edits file)\n" + "-h,--help Display usage information and exit\n" + + "-f,--fix-txids Renumber the transaction IDs in the input,\n" + + " so that there are no gaps or invalid " + + " transaction IDs.\n" + "-v,--verbose More verbose output, prints the input and\n" + " output filenames, for processors that write\n" + " to a file, also output to screen. On large\n" + @@ -124,11 +112,48 @@ public class OfflineEditsViewer extends options.addOption("p", "processor", true, ""); options.addOption("v", "verbose", false, ""); + options.addOption("f", "fix-txids", false, ""); options.addOption("h", "help", false, ""); return options; } + /** Process an edit log using the chosen processor or visitor. + * + * @param inputFilename The file to process + * @param outputFilename The output file name + * @param processor If visitor is null, the processor to use + * @param visitor If non-null, the visitor to use. + * + * @return 0 on success; error code otherwise + */ + public int go(String inputFileName, String outputFileName, String processor, + boolean printToScreen, boolean fixTxIds, OfflineEditsVisitor visitor) + { + if (printToScreen) { + System.out.println("input [" + inputFileName + "]"); + System.out.println("output [" + outputFileName + "]"); + } + try { + if (visitor == null) { + visitor = OfflineEditsVisitorFactory.getEditsVisitor( + outputFileName, processor, printToScreen); + } + boolean xmlInput = inputFileName.endsWith(".xml"); + OfflineEditsLoader loader = OfflineEditsLoaderFactory. + createLoader(visitor, inputFileName, xmlInput); + if (fixTxIds) { + loader.setFixTxIds(); + } + loader.loadEdits(); + } catch(Exception e) { + System.err.println("Encountered exception. Exiting: " + e.getMessage()); + e.printStackTrace(System.err); + return -1; + } + return 0; + } + /** * Main entry point for ToolRunner (see ToolRunner docs) * @@ -137,17 +162,13 @@ public class OfflineEditsViewer extends */ @Override public int run(String[] argv) throws Exception { - int exitCode = 0; - Options options = buildOptions(); if(argv.length == 0) { printHelp(); return -1; } - CommandLineParser parser = new PosixParser(); CommandLine cmd; - try { cmd = parser.parse(options, argv); } catch (ParseException e) { @@ -156,37 +177,20 @@ public class OfflineEditsViewer extends printHelp(); return -1; } - if(cmd.hasOption("h")) { // print help and exit printHelp(); return -1; } - - boolean printToScreen = false; - String inputFilenameArg = cmd.getOptionValue("i"); - String outputFilenameArg = cmd.getOptionValue("o"); - String processor = cmd.getOptionValue("p"); - if(processor == null) { processor = defaultProcessor; } - - if(cmd.hasOption("v")) { // print output to screen too - printToScreen = true; - System.out.println("input [" + inputFilenameArg + "]"); - System.out.println("output [" + outputFilenameArg + "]"); + String inputFileName = cmd.getOptionValue("i"); + String outputFileName = cmd.getOptionValue("o"); + String processor = cmd.getOptionValue("p"); + if(processor == null) { + processor = defaultProcessor; } - - try { - go(EditsVisitorFactory.getEditsVisitor( - outputFilenameArg, - processor, - TokenizerFactory.getTokenizer(inputFilenameArg), - printToScreen)); - } catch (EOFException e) { - System.err.println("Input file ended unexpectedly. Exiting"); - } catch(IOException e) { - System.err.println("Encountered exception. Exiting: " + e.getMessage()); - } - - return exitCode; + boolean printToScreen = cmd.hasOption("v"); + boolean fixTxIds = cmd.hasOption("f"); + return go(inputFileName, outputFileName, processor, + printToScreen, fixTxIds, null); } /** Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java Thu Apr 5 20:16:15 2012 @@ -19,12 +19,15 @@ package org.apache.hadoop.hdfs.tools.off import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; import java.util.Map; import java.util.HashMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; /** @@ -34,26 +37,14 @@ import org.apache.hadoop.hdfs.server.nam */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class StatisticsEditsVisitor extends EditsVisitor { - private boolean printToScreen = false; - private boolean okToWrite = false; - final private FileWriter fw; +public class StatisticsEditsVisitor implements OfflineEditsVisitor { + final private PrintStream out; - public final Map opCodeCount = + private int version = -1; + private final Map opCodeCount = new HashMap(); /** - * Create a processor that writes to the file named. - * - * @param filename Name of file to write output to - */ - public StatisticsEditsVisitor(String filename, Tokenizer tokenizer) - throws IOException { - - this(filename, tokenizer, false); - } - - /** * Create a processor that writes to the file named and may or may not * also output to the screen, as specified. * @@ -61,103 +52,29 @@ public class StatisticsEditsVisitor exte * @param tokenizer Input tokenizer * @param printToScreen Mirror output to screen? */ - public StatisticsEditsVisitor(String filename, - Tokenizer tokenizer, - boolean printToScreen) throws IOException { - - super(tokenizer); - this.printToScreen = printToScreen; - fw = new FileWriter(filename); - okToWrite = true; + public StatisticsEditsVisitor(OutputStream out) throws IOException { + this.out = new PrintStream(out); } - /** - * Start the visitor (initialization) - */ + /** Start the visitor */ @Override - void start() throws IOException { - // nothing to do + public void start(int version) throws IOException { + this.version = version; } - /* (non-Javadoc) - * @see org.apache.hadoop.hdfs.tools.offlineEditsViewer.EditsVisitor#finish() - */ - @Override - void finish() throws IOException { - write(getStatisticsString()); - close(); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.hdfs.tools.offlineEditsViewer.EditsVisitor#finishAbnormally() - */ - @Override - void finishAbnormally() throws IOException { - close(); - } - - /** - * Close output stream and prevent further writing - */ - private void close() throws IOException { - fw.close(); - okToWrite = false; - } - - /** - * Visit a enclosing element (element that has other elements in it) - */ - @Override - void visitEnclosingElement(Tokenizer.Token value) throws IOException { - // nothing to do - } - - /** - * End of eclosing element - */ + /** Close the visitor */ @Override - void leaveEnclosingElement() throws IOException { - // nothing to do - } - - /** - * Visit a Token, calculate statistics - * - * @param value a Token to visit - */ - @Override - Tokenizer.Token visit(Tokenizer.Token value) throws IOException { - // count the opCodes - if(value.getEditsElement() == EditsElement.OPCODE) { - if(value instanceof Tokenizer.ByteToken) { - incrementOpCodeCount( - FSEditLogOpCodes.fromByte(((Tokenizer.ByteToken)value).value)); - } else { - throw new IOException("Token for EditsElement.OPCODE should be " + - "of type Tokenizer.ByteToken, not " + value.getClass()); - } + public void close(Throwable error) throws IOException { + out.print(getStatisticsString()); + if (error != null) { + out.print("EXITING ON ERROR: " + error.toString() + "\n"); } - return value; + out.close(); } - /** - * Write parameter to output file (and possibly screen). - * - * @param toWrite Text to write to file - */ - protected void write(String toWrite) throws IOException { - if(!okToWrite) - throw new IOException("file not open for writing."); - - if(printToScreen) - System.out.print(toWrite); - - try { - fw.write(toWrite); - } catch (IOException e) { - okToWrite = false; - throw e; - } + @Override + public void visitOp(FSEditLogOp op) throws IOException { + incrementOpCodeCount(op.opCode); } /** @@ -189,13 +106,16 @@ public class StatisticsEditsVisitor exte */ public String getStatisticsString() { StringBuffer sb = new StringBuffer(); + sb.append(String.format( + " %-30.30s : %d%n", + "VERSION", version)); for(FSEditLogOpCodes opCode : FSEditLogOpCodes.values()) { sb.append(String.format( " %-30.30s (%3d): %d%n", - opCode, + opCode.toString(), opCode.getOpCode(), opCodeCount.get(opCode))); } return sb.toString(); } -} +} \ No newline at end of file Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java Thu Apr 5 20:16:15 2012 @@ -18,12 +18,19 @@ package org.apache.hadoop.hdfs.tools.offlineEditsViewer; import java.io.IOException; -import java.util.LinkedList; +import java.io.OutputStream; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.DepthCounter; +import org.apache.hadoop.hdfs.util.XMLUtils; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.AttributesImpl; + +import com.sun.org.apache.xml.internal.serialize.OutputFormat; +import com.sun.org.apache.xml.internal.serialize.XMLSerializer; /** * An XmlEditsVisitor walks over an EditLog structure and writes out @@ -31,140 +38,85 @@ import org.apache.hadoop.classification. */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class XmlEditsVisitor extends TextEditsVisitor { - final private LinkedList tagQ = - new LinkedList(); - - final private DepthCounter depthCounter = new DepthCounter(); +public class XmlEditsVisitor implements OfflineEditsVisitor { + private OutputStream out; + private ContentHandler contentHandler; /** * Create a processor that writes to the file named and may or may not * also output to the screen, as specified. * * @param filename Name of file to write output to - * @param tokenizer Input tokenizer + * @param printToScreen Mirror output to screen? */ - public XmlEditsVisitor(String filename, Tokenizer tokenizer) - throws IOException { - - super(filename, tokenizer, false); - } - - /** - * Create a processor that writes to the file named and may or may not - * also output to the screen, as specified. - * - * @param filename Name of file to write output to - * @param tokenizer Input tokenizer - * @param printToScreen Mirror output to screen? (ignored for binary) - */ - public XmlEditsVisitor(String filename, - Tokenizer tokenizer, - boolean printToScreen) throws IOException { - - super(filename, tokenizer, printToScreen); + public XmlEditsVisitor(OutputStream out) + throws IOException { + this.out = out; + OutputFormat outFormat = new OutputFormat("XML", "UTF-8", true); + outFormat.setIndenting(true); + outFormat.setIndent(2); + outFormat.setDoctype(null, null); + XMLSerializer serializer = new XMLSerializer(out, outFormat); + contentHandler = serializer.asContentHandler(); + try { + contentHandler.startDocument(); + contentHandler.startElement("", "", "EDITS", new AttributesImpl()); + } catch (SAXException e) { + throw new IOException("SAX error: " + e.getMessage()); + } } /** * Start visitor (initialization) */ @Override - void start() throws IOException { - write("\n"); - } - - /** - * Finish visitor - */ - @Override - void finish() throws IOException { - super.finish(); - } - - /** - * Finish with error - */ - @Override - void finishAbnormally() throws IOException { - write("\n\n"); - super.finishAbnormally(); - } - - /** - * Visit a Token - * - * @param value a Token to visit - */ - @Override - Tokenizer.Token visit(Tokenizer.Token value) throws IOException { - writeTag(value.getEditsElement().toString(), value.toString()); - return value; + public void start(int version) throws IOException { + try { + contentHandler.startElement("", "", "EDITS_VERSION", new AttributesImpl()); + StringBuilder bld = new StringBuilder(); + bld.append(version); + addString(bld.toString()); + contentHandler.endElement("", "", "EDITS_VERSION"); + } + catch (SAXException e) { + throw new IOException("SAX error: " + e.getMessage()); + } } - /** - * Visit an enclosing element (element that cntains other elements) - * - * @param value a Token to visit - */ - @Override - void visitEnclosingElement(Tokenizer.Token value) throws IOException { - printIndents(); - write("<" + value.getEditsElement().toString() + ">\n"); - tagQ.push(value.getEditsElement()); - depthCounter.incLevel(); + public void addString(String str) throws SAXException { + int slen = str.length(); + char arr[] = new char[slen]; + str.getChars(0, slen, arr, 0); + contentHandler.characters(arr, 0, slen); } - + /** - * Leave enclosing element + * Finish visitor */ @Override - void leaveEnclosingElement() throws IOException { - depthCounter.decLevel(); - if(tagQ.size() == 0) - throw new IOException("Tried to exit non-existent enclosing element " + - "in EditLog file"); - - EditsElement element = tagQ.pop(); - printIndents(); - write("\n"); - } - - /** - * Write an XML tag - * - * @param tag a tag name - * @param value a tag value - */ - private void writeTag(String tag, String value) throws IOException { - printIndents(); - if(value.length() > 0) { - write("<" + tag + ">" + value + "\n"); - } else { - write("<" + tag + "/>\n"); + public void close(Throwable error) throws IOException { + try { + contentHandler.endElement("", "", "EDITS"); + if (error != null) { + String msg = error.getMessage(); + XMLUtils.addSaxString(contentHandler, "ERROR", + (msg == null) ? "null" : msg); + } + contentHandler.endDocument(); } + catch (SAXException e) { + throw new IOException("SAX error: " + e.getMessage()); + } + out.close(); } - // prepared values that printIndents is likely to use - final private static String [] indents = { - "", - " ", - " ", - " ", - " ", - " ", - " " }; - - /** - * Prints the leading spaces based on depth level - */ - private void printIndents() throws IOException { + @Override + public void visitOp(FSEditLogOp op) throws IOException { try { - write(indents[depthCounter.getLevel()]); - } catch (IndexOutOfBoundsException e) { - // unlikely needed so can be slow - for(int i = 0; i < depthCounter.getLevel(); i++) - write(" "); + op.outputToXml(contentHandler); + } + catch (SAXException e) { + throw new IOException("SAX error: " + e.getMessage()); } - } } Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Thu Apr 5 20:16:15 2012 @@ -513,18 +513,13 @@ public class JsonUtil { final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes")); final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes)); - final int bytesPerCRC = in.readInt(); - final long crcPerBlock = in.readLong(); - final MD5Hash md5 = MD5Hash.read(in); - final MD5MD5CRC32FileChecksum checksum = new MD5MD5CRC32FileChecksum( - bytesPerCRC, crcPerBlock, md5); + final MD5MD5CRC32FileChecksum checksum = new MD5MD5CRC32FileChecksum(); + checksum.readFields(in); //check algorithm name - final String alg = "MD5-of-" + crcPerBlock + "MD5-of-" + bytesPerCRC + "CRC32"; - if (!alg.equals(algorithm)) { - throw new IOException("Algorithm not matched: algorithm=" + algorithm - + ", crcPerBlock=" + crcPerBlock - + ", bytesPerCRC=" + bytesPerCRC); + if (!checksum.getAlgorithmName().equals(algorithm)) { + throw new IOException("Algorithm not matched. Expected " + algorithm + + ", Received " + checksum.getAlgorithmName()); } //check length if (length != checksum.getLength()) { @@ -534,4 +529,4 @@ public class JsonUtil { return checksum; } -} \ No newline at end of file +} Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1308633-1310044 Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1308633-1310044 Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1308633-1310044 Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1308633-1310044 Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1308633-1310044 Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Thu Apr 5 20:16:15 2012 @@ -731,7 +731,9 @@ public class MiniDFSCluster { Preconditions.checkArgument(!dstDir.equals(srcDir)); File dstDirF = new File(dstDir); if (dstDirF.exists()) { - Files.deleteRecursively(dstDirF); + if (!FileUtil.fullyDelete(dstDirF)) { + throw new IOException("Unable to delete: " + dstDirF); + } } LOG.info("Copying namedir from primary node dir " + srcDir + " to " + dstDir); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Thu Apr 5 20:16:15 2012 @@ -179,6 +179,17 @@ public class TestDFSUtil { assertEquals("nn1", it.next().toString()); assertEquals("nn2", it.next().toString()); } + + @Test + public void testGetOnlyNameServiceIdOrNull() { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFS_FEDERATION_NAMESERVICES, "ns1,ns2"); + assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf)); + conf.set(DFS_FEDERATION_NAMESERVICES, ""); + assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf)); + conf.set(DFS_FEDERATION_NAMESERVICES, "ns1"); + assertEquals("ns1", DFSUtil.getOnlyNameServiceIdOrNull(conf)); + } /** * Test for {@link DFSUtil#getNNServiceRpcAddresses(Configuration)} Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Thu Apr 5 20:16:15 2012 @@ -23,8 +23,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.util.Daemon; @@ -143,8 +143,7 @@ public class BlockManagerTestUtil { * {@link DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY} to * a high value to ensure that all work is calculated. */ - public static int computeAllPendingWork(BlockManager bm) - throws IOException { + public static int computeAllPendingWork(BlockManager bm) { int work = computeInvalidationWork(bm); work += bm.computeReplicationWork(Integer.MAX_VALUE); return work; Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Thu Apr 5 20:16:15 2012 @@ -37,6 +37,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -181,7 +182,9 @@ public abstract class FSImageTestUtil { public static FSEditLog createStandaloneEditLog(File logDir) throws IOException { assertTrue(logDir.mkdirs() || logDir.exists()); - Files.deleteDirectoryContents(logDir); + if (!FileUtil.fullyDeleteContents(logDir)) { + throw new IOException("Unable to delete contents of " + logDir); + } NNStorage storage = Mockito.mock(NNStorage.class); StorageDirectory sd = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Thu Apr 5 20:16:15 2012 @@ -179,12 +179,22 @@ public class TestBackupNode { // do some edits assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down"))); - + // start a new backup node backup = startBackupNode(conf, StartupOption.BACKUP, 1); testBNInSync(cluster, backup, 4); assertNotNull(backup.getNamesystem().getFileInfo("/edit-while-bn-down", false)); + + // Trigger an unclean shutdown of the backup node. Backup node will not + // unregister from the active when this is done simulating a node crash. + backup.stop(false); + + // do some edits on the active. This should go through without failing. + // This will verify that active is still up and can add entries to + // master editlog. + assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down-2"))); + } finally { LOG.info("Shutting down..."); if (backup != null) backup.stop(); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java Thu Apr 5 20:16:15 2012 @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.GenericTestUtils; /** * This class tests the validation of the configuration object when passed @@ -72,6 +74,7 @@ public class TestValidateConfigurationSe conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:9000"); DFSTestUtil.formatNameNode(conf); NameNode nameNode = new NameNode(conf); // should be OK! + nameNode.stop(); } /** @@ -82,16 +85,30 @@ public class TestValidateConfigurationSe public void testGenericKeysForNameNodeFormat() throws IOException { Configuration conf = new HdfsConfiguration(); - FileSystem.setDefaultUri(conf, "hdfs://localhost:8070"); + + // Set ephemeral ports + conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, + "127.0.0.1:0"); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, + "127.0.0.1:0"); + conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1"); - String nameDir = System.getProperty("java.io.tmpdir") + "/test.dfs.name"; - File dir = new File(nameDir); + + // Set a nameservice-specific configuration for name dir + File dir = new File(MiniDFSCluster.getBaseDirectory(), + "testGenericKeysForNameNodeFormat"); if (dir.exists()) { FileUtil.fullyDelete(dir); } - conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + ".ns1", nameDir); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + ".ns1", + dir.getAbsolutePath()); + + // Format and verify the right dir is formatted. DFSTestUtil.formatNameNode(conf); + GenericTestUtils.assertExists(dir); + + // Ensure that the same dir is picked up by the running NN NameNode nameNode = new NameNode(conf); - FileUtil.fullyDelete(dir); + nameNode.stop(); } } Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java Thu Apr 5 20:16:15 2012 @@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; -import org.codehaus.jackson.sym.NameN; import org.junit.After; import org.junit.Before; import org.junit.Test; Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java Thu Apr 5 20:16:15 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.tools; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.PrintStream; @@ -41,6 +42,7 @@ import org.junit.Test; import com.google.common.base.Charsets; import com.google.common.base.Joiner; +import com.google.common.io.Files; /** * Tests for HAAdmin command with {@link MiniDFSCluster} set up in HA mode. @@ -59,6 +61,8 @@ public class TestDFSHAAdminMiniCluster { private String errOutput; + private int nn1Port; + @Before public void setup() throws IOException { conf = new Configuration(); @@ -69,6 +73,8 @@ public class TestDFSHAAdminMiniCluster { tool.setConf(conf); tool.setErrOut(new PrintStream(errOutBytes)); cluster.waitActive(); + + nn1Port = cluster.getNameNodePort(0); } @After @@ -124,9 +130,17 @@ public class TestDFSHAAdminMiniCluster { public void testFencer() throws Exception { // Test failover with no fencer assertEquals(-1, runTool("-failover", "nn1", "nn2")); - + + // Set up fencer to write info about the fencing target into a + // tmp file, so we can verify that the args were substituted right + File tmpFile = File.createTempFile("testFencer", ".txt"); + tmpFile.deleteOnExit(); + conf.set(NodeFencer.CONF_METHODS_KEY, + "shell(echo -n $target_nameserviceid.$target_namenodeid " + + "$target_port $dfs_ha_namenode_id > " + + tmpFile.getAbsolutePath() + ")"); + // Test failover with fencer - conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)"); tool.setConf(conf); assertEquals(0, runTool("-transitionToActive", "nn1")); assertEquals(0, runTool("-failover", "nn1", "nn2")); @@ -134,21 +148,36 @@ public class TestDFSHAAdminMiniCluster { // Test failover with fencer and nameservice assertEquals(0, runTool("-ns", "minidfs-ns", "-failover", "nn2", "nn1")); + // Fencer has not run yet, since none of the above required fencing + assertEquals("", Files.toString(tmpFile, Charsets.UTF_8)); + // Test failover with fencer and forcefence option assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence")); - + + // The fence script should run with the configuration from the target + // node, rather than the configuration from the fencing node + assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", + Files.toString(tmpFile, Charsets.UTF_8)); + tmpFile.delete(); + // Test failover with forceactive option assertEquals(0, runTool("-failover", "nn2", "nn1", "--forceactive")); + + // Fencing should not occur, since it was graceful + assertFalse(tmpFile.exists()); + // Test failover with not fencer and forcefence option conf.unset(NodeFencer.CONF_METHODS_KEY); tool.setConf(conf); assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence")); - + assertFalse(tmpFile.exists()); + // Test failover with bad fencer and forcefence option conf.set(NodeFencer.CONF_METHODS_KEY, "foobar!"); tool.setConf(conf); assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence")); + assertFalse(tmpFile.exists()); // Test failover with force fence listed before the other arguments conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)"); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1310048&r1=1310047&r2=1310048&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java Thu Apr 5 20:16:15 2012 @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.tools.offlineEditsViewer; +import java.io.FileOutputStream; import java.io.IOException; import java.io.File; import java.nio.ByteBuffer; @@ -33,8 +34,6 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer; -import org.apache.hadoop.hdfs.tools.offlineEditsViewer.TokenizerFactory; -import org.apache.hadoop.hdfs.tools.offlineEditsViewer.EditsVisitorFactory; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper; @@ -158,11 +157,8 @@ public class TestOfflineEditsViewer { LOG.info("Running oev [" + inFilename + "] [" + outFilename + "]"); OfflineEditsViewer oev = new OfflineEditsViewer(); - oev.go( EditsVisitorFactory.getEditsVisitor( - outFilename, - processor, - TokenizerFactory.getTokenizer(inFilename), - false)); + if (oev.go(inFilename, outFilename, processor, true, false, null) != 0) + throw new RuntimeException("oev failed"); } /** @@ -173,14 +169,11 @@ public class TestOfflineEditsViewer { */ private boolean hasAllOpCodes(String inFilename) throws IOException { String outFilename = inFilename + ".stats"; - StatisticsEditsVisitor visitor = - (StatisticsEditsVisitor)EditsVisitorFactory.getEditsVisitor( - outFilename, - "stats", - TokenizerFactory.getTokenizer(inFilename), - false); + FileOutputStream fout = new FileOutputStream(outFilename); + StatisticsEditsVisitor visitor = new StatisticsEditsVisitor(fout); OfflineEditsViewer oev = new OfflineEditsViewer(); - oev.go(visitor); + if (oev.go(inFilename, outFilename, "stats", false, false, visitor) != 0) + return false; LOG.info("Statistics for " + inFilename + "\n" + visitor.getStatisticsString()); @@ -190,6 +183,8 @@ public class TestOfflineEditsViewer { if(obsoleteOpCodes.containsKey(opCode)) { continue; } + if (opCode == FSEditLogOpCodes.OP_INVALID) + continue; Long count = visitor.getStatistics().get(opCode); if((count == null) || (count == 0)) { hasAllOpCodes = false;