Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2E82A10D59 for ; Tue, 4 Mar 2014 23:54:37 +0000 (UTC) Received: (qmail 3779 invoked by uid 500); 4 Mar 2014 23:54:13 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 3710 invoked by uid 500); 4 Mar 2014 23:54:12 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 3645 invoked by uid 99); 4 Mar 2014 23:54:08 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Mar 2014 23:54:08 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Mar 2014 23:53:58 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id DC32D2388A02; Tue, 4 Mar 2014 23:53:34 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1574259 [1/7] - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/ src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/ src/main/java/ sr... Date: Tue, 04 Mar 2014 23:53:29 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140304235334.DC32D2388A02@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Tue Mar 4 23:53:26 2014 New Revision: 1574259 URL: http://svn.apache.org/r1574259 Log: svn merge --reintegrate https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-5535 back to trunk. Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeLocalInfo.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeLocalInfo.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeException.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeException.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeInfo.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeInfo.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeStatus.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeStatus.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeStartupOptions.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeStartupOptions.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java - copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/IncorrectVersionException.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs:r1550130-1574256 Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Mar 4 23:53:26 2014 @@ -250,6 +250,148 @@ Trunk (Unreleased) HDFS-5794. Fix the inconsistency of layout version number of ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9) +BREAKDOWN OF HDFS-5535 ROLLING UPGRADE SUBTASKS AND RELATED JIRAS + + HDFS-5496. Make replication queue initialization asynchronous. (Vinay via + jing9) + + HDFS-5645. Support upgrade marker in editlog streams. (szetszwo) + + HDFS-5752. Add a new DFSAdmin command to query, start and finalize rolling + upgrade. (szetszwo) + + HDFS-5786. Support QUERY and FINALIZE actions of rolling upgrade. (szetszwo) + + HDFS-5753. Add new Namenode startup options for downgrade and rollback using + upgrade marker. (szetszwo) + + HDFS-5835. Add a new option for starting Namenode when rolling upgrade is + in progress. (szetszwo) + + HDFS-5754. Split LayoutVerion into NameNodeLayoutVersion and + DataNodeLayoutVersion. (Brandon Li via szetszwo) + + HDFS-5848. Add rolling upgrade status to heartbeat response. (szetszwo) + + HDFS-5890. Avoid NPE in Datanode heartbeat. (Vinay via brandonli) + + HDFS-5869. When starting rolling upgrade or NN restarts, NN should create + a checkpoint right before the upgrade marker. (szetszwo) + + HDFS-5874. Should not compare DataNode current layout version with that of + NameNode in DataStrorage. (brandonli) + + HDFS-5889. When starting rolling upgrade, create a fs image for rollback + so that the standby namenode can create checkpoints during upgrade. + (szetszwo & jing9) + + HDFS-5907. Add BlockPoolSliceStorage 'trash' to handle block deletions + during rolling upgrades. (Arpit Agarwal) + + HDFS-5494. Merge Protobuf-based-FSImage code from trunk - fix build + break after the merge. (Jing Zhao via Arpit Agarwal) + + HDFS-5585. Provide admin commands for data node upgrade (kihwal) + + HDFS-5920. Support rollback of rolling upgrade in NameNode and JournalNodes. + (jing9) + + HDFS-5945. Add rolling upgrade information to fsimage; and disallow upgrade + and rolling upgrade to be started simultaneously. (szetszwo & jing9) + + HDFS-5966. Fix rollback of rolling upgrade in NameNode HA setup. (jing9 + via szetszwo) + + HDFS-5974. Fix compilation error, NameNodeLayoutVersion and + DataNodeLayoutVersion after merge from trunk. (szetszwo) + + HDFS-5963. TestRollingUpgrade#testSecondaryNameNode causes subsequent + tests to fail. (szetszwo via Arpit Agarwal) + + HDFS-5976. Create unit tests for downgrade and finalize rolling upgrade. + (Haohui Mai via Arpit Agarwal) + + HDFS-5980. Rollback does not need to load edits. (jing9 via szetszwo) + + HDFS-5984. Fix TestEditLog and TestStandbyCheckpoints. (jing9 via szetszwo) + + HDFS-5985. SimulatedFSDataset#disableAndPurgeTrashStorage should not throw + UnsupportedOperationException. (jing9 via kihwal) + + HDFS-5987. Fix findbugs warnings in Rolling Upgrade branch. (seztszwo via + Arpit Agarwal) + + HDFS-5992. Fix NPE in MD5FileUtils and update editsStored for + TestOfflineEditsViewer. (szetszwo) + + HDFS-5994. Fix TestDataNodeRollingUpgrade. (Arpit Agarwal via szetszwo) + + HDFS-5999. Do not create rollback fsimage when it already exists. (jing9) + + HDFS-6005. Simplify Datanode rollback and downgrade. (Suresh Srinivas via + Arpit Agarwal) + + HDFS-6004. Change DFSAdmin for rolling upgrade commands. (szetszwo via + Arpit Agarwal) + + HDFS-5583. Make DN send an OOB Ack on shutdown before restarting. (kihwal) + + HDFS-5778. Add rolling upgrade user document. (szetszwo) + + HDFS-6003. Add the new -rollingUpgrade startup option to the namenode + usage message. (Vinayakumar B via szetszwo) + + HDFS-6014. Fix findbug warnings introduced by HDFS-5583. (kihwal) + + HDFS-6015. Fix TestBlockRecovery + #testRaceBetweenReplicaRecoveryAndFinalizeBlock. (kihwal) + + HDFS-5924. Utilize OOB upgrade message processing for writes. (kihwal) + + HDFS-5498. Improve datanode startup time. (kihwal) + + HDFS-6000. Avoid saving namespace when starting rolling upgrade. (jing9) + + HDFS-6017. Query the status of rolling upgrade in the preparation stage in + TestRollingUpgrade and TestRollingUpgradeRollback. (Haohui Mai via + Arpit Agarwal) + + HDFS-6020. Fix the five findbugs warnings. (kihwal) + + HDFS-6019. Standby NN might not checkpoint when processing the rolling + upgrade marker. (Haohui Mai via jing9) + + HDFS-6023. Test whether the standby NN continues to checkpoint after the + prepare stage. (Haohui Mai via jing9) + + HDFS-6024. Test whether the NN will reject the downgrade if it has a + fsimage from a newer release. (Haohui Mai via jing9) + + HDFS-6026. Fix TestDFSUpgrade and TestDataNodeRollingUpgrade. + (jing9 via szetszwo) + + HDFS-6029. Secondary NN fails to checkpoint after -rollingUpgrade prepare. + (jing9) + + HDFS-6032. -rollingUpgrade query hits NPE after the NN restarts. (Haohui Mai + via jing9) + + HDFS-6031. Add back the "-rollingUpgrade started" namenode startup option; + otherwise, namenode cannot start when the layout version is changed. + (szetszwo) + + HDFS-6034. Use DataNodeLayoutVersion for DN registration check and do not + verify layout version if there is a rolling upgrade in progress. (szetszwo) + + HDFS-6013. add rollingUpgrade information to latest UI. + (Vinayakumar B via wheat9) + + HDFS-6042. Fix rolling upgrade documentation and error messages. (szetszwo + via Arpit Agarwal) + + HDFS-6041. Downgrade/Finalize should rename the rollback image instead of + purging it. (jing9) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Tue Mar 4 23:53:26 2014 @@ -397,7 +397,7 @@ public class BookKeeperJournalManager im try { String znodePath = inprogressZNode(txId); EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath, - HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId); + HdfsConstants.NAMENODE_LAYOUT_VERSION, currentLedger.getId(), txId); /* Write the ledger metadata out to the inprogress ledger znode * This can fail if for some reason our write lock has * expired (@see WriteLock) and another process has managed to @@ -693,6 +693,11 @@ public class BookKeeperJournalManager im } @Override + public void discardSegments(long startTxId) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public void close() throws IOException { try { bkc.close(); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java Tue Mar 4 23:53:26 2014 @@ -68,7 +68,7 @@ public class TestBookKeeperEditLogStream lh.close(); EditLogLedgerMetadata metadata = new EditLogLedgerMetadata("/foobar", - HdfsConstants.LAYOUT_VERSION, lh.getId(), 0x1234); + HdfsConstants.NAMENODE_LAYOUT_VERSION, lh.getId(), 0x1234); try { new BookKeeperEditLogInputStream(lh, metadata, -1); fail("Shouldn't get this far, should have thrown"); @@ -77,7 +77,7 @@ public class TestBookKeeperEditLogStream } metadata = new EditLogLedgerMetadata("/foobar", - HdfsConstants.LAYOUT_VERSION, lh.getId(), 0x1234); + HdfsConstants.NAMENODE_LAYOUT_VERSION, lh.getId(), 0x1234); try { new BookKeeperEditLogInputStream(lh, metadata, 0); fail("Shouldn't get this far, should have thrown"); Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1550130-1574256 Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Tue Mar 4 23:53:26 2014 @@ -30,6 +30,8 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT; @@ -136,11 +138,13 @@ import org.apache.hadoop.hdfs.protocol.E import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; +import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; @@ -267,6 +271,7 @@ public class DFSClient implements java.i final int getFileBlockStorageLocationsTimeoutMs; final int retryTimesForGetLastBlockLength; final int retryIntervalForGetLastBlockLength; + final long datanodeRestartTimeout; final boolean useLegacyBlockReader; final boolean useLegacyBlockReaderLocal; @@ -417,6 +422,10 @@ public class DFSClient implements java.i shortCircuitSharedMemoryWatcherInterruptCheckMs = conf.getInt( DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT); + + datanodeRestartTimeout = conf.getLong( + DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, + DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000; } private DataChecksum.Type getChecksumType(Configuration conf) { @@ -2458,6 +2467,10 @@ public class DFSClient implements java.i namenode.finalizeUpgrade(); } + RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { + return namenode.rollingUpgrade(action); + } + /** */ @Deprecated Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Mar 4 23:53:26 2014 @@ -94,6 +94,10 @@ public class DFSConfigKeys extends Commo public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 3000; public static final String DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL = "dfs.client.write.exclude.nodes.cache.expiry.interval.millis"; public static final long DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10 * 60 * 1000; // 10 minutes, in ms + public static final String DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY = "dfs.client.datanode-restart.timeout"; + public static final long DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT = 30; + public static final String DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY = "dfs.datanode.restart.replica.expiration"; + public static final long DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT = 50; public static final String DFS_NAMENODE_BACKUP_ADDRESS_KEY = "dfs.namenode.backup.address"; public static final String DFS_NAMENODE_BACKUP_ADDRESS_DEFAULT = "localhost:50100"; public static final String DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY = "dfs.namenode.backup.http-address"; @@ -225,6 +229,8 @@ public class DFSConfigKeys extends Commo public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false; public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive"; public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 4000; + public static final String DFS_DATANODE_OOB_TIMEOUT_KEY = "dfs.datanode.oob.timeout-ms"; + public static final String DFS_DATANODE_OOB_TIMEOUT_DEFAULT = "1500,0,0,0"; // OOB_TYPE1, OOB_TYPE2, OOB_TYPE3, OOB_TYPE4 public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check"; public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true; @@ -416,7 +422,9 @@ public class DFSConfigKeys extends Commo public static final int DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT = 1000; public static final String DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY = "dfs.corruptfilesreturned.max"; public static final int DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED = 500; - + /* Maximum number of blocks to process for initializing replication queues */ + public static final String DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT = "dfs.block.misreplication.processing.limit"; + public static final int DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT = 10000; public static final String DFS_CLIENT_READ_SHORTCIRCUIT_KEY = "dfs.client.read.shortcircuit"; public static final boolean DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT = false; public static final String DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY = "dfs.client.read.shortcircuit.skip.checksum"; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Tue Mar 4 23:53:26 2014 @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.io.OutputStream; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; import java.nio.BufferOverflowException; @@ -335,6 +336,8 @@ public class DFSOutputStream extends FSO private String[] favoredNodes; volatile boolean hasError = false; volatile int errorIndex = -1; + volatile int restartingNodeIndex = -1; // Restarting node index + private long restartDeadline = 0; // Deadline of DN restart private BlockConstructionStage stage; // block construction stage private long bytesSent = 0; // number of bytes that've been sent @@ -471,7 +474,7 @@ public class DFSOutputStream extends FSO try { // process datanode IO errors if any boolean doSleep = false; - if (hasError && errorIndex>=0) { + if (hasError && (errorIndex >= 0 || restartingNodeIndex >= 0)) { doSleep = processDatanodeError(); } @@ -571,8 +574,12 @@ public class DFSOutputStream extends FSO blockStream.flush(); } catch (IOException e) { // HDFS-3398 treat primary DN is down since client is unable to - // write to primary DN - errorIndex = 0; + // write to primary DN. If a failed or restarting node has already + // been recorded by the responder, the following call will have no + // effect. Pipeline recovery can handle only one node error at a + // time. If the primary node fails again during the recovery, it + // will be taken out then. + tryMarkPrimaryDatanodeFailed(); throw e; } lastPacket = Time.now(); @@ -609,12 +616,16 @@ public class DFSOutputStream extends FSO Thread.sleep(artificialSlowdown); } } catch (Throwable e) { - DFSClient.LOG.warn("DataStreamer Exception", e); + // Log warning if there was a real error. + if (restartingNodeIndex == -1) { + DFSClient.LOG.warn("DataStreamer Exception", e); + } if (e instanceof IOException) { setLastException((IOException)e); } hasError = true; - if (errorIndex == -1) { // not a datanode error + if (errorIndex == -1 && restartingNodeIndex == -1) { + // Not a datanode issue streamerClosed = true; } } @@ -694,6 +705,65 @@ public class DFSOutputStream extends FSO } } + // The following synchronized methods are used whenever + // errorIndex or restartingNodeIndex is set. This is because + // check & set needs to be atomic. Simply reading variables + // does not require a synchronization. When responder is + // not running (e.g. during pipeline recovery), there is no + // need to use these methods. + + /** Set the error node index. Called by responder */ + synchronized void setErrorIndex(int idx) { + errorIndex = idx; + } + + /** Set the restarting node index. Called by responder */ + synchronized void setRestartingNodeIndex(int idx) { + restartingNodeIndex = idx; + // If the data streamer has already set the primary node + // bad, clear it. It is likely that the write failed due to + // the DN shutdown. Even if it was a real failure, the pipeline + // recovery will take care of it. + errorIndex = -1; + } + + /** + * This method is used when no explicit error report was received, + * but something failed. When the primary node is a suspect or + * unsure about the cause, the primary node is marked as failed. + */ + synchronized void tryMarkPrimaryDatanodeFailed() { + // There should be no existing error and no ongoing restart. + if ((errorIndex == -1) && (restartingNodeIndex == -1)) { + errorIndex = 0; + } + } + + /** + * Examine whether it is worth waiting for a node to restart. + * @param index the node index + */ + boolean shouldWaitForRestart(int index) { + // Only one node in the pipeline. + if (nodes.length == 1) { + return true; + } + + // Is it a local node? + InetAddress addr = null; + try { + addr = InetAddress.getByName(nodes[index].getIpAddr()); + } catch (java.net.UnknownHostException e) { + // we are passing an ip address. this should not happen. + assert false; + } + + if (addr != null && NetUtils.isLocalAddress(addr)) { + return true; + } + return false; + } + // // Processes responses from the datanodes. A packet is removed // from the ackQueue when its response arrives. @@ -727,8 +797,20 @@ public class DFSOutputStream extends FSO // processes response status from datanodes. for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { final Status reply = ack.getReply(i); + // Restart will not be treated differently unless it is + // the local node or the only one in the pipeline. + if (PipelineAck.isRestartOOBStatus(reply) && + shouldWaitForRestart(i)) { + restartDeadline = dfsClient.getConf().datanodeRestartTimeout + + Time.now(); + setRestartingNodeIndex(i); + String message = "A datanode is restarting: " + targets[i]; + DFSClient.LOG.info(message); + throw new IOException(message); + } + // node error if (reply != SUCCESS) { - errorIndex = i; // first bad datanode + setErrorIndex(i); // first bad datanode throw new IOException("Bad response " + reply + " for block " + block + " from datanode " + @@ -777,12 +859,16 @@ public class DFSOutputStream extends FSO setLastException((IOException)e); } hasError = true; - errorIndex = errorIndex==-1 ? 0 : errorIndex; + // If no explicit error report was received, mark the primary + // node as failed. + tryMarkPrimaryDatanodeFailed(); synchronized (dataQueue) { dataQueue.notifyAll(); } - DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception " - + " for block " + block, e); + if (restartingNodeIndex == -1) { + DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception " + + " for block " + block, e); + } responderClosed = true; } } @@ -1001,6 +1087,24 @@ public class DFSOutputStream extends FSO boolean success = false; long newGS = 0L; while (!success && !streamerClosed && dfsClient.clientRunning) { + // Sleep before reconnect if a dn is restarting. + // This process will be repeated until the deadline or the datanode + // starts back up. + if (restartingNodeIndex >= 0) { + // 4 seconds or the configured deadline period, whichever is shorter. + // This is the retry interval and recovery will be retried in this + // interval until timeout or success. + long delay = Math.min(dfsClient.getConf().datanodeRestartTimeout, + 4000L); + try { + Thread.sleep(delay); + } catch (InterruptedException ie) { + lastException.set(new IOException("Interrupted while waiting for " + + "datanode to restart. " + nodes[restartingNodeIndex])); + streamerClosed = true; + return false; + } + } boolean isRecovery = hasError; // remove bad datanode from list of datanodes. // If errorIndex was not set (i.e. appends), then do not remove @@ -1037,7 +1141,24 @@ public class DFSOutputStream extends FSO setPipeline(newnodes, newStorageIDs); - hasError = false; + // Just took care of a node error while waiting for a node restart + if (restartingNodeIndex >= 0) { + // If the error came from a node further away than the restarting + // node, the restart must have been complete. + if (errorIndex > restartingNodeIndex) { + restartingNodeIndex = -1; + } else if (errorIndex < restartingNodeIndex) { + // the node index has shifted. + restartingNodeIndex--; + } else { + // this shouldn't happen... + assert false; + } + } + + if (restartingNodeIndex == -1) { + hasError = false; + } lastException.set(null); errorIndex = -1; } @@ -1066,7 +1187,34 @@ public class DFSOutputStream extends FSO } else { success = createBlockOutputStream(nodes, newGS, isRecovery); } - } + + if (restartingNodeIndex >= 0) { + assert hasError == true; + // check errorIndex set above + if (errorIndex == restartingNodeIndex) { + // ignore, if came from the restarting node + errorIndex = -1; + } + // still within the deadline + if (Time.now() < restartDeadline) { + continue; // with in the deadline + } + // expired. declare the restarting node dead + restartDeadline = 0; + int expiredNodeIndex = restartingNodeIndex; + restartingNodeIndex = -1; + DFSClient.LOG.warn("Datanode did not restart in time: " + + nodes[expiredNodeIndex]); + // Mark the restarting node as failed. If there is any other failed + // node during the last pipeline construction attempt, it will not be + // overwritten/dropped. In this case, the restarting node will get + // excluded in the following attempt, if it still does not come up. + if (errorIndex == -1) { + errorIndex = expiredNodeIndex; + } + // From this point on, normal pipeline recovery applies. + } + } // while if (success) { // update pipeline at the namenode @@ -1144,6 +1292,7 @@ public class DFSOutputStream extends FSO } Status pipelineStatus = SUCCESS; String firstBadLink = ""; + boolean checkRestart = false; if (DFSClient.LOG.isDebugEnabled()) { for (int i = 0; i < nodes.length; i++) { DFSClient.LOG.debug("pipeline = " + nodes[i]); @@ -1192,6 +1341,16 @@ public class DFSOutputStream extends FSO pipelineStatus = resp.getStatus(); firstBadLink = resp.getFirstBadLink(); + // Got an restart OOB ack. + // If a node is already restarting, this status is not likely from + // the same node. If it is from a different node, it is not + // from the local datanode. Thus it is safe to treat this as a + // regular node error. + if (PipelineAck.isRestartOOBStatus(pipelineStatus) && + restartingNodeIndex == -1) { + checkRestart = true; + throw new IOException("A datanode is restarting."); + } if (pipelineStatus != SUCCESS) { if (pipelineStatus == Status.ERROR_ACCESS_TOKEN) { throw new InvalidBlockTokenException( @@ -1205,9 +1364,12 @@ public class DFSOutputStream extends FSO assert null == blockStream : "Previous blockStream unclosed"; blockStream = out; result = true; // success - + restartingNodeIndex = -1; + hasError = false; } catch (IOException ie) { - DFSClient.LOG.info("Exception in createBlockOutputStream", ie); + if (restartingNodeIndex == -1) { + DFSClient.LOG.info("Exception in createBlockOutputStream", ie); + } if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) { DFSClient.LOG.info("Will fetch a new encryption key and retry, " + "encryption key was invalid when connecting to " @@ -1230,8 +1392,18 @@ public class DFSOutputStream extends FSO } } } else { + assert checkRestart == false; errorIndex = 0; } + // Check whether there is a restart worth waiting for. + if (checkRestart && shouldWaitForRestart(errorIndex)) { + restartDeadline = dfsClient.getConf().datanodeRestartTimeout + + Time.now(); + restartingNodeIndex = errorIndex; + errorIndex = -1; + DFSClient.LOG.info("Waiting for the datanode to be restarted: " + + nodes[restartingNodeIndex]); + } hasError = true; setLastException(ie); result = false; // error Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Tue Mar 4 23:53:26 2014 @@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.client.Hdf import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -68,11 +69,12 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; @@ -993,6 +995,14 @@ public class DistributedFileSystem exten dfs.finalizeUpgrade(); } + /** + * Rolling upgrade: start/finalize/query. + */ + public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) + throws IOException { + return dfs.rollingUpgrade(action); + } + /* * Requests the namenode to dump data strcutures into specified * file. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java Tue Mar 4 23:53:26 2014 @@ -125,4 +125,22 @@ public interface ClientDatanodeProtocol */ HdfsBlocksMetadata getHdfsBlocksMetadata(String blockPoolId, long []blockIds, List> tokens) throws IOException; + + /** + * Shuts down a datanode. + * + * @param forUpgrade If true, data node does extra prep work before shutting + * down. The work includes advising clients to wait and saving + * certain states for quick restart. This should only be used when + * the stored data will remain the same during upgrade/restart. + * @throws IOException + */ + void shutdownDatanode(boolean forUpgrade) throws IOException; + + /** + * Obtains datanode info + * + * @return software/config version and uptime of the datanode + */ + DatanodeLocalInfo getDatanodeInfo() throws IOException; } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Tue Mar 4 23:53:26 2014 @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; @@ -764,6 +765,15 @@ public interface ClientProtocol { public void finalizeUpgrade() throws IOException; /** + * Rolling upgrade operations. + * @param action either query, start or finailze. + * @return rolling upgrade information. + */ + @Idempotent + public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) + throws IOException; + + /** * @return CorruptFileBlocks, containing a list of corrupt files (with * duplicates if there is more than one corrupt block in a file) * and a cookie Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java Tue Mar 4 23:53:26 2014 @@ -17,11 +17,16 @@ */ package org.apache.hadoop.hdfs.protocol; +import java.util.HashMap; +import java.util.Map; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; +import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; /************************************ @@ -79,6 +84,24 @@ public class HdfsConstants { SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET; } + public static enum RollingUpgradeAction { + QUERY, PREPARE, FINALIZE; + + private static final Map MAP + = new HashMap(); + static { + MAP.put("", QUERY); + for(RollingUpgradeAction a : values()) { + MAP.put(a.name(), a); + } + } + + /** Covert the given String to a RollingUpgradeAction. */ + public static RollingUpgradeAction fromString(String s) { + return MAP.get(s.toUpperCase()); + } + } + // type of the datanode report public static enum DatanodeReportType { ALL, LIVE, DEAD @@ -105,10 +128,18 @@ public class HdfsConstants { /** - * Please see {@link LayoutVersion} on adding new layout version. + * Current layout version for NameNode. + * Please see {@link NameNodeLayoutVersion.Feature} on adding new layout version. + */ + public static final int NAMENODE_LAYOUT_VERSION + = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION; + + /** + * Current layout version for DataNode. + * Please see {@link DataNodeLayoutVersion.Feature} on adding new layout version. */ - public static final int LAYOUT_VERSION = LayoutVersion - .getCurrentLayoutVersion(); + public static final int DATANODE_LAYOUT_VERSION + = DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION; /** * Path components that are reserved in HDFS. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Tue Mar 4 23:53:26 2014 @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hdfs.protocol; -import java.util.EnumSet; -import java.util.HashMap; +import java.util.Comparator; import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; import org.apache.hadoop.classification.InterfaceAudience; @@ -43,7 +44,6 @@ import org.apache.hadoop.classification. */ @InterfaceAudience.Private public class LayoutVersion { - /** * Version in which HDFS-2991 was fixed. This bug caused OP_ADD to * sometimes be skipped for append() calls. If we see such a case when @@ -54,19 +54,27 @@ public class LayoutVersion { public static final int BUGFIX_HDFS_2991_VERSION = -40; /** - * Enums for features that change the layout version. + * The interface to be implemented by NameNode and DataNode layout features + */ + public interface LayoutFeature { + public FeatureInfo getInfo(); + } + + /** + * Enums for features that change the layout version before rolling + * upgrade is supported. *

* To add a new layout version: *
    *
  • Define a new enum constant with a short enum name, the new layout version * and description of the added feature.
    • *
  • When adding a layout version with an ancestor that is not same as - * its immediate predecessor, use the constructor where a spacific ancestor + * its immediate predecessor, use the constructor where a specific ancestor * can be passed. *
    • *
*/ - public static enum Feature { + public static enum Feature implements LayoutFeature { NAMESPACE_QUOTA(-16, "Support for namespace quotas"), FILE_ACCESS_TIME(-17, "Support for access time on files"), DISKSPACE_QUOTA(-18, "Support for disk space quotas"), @@ -119,12 +127,8 @@ public class LayoutVersion { RESERVED_REL2_4_0(-54, -51, "Reserved for release 2.4.0", true, PROTOBUF_FORMAT, EXTENDED_ACL); - final int lv; - final int ancestorLV; - final String description; - final boolean reserved; - final Feature[] specialFeatures; - + private final FeatureInfo info; + /** * Feature that is added at layout version {@code lv} - 1. * @param lv new layout version with the addition of this feature @@ -140,16 +144,35 @@ public class LayoutVersion { * @param ancestorLV layout version from which the new lv is derived from. * @param description description of the feature * @param reserved true when this is a layout version reserved for previous - * verions + * version * @param features set of features that are to be enabled for this version */ Feature(final int lv, final int ancestorLV, final String description, boolean reserved, Feature... features) { + info = new FeatureInfo(lv, ancestorLV, description, reserved, features); + } + + @Override + public FeatureInfo getInfo() { + return info; + } + } + + /** Feature information. */ + public static class FeatureInfo { + private final int lv; + private final int ancestorLV; + private final String description; + private final boolean reserved; + private final LayoutFeature[] specialFeatures; + + public FeatureInfo(final int lv, final int ancestorLV, final String description, + boolean reserved, LayoutFeature... specialFeatures) { this.lv = lv; this.ancestorLV = ancestorLV; this.description = description; this.reserved = reserved; - specialFeatures = features; + this.specialFeatures = specialFeatures; } /** @@ -179,80 +202,91 @@ public class LayoutVersion { public boolean isReservedForOldRelease() { return reserved; } + + public LayoutFeature[] getSpecialFeatures() { + return specialFeatures; + } } - - // Build layout version and corresponding feature matrix - static final Map>map = - new HashMap>(); - - // Static initialization - static { - initMap(); + + static class LayoutFeatureComparator implements Comparator { + @Override + public int compare(LayoutFeature arg0, LayoutFeature arg1) { + return arg0.getInfo().getLayoutVersion() + - arg1.getInfo().getLayoutVersion(); + } } - - /** - * Initialize the map of a layout version and EnumSet of {@link Feature}s - * supported. - */ - private static void initMap() { + + public static void updateMap(Map> map, + LayoutFeature[] features) { // Go through all the enum constants and build a map of - // LayoutVersion <-> EnumSet of all supported features in that LayoutVersion - for (Feature f : Feature.values()) { - EnumSet ancestorSet = map.get(f.ancestorLV); + // LayoutVersion <-> Set of all supported features in that LayoutVersion + for (LayoutFeature f : features) { + final FeatureInfo info = f.getInfo(); + SortedSet ancestorSet = map.get(info.getAncestorLayoutVersion()); if (ancestorSet == null) { - ancestorSet = EnumSet.noneOf(Feature.class); // Empty enum set - map.put(f.ancestorLV, ancestorSet); + // Empty set + ancestorSet = new TreeSet(new LayoutFeatureComparator()); + map.put(info.getAncestorLayoutVersion(), ancestorSet); } - EnumSet featureSet = EnumSet.copyOf(ancestorSet); - if (f.specialFeatures != null) { - for (Feature specialFeature : f.specialFeatures) { + SortedSet featureSet = new TreeSet(ancestorSet); + if (info.getSpecialFeatures() != null) { + for (LayoutFeature specialFeature : info.getSpecialFeatures()) { featureSet.add(specialFeature); } } featureSet.add(f); - map.put(f.lv, featureSet); + map.put(info.getLayoutVersion(), featureSet); } } /** * Gets formatted string that describes {@link LayoutVersion} information. */ - public static String getString() { + public String getString(Map> map, + LayoutFeature[] values) { final StringBuilder buf = new StringBuilder(); buf.append("Feature List:\n"); - for (Feature f : Feature.values()) { + for (LayoutFeature f : values) { + final FeatureInfo info = f.getInfo(); buf.append(f).append(" introduced in layout version ") - .append(f.lv).append(" ("). - append(f.description).append(")\n"); + .append(info.getLayoutVersion()).append(" (") + .append(info.getDescription()).append(")\n"); } - + buf.append("\n\nLayoutVersion and supported features:\n"); - for (Feature f : Feature.values()) { - buf.append(f.lv).append(": ").append(map.get(f.lv)) - .append("\n"); + for (LayoutFeature f : values) { + final FeatureInfo info = f.getInfo(); + buf.append(info.getLayoutVersion()).append(": ") + .append(map.get(info.getLayoutVersion())).append("\n"); } return buf.toString(); } /** * Returns true if a given feature is supported in the given layout version + * @param map layout feature map * @param f Feature * @param lv LayoutVersion * @return true if {@code f} is supported in layout version {@code lv} */ - public static boolean supports(final Feature f, final int lv) { - final EnumSet set = map.get(lv); + public static boolean supports(Map> map, + final LayoutFeature f, final int lv) { + final SortedSet set = map.get(lv); return set != null && set.contains(f); } /** * Get the current layout version */ - public static int getCurrentLayoutVersion() { - Feature[] values = Feature.values(); - for (int i = values.length -1; i >= 0; i--) { - if (!values[i].isReservedForOldRelease()) { - return values[i].lv; + public static int getCurrentLayoutVersion(LayoutFeature[] features) { + return getLastNonReservedFeature(features).getInfo().getLayoutVersion(); + } + + static LayoutFeature getLastNonReservedFeature(LayoutFeature[] features) { + for (int i = features.length -1; i >= 0; i--) { + final FeatureInfo info = features[i].getInfo(); + if (!info.isReservedForOldRelease()) { + return features[i]; } } throw new AssertionError("All layout versions are reserved."); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java Tue Mar 4 23:53:26 2014 @@ -26,10 +26,12 @@ import java.util.Arrays; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_DEFAULT; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; - import com.google.protobuf.TextFormat; /** Pipeline Acknowledgment **/ @@ -38,6 +40,21 @@ import com.google.protobuf.TextFormat; public class PipelineAck { PipelineAckProto proto; public final static long UNKOWN_SEQNO = -2; + final static int OOB_START = Status.OOB_RESTART_VALUE; // the first OOB type + final static int OOB_END = Status.OOB_RESERVED3_VALUE; // the last OOB type + final static int NUM_OOB_TYPES = OOB_END - OOB_START + 1; + // place holder for timeout value of each OOB type + final static long[] OOB_TIMEOUT; + + static { + OOB_TIMEOUT = new long[NUM_OOB_TYPES]; + HdfsConfiguration conf = new HdfsConfiguration(); + String[] ele = conf.get(DFS_DATANODE_OOB_TIMEOUT_KEY, + DFS_DATANODE_OOB_TIMEOUT_DEFAULT).split(","); + for (int i = 0; i < NUM_OOB_TYPES; i++) { + OOB_TIMEOUT[i] = (i < ele.length) ? Long.valueOf(ele[i]) : 0; + } + } /** default constructor **/ public PipelineAck() { @@ -103,14 +120,57 @@ public class PipelineAck { * @return true if all statuses are SUCCESS */ public boolean isSuccess() { - for (DataTransferProtos.Status reply : proto.getStatusList()) { - if (reply != DataTransferProtos.Status.SUCCESS) { + for (Status reply : proto.getStatusList()) { + if (reply != Status.SUCCESS) { return false; } } return true; } - + + /** + * Returns the OOB status if this ack contains one. + * @return null if it is not an OOB ack. + */ + public Status getOOBStatus() { + // Normal data transfer acks will have a valid sequence number, so + // this will return right away in most cases. + if (getSeqno() != UNKOWN_SEQNO) { + return null; + } + for (Status reply : proto.getStatusList()) { + // The following check is valid because protobuf guarantees to + // preserve the ordering of enum elements. + if (reply.getNumber() >= OOB_START && reply.getNumber() <= OOB_END) { + return reply; + } + } + return null; + } + + /** + * Get the timeout to be used for transmitting the OOB type + * @return the timeout in milliseconds + */ + public static long getOOBTimeout(Status status) throws IOException { + int index = status.getNumber() - OOB_START; + if (index >= 0 && index < NUM_OOB_TYPES) { + return OOB_TIMEOUT[index]; + } + // Not an OOB. + throw new IOException("Not an OOB status: " + status); + } + + /** Get the Restart OOB ack status */ + public static Status getRestartOOBStatus() { + return Status.OOB_RESTART; + } + + /** return true if it is the restart OOB status code */ + public static boolean isRestartOOBStatus(Status st) { + return st.equals(Status.OOB_RESTART); + } + /**** Writable interface ****/ public void readFields(InputStream in) throws IOException { proto = PipelineAckProto.parseFrom(vintPrefixed(in)); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java Tue Mar 4 23:53:26 2014 @@ -30,6 +30,8 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto.Builder; @@ -37,10 +39,13 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ShutdownDatanodeRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ShutdownDatanodeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.VersionInfo; import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; @@ -59,6 +64,8 @@ public class ClientDatanodeProtocolServe RefreshNamenodesResponseProto.newBuilder().build(); private final static DeleteBlockPoolResponseProto DELETE_BLOCKPOOL_RESP = DeleteBlockPoolResponseProto.newBuilder().build(); + private final static ShutdownDatanodeResponseProto SHUTDOWN_DATANODE_RESP = + ShutdownDatanodeResponseProto.newBuilder().build(); private final ClientDatanodeProtocol impl; @@ -151,4 +158,28 @@ public class ClientDatanodeProtocolServe builder.addAllVolumeIndexes(resp.getVolumeIndexes()); return builder.build(); } + + @Override + public ShutdownDatanodeResponseProto shutdownDatanode( + RpcController unused, ShutdownDatanodeRequestProto request) + throws ServiceException { + try { + impl.shutdownDatanode(request.getForUpgrade()); + } catch (IOException e) { + throw new ServiceException(e); + } + return SHUTDOWN_DATANODE_RESP; + } + + public GetDatanodeInfoResponseProto getDatanodeInfo(RpcController unused, + GetDatanodeInfoRequestProto request) throws ServiceException { + GetDatanodeInfoResponseProto res; + try { + res = GetDatanodeInfoResponseProto.newBuilder() + .setLocalInfo(PBHelper.convert(impl.getDatanodeInfo())).build(); + } catch (IOException e) { + throw new ServiceException(e); + } + return res; + } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java?rev=1574259&r1=1574258&r2=1574259&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java Tue Mar 4 23:53:26 2014 @@ -35,16 +35,20 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ShutdownDatanodeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.ipc.ProtobufHelper; @@ -81,6 +85,8 @@ public class ClientDatanodeProtocolTrans private final ClientDatanodeProtocolPB rpcProxy; private final static RefreshNamenodesRequestProto VOID_REFRESH_NAMENODES = RefreshNamenodesRequestProto.newBuilder().build(); + private final static GetDatanodeInfoRequestProto VOID_GET_DATANODE_INFO = + GetDatanodeInfoRequestProto.newBuilder().build(); public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid, Configuration conf, int socketTimeout, boolean connectToDnViaHostname, @@ -253,4 +259,27 @@ public class ClientDatanodeProtocolTrans return new HdfsBlocksMetadata(blockPoolId, blockIds, volumeIds, volumeIndexes); } + + @Override + public void shutdownDatanode(boolean forUpgrade) throws IOException { + ShutdownDatanodeRequestProto request = ShutdownDatanodeRequestProto + .newBuilder().setForUpgrade(forUpgrade).build(); + try { + rpcProxy.shutdownDatanode(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public DatanodeLocalInfo getDatanodeInfo() throws IOException { + GetDatanodeInfoResponseProto response; + try { + response = rpcProxy.getDatanodeInfo(NULL_CONTROLLER, VOID_GET_DATANODE_INFO); + return PBHelper.convert(response.getLocalInfo()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + }