hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Date Tue, 25 Aug 2015 22:22:44 GMT
Merge branch 'trunk' into HDFS-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a070ee4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a070ee4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a070ee4

Branch: refs/heads/HDFS-7240
Commit: 8a070ee489127fdb9d7048b7dbcb7eae74ba539d
Parents: c785187 eee0d45
Author: Anu Engineer <aengineer@apache.org>
Authored: Tue Aug 25 15:20:49 2015 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Tue Aug 25 15:20:49 2015 -0700

----------------------------------------------------------------------
 BUILDING.txt                                    |  38 +-
 dev-support/docker/Dockerfile                   |   8 +
 dev-support/test-patch.sh                       |   6 +
 .../server/AuthenticationFilter.java            |  63 +-
 .../server/AuthenticationToken.java             |  12 +
 .../security/authentication/util/AuthToken.java |  35 +-
 .../server/TestAuthenticationFilter.java        | 163 +++-
 hadoop-common-project/hadoop-common/CHANGES.txt |  99 ++-
 hadoop-common-project/hadoop-common/pom.xml     |  33 +
 .../hadoop-common/src/main/bin/hadoop           |   8 +-
 .../hadoop-common/src/main/bin/hadoop-config.sh |  73 +-
 .../src/main/bin/hadoop-functions.sh            | 369 +++++++--
 .../hadoop-common/src/main/bin/slaves.sh        |   3 +-
 .../src/main/conf/log4j.properties              |  13 +
 .../org/apache/hadoop/conf/Configuration.java   |   9 +-
 .../org/apache/hadoop/fs/BlockLocation.java     |  55 +-
 .../fs/CommonConfigurationKeysPublic.java       |   5 +
 .../java/org/apache/hadoop/fs/CreateFlag.java   |   2 +-
 .../apache/hadoop/fs/DelegateToFileSystem.java  |  19 +-
 .../main/java/org/apache/hadoop/fs/FsShell.java |  19 +-
 .../apache/hadoop/fs/RawLocalFileSystem.java    |  36 +-
 .../apache/hadoop/fs/TrashPolicyDefault.java    |  11 +-
 .../hadoop/fs/permission/UmaskParser.java       |   2 +-
 .../org/apache/hadoop/io/ReadaheadPool.java     |   4 +-
 .../org/apache/hadoop/io/nativeio/NativeIO.java |  83 +-
 .../apache/hadoop/ipc/ProtobufRpcEngine.java    |   5 +-
 .../main/java/org/apache/hadoop/ipc/Server.java |  60 ++
 .../apache/hadoop/ipc/WritableRpcEngine.java    |   3 +
 .../apache/hadoop/ipc/metrics/RpcMetrics.java   |  54 +-
 .../metrics2/impl/MetricsSinkAdapter.java       |   6 +-
 .../apache/hadoop/metrics2/lib/MutableStat.java |   7 +-
 .../org/apache/hadoop/metrics2/util/MBeans.java |  37 +-
 .../org/apache/hadoop/net/NetworkTopology.java  |  38 +-
 .../apache/hadoop/security/SaslPlainServer.java |   2 +-
 .../org/apache/hadoop/util/HostsFileReader.java |   7 +-
 .../org/apache/hadoop/util/JvmPauseMonitor.java |  12 +-
 .../java/org/apache/hadoop/util/RunJar.java     |   3 +-
 .../main/java/org/apache/hadoop/util/Shell.java |  11 +-
 .../org/apache/hadoop/util/StringUtils.java     |  29 +-
 .../org/apache/hadoop/io/nativeio/NativeIO.c    | 149 ++--
 .../src/main/resources/core-default.xml         |  27 +
 .../src/site/markdown/HttpAuthentication.md     |   8 +-
 .../hadoop-common/src/site/markdown/Metrics.md  |   4 +-
 .../src/site/markdown/SingleCluster.md.vm       |   2 +-
 .../apache/hadoop/crypto/TestCryptoCodec.java   |  11 +-
 ...yptoStreamsWithOpensslAesCtrCryptoCodec.java |   2 +-
 .../fs/FileContextMainOperationsBaseTest.java   |   4 +-
 .../org/apache/hadoop/fs/TestBlockLocation.java |  23 +-
 .../java/org/apache/hadoop/fs/TestFsShell.java  |  28 +
 .../fs/contract/AbstractContractAppendTest.java |   4 +-
 .../AbstractContractGetFileStatusTest.java      |  61 ++
 .../contract/AbstractContractSetTimesTest.java  |  61 ++
 .../hadoop/fs/contract/ContractOptions.java     |  12 +
 .../TestLocalFSContractGetFileStatus.java       |  33 +
 .../localfs/TestLocalFSContractSetTimes.java    |  33 +
 .../TestRawlocalContractGetFileStatus.java      |  33 +
 .../rawlocal/TestRawlocalContractSetTimes.java  |  33 +
 .../hadoop/io/TestSequenceFileAppend.java       |   6 +-
 .../apache/hadoop/io/nativeio/TestNativeIO.java |  90 ++-
 .../org/apache/hadoop/ipc/TestProtoBufRpc.java  |  77 +-
 .../metrics2/impl/TestMetricsSystemImpl.java    |  60 ++
 .../apache/hadoop/test/GenericTestUtils.java    |   9 +-
 .../org/apache/hadoop/test/MetricsAsserts.java  |   2 +-
 .../apache/hadoop/tracing/SetSpanReceiver.java  |   2 +-
 .../java/org/apache/hadoop/util/TestShell.java  |  39 +
 .../hadoop-common/src/test/proto/test.proto     |   7 +
 .../src/test/proto/test_rpc_service.proto       |   1 +
 .../src/test/resources/contract/localfs.xml     |  10 +
 .../src/test/resources/contract/rawlocal.xml    |  10 +
 .../scripts/hadoop-functions_test_helper.bash   |  56 ++
 .../src/test/scripts/hadoop_add_classpath.bats  | 100 +++
 .../src/test/scripts/hadoop_add_colonpath.bats  |  96 +++
 .../scripts/hadoop_add_common_to_classpath.bats |  71 ++
 .../test/scripts/hadoop_add_javalibpath.bats    |  98 +++
 .../src/test/scripts/hadoop_add_ldlibpath.bats  |  97 +++
 .../src/test/scripts/hadoop_add_param.bats      |  49 ++
 .../hadoop_add_to_classpath_userpath.bats       |  98 +++
 .../src/test/scripts/hadoop_basic_init.bats     |  94 +++
 .../src/test/scripts/hadoop_bootstrap.bats      |  51 ++
 .../src/test/scripts/hadoop_confdir.bats        |  92 +++
 .../test/scripts/hadoop_deprecate_envvar.bats   |  32 +
 .../src/test/scripts/hadoop_finalize.bats       | 206 +++++
 .../scripts/hadoop_finalize_catalina_opts.bats  |  56 ++
 .../test/scripts/hadoop_finalize_classpath.bats |  64 ++
 .../scripts/hadoop_finalize_hadoop_heap.bats    |  87 ++
 .../scripts/hadoop_finalize_hadoop_opts.bats    |  52 ++
 .../test/scripts/hadoop_finalize_libpaths.bats  |  30 +
 .../src/test/scripts/hadoop_java_setup.bats     |  47 ++
 .../src/test/scripts/hadoop_os_tricks.bats      |  34 +
 .../src/test/scripts/hadoop_rotate_log.bats     |  52 ++
 .../src/test/scripts/hadoop_shellprofile.bats   |  91 +++
 .../src/test/scripts/hadoop_slaves.bats         |  37 +
 .../src/test/scripts/hadoop_ssh.bats            |  51 ++
 .../scripts/hadoop_translate_cygwin_path.bats   |  48 ++
 .../test/scripts/hadoop_validate_classname.bats |  26 +
 .../hadoop-common/src/test/scripts/run-bats.sh  |  43 +
 .../hadoop-kms/src/main/sbin/kms.sh             |  29 +-
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |   2 +
 .../org/apache/hadoop/hdfs/ExtendedBlockId.java |  82 ++
 .../org/apache/hadoop/hdfs/ReplicaAccessor.java |  88 +++
 .../hadoop/hdfs/ReplicaAccessorBuilder.java     | 101 +++
 .../hdfs/client/HdfsClientConfigKeys.java       |  76 +-
 .../hadoop/hdfs/client/impl/DfsClientConf.java  | 785 +++++++++++++++++++
 .../hadoop/hdfs/client/impl/package-info.java   |  18 +
 .../org/apache/hadoop/hdfs/net/DomainPeer.java  | 132 ++++
 .../java/org/apache/hadoop/hdfs/net/Peer.java   | 123 +++
 .../hadoop/hdfs/protocol/ClientProtocol.java    |  14 +
 .../hadoop/hdfs/protocol/HdfsConstants.java     |  10 +
 .../datatransfer/BlockConstructionStage.java    |  62 ++
 .../datatransfer/DataTransferProtoUtil.java     | 146 ++++
 .../datatransfer/DataTransferProtocol.java      | 202 +++++
 .../hadoop/hdfs/protocol/datatransfer/Op.java   |  66 ++
 .../hdfs/protocol/datatransfer/Sender.java      | 261 ++++++
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  | 254 ++++++
 .../token/block/InvalidBlockTokenException.java |  41 +
 .../hdfs/server/datanode/CachingStrategy.java   |  76 ++
 .../hadoop/hdfs/shortcircuit/DfsClientShm.java  | 119 +++
 .../hdfs/shortcircuit/DfsClientShmManager.java  | 522 ++++++++++++
 .../hdfs/shortcircuit/ShortCircuitShm.java      | 647 +++++++++++++++
 .../hadoop/hdfs/util/ByteArrayManager.java      | 422 ++++++++++
 .../hadoop/hdfs/util/ExactSizeInputStream.java  | 125 +++
 .../apache/hadoop/hdfs/util/package-info.java   |  18 +
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |  20 +
 .../hdfs/web/resources/CreateFlagParam.java     |  48 ++
 .../hdfs/web/resources/CreateParentParam.java   |   2 +-
 .../src/main/proto/ClientDatanodeProtocol.proto |  33 -
 .../src/main/proto/ClientNamenodeProtocol.proto |  10 +
 .../src/main/proto/datatransfer.proto           |   4 +
 hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml  |   4 +-
 .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh  |  29 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     | 208 ++++-
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |   4 +
 .../hadoop-hdfs/src/CMakeLists.txt              |   1 +
 .../hadoop-hdfs/src/main/bin/hdfs               |  10 +-
 .../hadoop-hdfs/src/main/bin/start-balancer.sh  |  12 +-
 .../hadoop-hdfs/src/main/bin/stop-balancer.sh   |  10 +-
 .../apache/hadoop/fs/BlockStorageLocation.java  |  51 --
 .../java/org/apache/hadoop/fs/HdfsVolumeId.java |  73 --
 .../java/org/apache/hadoop/fs/SWebHdfs.java     |  51 ++
 .../java/org/apache/hadoop/fs/VolumeId.java     |  40 -
 .../main/java/org/apache/hadoop/fs/WebHdfs.java |  51 ++
 .../main/java/org/apache/hadoop/fs/package.html |  26 +
 .../apache/hadoop/hdfs/BlockReaderFactory.java  |  60 +-
 .../hadoop/hdfs/BlockStorageLocationUtil.java   | 368 ---------
 .../org/apache/hadoop/hdfs/ClientContext.java   |   5 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 132 +---
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   | 195 +++--
 .../org/apache/hadoop/hdfs/DFSOutputStream.java |   3 +-
 .../org/apache/hadoop/hdfs/DataStreamer.java    |   6 +-
 .../hadoop/hdfs/DistributedFileSystem.java      |  33 -
 .../org/apache/hadoop/hdfs/ExtendedBlockId.java |  82 --
 .../apache/hadoop/hdfs/ExternalBlockReader.java | 120 +++
 .../apache/hadoop/hdfs/HdfsConfiguration.java   |   8 +-
 .../apache/hadoop/hdfs/RemoteBlockReader.java   |   4 +-
 .../apache/hadoop/hdfs/RemoteBlockReader2.java  |   4 +-
 .../org/apache/hadoop/hdfs/XAttrHelper.java     |  13 +-
 .../hadoop/hdfs/client/impl/DfsClientConf.java  | 747 ------------------
 .../org/apache/hadoop/hdfs/net/DomainPeer.java  | 132 ----
 .../java/org/apache/hadoop/hdfs/net/Peer.java   | 123 ---
 .../hdfs/protocol/ClientDatanodeProtocol.java   |  19 -
 .../hdfs/protocol/HdfsBlocksMetadata.java       | 111 ---
 .../datatransfer/BlockConstructionStage.java    |  62 --
 .../datatransfer/DataTransferProtoUtil.java     | 148 ----
 .../datatransfer/DataTransferProtocol.java      | 201 -----
 .../hadoop/hdfs/protocol/datatransfer/Op.java   |  65 --
 .../hdfs/protocol/datatransfer/PipelineAck.java |   2 +-
 .../hdfs/protocol/datatransfer/Receiver.java    |   8 +-
 .../hdfs/protocol/datatransfer/Sender.java      | 261 ------
 .../datatransfer/sasl/DataTransferSaslUtil.java |   2 +-
 ...tDatanodeProtocolServerSideTranslatorPB.java |  43 +-
 .../ClientDatanodeProtocolTranslatorPB.java     |  49 +-
 ...tNamenodeProtocolServerSideTranslatorPB.java |  23 +-
 .../ClientNamenodeProtocolTranslatorPB.java     |  41 +-
 .../DatanodeProtocolClientSideTranslatorPB.java |   4 +-
 .../InterDatanodeProtocolTranslatorPB.java      |   2 +-
 .../NamenodeProtocolTranslatorPB.java           |   2 +-
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 228 +-----
 .../token/block/InvalidBlockTokenException.java |  41 -
 .../hadoop/hdfs/server/balancer/Balancer.java   | 182 +++--
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 186 +++--
 .../hdfs/server/balancer/MovedBlocks.java       |   5 +
 .../server/blockmanagement/BlockCollection.java |   9 +-
 .../hdfs/server/blockmanagement/BlockInfo.java  | 164 ++--
 .../blockmanagement/BlockInfoContiguous.java    |  82 +-
 .../BlockInfoUnderConstruction.java             | 405 ----------
 .../BlockInfoUnderConstructionContiguous.java   | 110 ---
 .../server/blockmanagement/BlockManager.java    | 780 ++++++++----------
 .../BlockPlacementPolicyDefault.java            | 177 +++--
 .../BlockPlacementPolicyWithNodeGroup.java      |  35 +-
 .../BlockStoragePolicySuite.java                |  11 +-
 .../BlockUnderConstructionFeature.java          | 254 ++++++
 .../ContiguousBlockStorageOp.java               | 106 ---
 .../blockmanagement/CorruptReplicasMap.java     |   4 +-
 .../blockmanagement/DatanodeDescriptor.java     |  75 +-
 .../server/blockmanagement/DatanodeManager.java |  19 +-
 .../blockmanagement/DatanodeStorageInfo.java    |  15 +-
 .../blockmanagement/DecommissionManager.java    |  15 +-
 .../blockmanagement/InvalidateBlocks.java       |   2 +-
 .../ReplicaUnderConstruction.java               | 119 +++
 .../blockmanagement/UnderReplicatedBlocks.java  |  33 +-
 .../hdfs/server/common/HdfsServerConstants.java |   9 -
 .../hdfs/server/datanode/BlockReceiver.java     |  14 +-
 .../hdfs/server/datanode/BlockSender.java       |  11 +-
 .../hdfs/server/datanode/CachingStrategy.java   |  76 --
 .../hadoop/hdfs/server/datanode/DNConf.java     |  30 +-
 .../hadoop/hdfs/server/datanode/DataNode.java   |  49 +-
 .../hdfs/server/datanode/DataXceiver.java       |  77 +-
 .../server/datanode/SecureDataNodeStarter.java  |   4 +-
 .../hdfs/server/datanode/VolumeScanner.java     |  12 +-
 .../server/datanode/fsdataset/FsDatasetSpi.java |  18 +-
 .../datanode/fsdataset/impl/BlockPoolSlice.java |   2 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  45 --
 .../datanode/web/webhdfs/ParameterParser.java   |  14 +
 .../datanode/web/webhdfs/WebHdfsHandler.java    |  23 +-
 .../hdfs/server/namenode/Checkpointer.java      |   4 +-
 .../hdfs/server/namenode/FSDirAppendOp.java     |   4 +-
 .../hdfs/server/namenode/FSDirAttrOp.java       |  73 +-
 .../hdfs/server/namenode/FSDirConcatOp.java     |   9 +-
 .../hdfs/server/namenode/FSDirDeleteOp.java     |   5 +-
 .../hdfs/server/namenode/FSDirRenameOp.java     |   7 +-
 .../hdfs/server/namenode/FSDirSnapshotOp.java   |   2 +
 .../server/namenode/FSDirStatAndListingOp.java  |   9 +-
 .../hdfs/server/namenode/FSDirTruncateOp.java   |  40 +-
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |  30 +-
 .../hdfs/server/namenode/FSDirXAttrOp.java      |  29 +-
 .../hdfs/server/namenode/FSDirectory.java       |  77 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  11 +
 .../hdfs/server/namenode/FSEditLogLoader.java   |  30 +-
 .../hadoop/hdfs/server/namenode/FSImage.java    |   6 +-
 .../hdfs/server/namenode/FSImageFormat.java     |   6 +-
 .../server/namenode/FSImageFormatPBINode.java   |  18 +-
 .../server/namenode/FSImageSerialization.java   |  10 +-
 .../hdfs/server/namenode/FSNamesystem.java      | 161 ++--
 .../hdfs/server/namenode/FSNamesystemLock.java  |  11 +
 .../namenode/FileUnderConstructionFeature.java  |  10 +-
 .../hadoop/hdfs/server/namenode/INode.java      |  52 +-
 .../hdfs/server/namenode/INodeDirectory.java    |  11 +-
 .../hadoop/hdfs/server/namenode/INodeFile.java  |  60 +-
 .../hdfs/server/namenode/LeaseManager.java      |  17 +-
 .../hadoop/hdfs/server/namenode/NameNode.java   | 176 ++++-
 .../hdfs/server/namenode/NameNodeRpcServer.java |   6 +
 .../hdfs/server/namenode/NamenodeFsck.java      |  25 +-
 .../hadoop/hdfs/server/namenode/Namesystem.java |   4 +-
 .../server/namenode/SerialNumberManager.java    |  44 --
 .../hdfs/server/namenode/SerialNumberMap.java   |  79 ++
 .../hdfs/server/namenode/XAttrFeature.java      |  78 +-
 .../hdfs/server/namenode/XAttrFormat.java       | 155 ++++
 .../server/namenode/XAttrPermissionFilter.java  |   6 +-
 .../hdfs/server/namenode/XAttrStorage.java      |  62 +-
 .../hdfs/server/namenode/ha/StandbyState.java   |   3 +-
 .../namenode/metrics/FSNamesystemMBean.java     |  19 +
 .../snapshot/FSImageFormatPBSnapshot.java       |  12 +-
 .../server/namenode/snapshot/FileDiffList.java  |   3 +-
 .../snapshot/FileWithSnapshotFeature.java       |  44 +-
 .../web/resources/NamenodeWebHdfsMethods.java   |  20 +-
 .../hdfs/server/protocol/NamespaceInfo.java     |  17 +-
 .../hadoop/hdfs/shortcircuit/DfsClientShm.java  | 119 ---
 .../hdfs/shortcircuit/DfsClientShmManager.java  | 514 ------------
 .../hdfs/shortcircuit/DomainSocketFactory.java  |   4 +-
 .../hdfs/shortcircuit/ShortCircuitCache.java    |   4 +-
 .../hdfs/shortcircuit/ShortCircuitShm.java      | 646 ---------------
 .../offlineImageViewer/FSImageHandler.java      |   3 +
 .../tools/offlineImageViewer/FSImageLoader.java |  90 +++
 .../hadoop/hdfs/util/ByteArrayManager.java      | 418 ----------
 .../hadoop/hdfs/util/ExactSizeInputStream.java  | 125 ---
 .../hadoop/hdfs/util/LightWeightHashSet.java    |  21 +-
 .../org/apache/hadoop/hdfs/web/JsonUtil.java    |   6 +-
 .../StorageContainerNameService.java            |   3 +-
 .../src/main/resources/hdfs-default.xml         |  40 +-
 .../src/main/webapps/hdfs/dfshealth.html        |  62 +-
 .../src/main/webapps/hdfs/dfshealth.js          |  63 +-
 .../webapps/static/dataTables.bootstrap.css     | 371 +++++++++
 .../main/webapps/static/dataTables.bootstrap.js | 205 +++++
 .../src/main/webapps/static/hadoop.css          |  51 ++
 .../webapps/static/jquery.dataTables.min.js     | 160 ++++
 .../src/main/webapps/static/moment.min.js       |   7 +
 .../hadoop-hdfs/src/site/markdown/HdfsDesign.md |   5 +-
 .../src/site/markdown/HdfsImageViewer.md        |   1 +
 .../src/site/markdown/HdfsMultihoming.md        |   2 +
 .../hadoop-hdfs/src/site/markdown/WebHDFS.md    |  23 +-
 .../org/apache/hadoop/hdfs/TestFiPipelines.java |   9 +-
 .../datanode/TestFiDataTransferProtocol.java    |   3 +-
 .../datanode/TestFiDataTransferProtocol2.java   |   5 +-
 .../hadoop/fs/TestEnhancedByteBufferAccess.java |  10 +-
 .../TestSWebHdfsFileContextMainOperations.java  | 110 +++
 .../java/org/apache/hadoop/fs/TestUnbuffer.java |   7 +-
 .../java/org/apache/hadoop/fs/TestVolumeId.java | 146 ----
 .../TestWebHdfsFileContextMainOperations.java   | 157 ++++
 .../contract/hdfs/TestHDFSContractAppend.java   |  15 -
 .../hdfs/TestHDFSContractGetFileStatus.java     |  46 ++
 .../contract/hdfs/TestHDFSContractSetTimes.java |  45 ++
 .../fs/viewfs/TestViewFsDefaultValue.java       |   8 +-
 .../apache/hadoop/hdfs/BlockReaderTestUtil.java |   6 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |  11 +-
 .../org/apache/hadoop/hdfs/FileAppendTest4.java |   5 +-
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  30 +-
 .../hadoop/hdfs/TestAppendSnapshotTruncate.java |  13 +-
 .../hadoop/hdfs/TestBlockReaderFactory.java     |   4 +-
 .../hadoop/hdfs/TestBlockReaderLocal.java       |   4 +-
 .../hadoop/hdfs/TestBlockReaderLocalLegacy.java |   6 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java     | 141 +++-
 .../TestClientProtocolForPipelineRecovery.java  |   4 +-
 .../org/apache/hadoop/hdfs/TestConnCache.java   |   5 +-
 .../hadoop/hdfs/TestDFSClientRetries.java       |   8 +-
 .../apache/hadoop/hdfs/TestDFSInputStream.java  |   2 +-
 .../hadoop/hdfs/TestDataTransferKeepalive.java  |   8 +-
 .../hadoop/hdfs/TestDataTransferProtocol.java   |   6 +-
 .../apache/hadoop/hdfs/TestDatanodeDeath.java   |   5 +-
 .../apache/hadoop/hdfs/TestDecommission.java    |  98 +++
 .../hadoop/hdfs/TestDisableConnCache.java       |   3 +-
 .../hadoop/hdfs/TestDistributedFileSystem.java  | 263 ++-----
 .../hadoop/hdfs/TestExternalBlockReader.java    | 298 +++++++
 .../org/apache/hadoop/hdfs/TestFileAppend.java  |   4 +-
 .../org/apache/hadoop/hdfs/TestFileAppend2.java |   6 +-
 .../org/apache/hadoop/hdfs/TestFileAppend4.java |   5 +-
 .../apache/hadoop/hdfs/TestFileCreation.java    | 165 ++--
 .../java/org/apache/hadoop/hdfs/TestHFlush.java |   3 +-
 .../apache/hadoop/hdfs/TestParallelRead.java    |   2 +-
 .../TestParallelShortCircuitLegacyRead.java     |   4 +-
 .../TestParallelShortCircuitReadUnCached.java   |   6 +-
 .../hadoop/hdfs/TestParallelUnixDomainRead.java |   2 +-
 .../org/apache/hadoop/hdfs/TestPipelines.java   |   9 +-
 .../java/org/apache/hadoop/hdfs/TestPread.java  |   6 +-
 .../java/org/apache/hadoop/hdfs/TestRead.java   |   5 +-
 .../hadoop/hdfs/TestRemoteBlockReader.java      |   4 +-
 .../hdfs/TestReplaceDatanodeOnFailure.java      |   4 +-
 .../org/apache/hadoop/hdfs/TestReplication.java |  26 +-
 .../hdfs/protocol/TestBlockListAsLongs.java     |   9 +-
 .../hadoop/hdfs/protocolPB/TestPBHelper.java    |  20 +-
 .../hdfs/server/balancer/TestBalancer.java      | 188 ++++-
 .../server/blockmanagement/TestBlockInfo.java   |   2 +-
 .../TestBlockInfoUnderConstruction.java         |  82 --
 .../blockmanagement/TestBlockManager.java       |  16 +-
 .../TestBlockReportRateLimiting.java            |   2 -
 .../blockmanagement/TestBlockTokenWithDFS.java  |   6 +-
 .../TestBlockUnderConstructionFeature.java      |  80 ++
 .../blockmanagement/TestHeartbeatHandling.java  |  22 +-
 .../server/blockmanagement/TestNodeCount.java   |   2 +-
 .../TestOverReplicatedBlocks.java               |   4 +-
 .../blockmanagement/TestPendingReplication.java |   1 -
 .../blockmanagement/TestReplicationPolicy.java  |  98 ++-
 .../TestUnderReplicatedBlocks.java              |  48 --
 .../server/datanode/SimulatedFSDataset.java     |   8 -
 .../server/datanode/TestBlockReplacement.java   |   7 +-
 .../server/datanode/TestCachingStrategy.java    |  12 +-
 .../datanode/TestDataNodeVolumeFailure.java     |   6 +-
 .../TestDataXceiverLazyPersistHint.java         | 180 +++++
 .../extdataset/ExternalDatasetImpl.java         |   6 -
 .../fsdataset/impl/LazyPersistTestCase.java     |   5 +-
 .../fsdataset/impl/TestDatanodeRestart.java     |   7 +-
 .../hdfs/server/mover/TestStorageMover.java     |   8 +-
 .../namenode/TestBlockUnderConstruction.java    |   6 +-
 .../TestCommitBlockSynchronization.java         |  12 +-
 .../TestDefaultBlockPlacementPolicy.java        |  49 +-
 .../hdfs/server/namenode/TestFSNamesystem.java  |  28 +
 .../server/namenode/TestFSNamesystemMBean.java  |  34 +-
 .../hdfs/server/namenode/TestFileTruncate.java  |   5 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |   2 +
 .../hdfs/server/namenode/TestHDFSConcat.java    |  24 +-
 .../hdfs/server/namenode/TestINodeFile.java     |   7 +-
 .../hdfs/server/namenode/TestLeaseManager.java  |  65 +-
 .../server/namenode/TestNameNodeMXBean.java     |  23 +-
 .../namenode/TestNameNodeMetricsLogger.java     | 193 +++++
 .../server/namenode/TestQuotaByStorageType.java |  14 +-
 .../hdfs/server/namenode/TestStartup.java       |  27 +-
 .../hdfs/server/namenode/TestXAttrFeature.java  | 107 +++
 .../namenode/ha/TestRetryCacheWithHA.java       |  11 +-
 .../server/namenode/ha/TestStandbyIsHot.java    |   2 +
 .../namenode/snapshot/SnapshotTestHelper.java   |   9 +-
 .../snapshot/TestFileWithSnapshotFeature.java   |   7 +-
 .../namenode/snapshot/TestSnapshotDeletion.java |  16 +-
 .../snapshot/TestSnapshotReplication.java       |  31 +-
 .../shortcircuit/TestShortCircuitCache.java     |  13 +-
 .../shortcircuit/TestShortCircuitLocalRead.java |   6 +-
 ...TestOfflineImageViewerForContentSummary.java | 248 ++++++
 .../hdfs/util/TestLightWeightHashSet.java       |  29 +-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |  25 +
 .../apache/hadoop/net/TestNetworkTopology.java  |   1 +
 .../apache/hadoop/security/TestPermission.java  |  26 +
 .../src/test/resources/contract/hdfs.xml        |  10 +
 .../src/test/resources/log4j.properties         |  13 +
 .../src/test/resources/testHDFSConf.xml         |   2 +-
 hadoop-mapreduce-project/CHANGES.txt            |  34 +
 hadoop-mapreduce-project/bin/mapred             |   2 +-
 .../hadoop/mapreduce/v2/app/MRAppMaster.java    |   2 +-
 .../v2/app/job/event/JobStartEvent.java         |   2 +-
 .../mapreduce/v2/app/job/impl/JobImpl.java      |  40 +-
 .../mapreduce/v2/app/rm/RMCommunicator.java     |  51 +-
 .../app/rm/RMContainerAllocationException.java  |  31 +
 .../v2/app/rm/RMContainerAllocator.java         |   4 +-
 .../v2/api/records/TestTaskAttemptReport.java   | 131 ++++
 .../v2/api/records/TestTaskReport.java          | 139 ++++
 .../mapreduce/v2/app/TestMRAppMaster.java       |  88 ++-
 .../mapreduce/v2/app/job/impl/TestJobImpl.java  | 130 ++-
 .../mapreduce/v2/app/rm/TestRMCommunicator.java |  99 +++
 .../v2/app/rm/TestRMContainerAllocator.java     |  48 +-
 .../v2/api/records/TaskAttemptReport.java       |   3 +
 .../mapreduce/v2/api/records/TaskReport.java    |   5 +-
 .../impl/pb/TaskAttemptReportPBImpl.java        |  38 +-
 .../api/records/impl/pb/TaskReportPBImpl.java   |  38 +-
 .../java/org/apache/hadoop/mapred/TaskLog.java  |   4 +-
 .../mapreduce/jobhistory/EventWriter.java       |  19 +-
 .../mapreduce/lib/output/MultipleOutputs.java   |  14 +-
 .../src/main/resources/mapred-default.xml       |  20 +
 .../src/site/markdown/EncryptedShuffle.md       |   8 +-
 .../hadoop/mapreduce/v2/hs/CompletedTask.java   |   2 +-
 .../mapreduce/v2/hs/CompletedTaskAttempt.java   |   2 +-
 .../mapreduce/v2/hs/JobHistoryServer.java       |  14 +-
 .../hadoop/mapred/ResourceMgrDelegate.java      |   7 +
 .../hadoop/mapred/TestClientRedirect.java       |   9 +
 .../hadoop/mapred/FadvisedChunkedFile.java      |   5 +-
 .../hadoop/mapred/FadvisedFileRegion.java       |   6 +-
 hadoop-project/pom.xml                          |   4 +-
 hadoop-tools/hadoop-aws/pom.xml                 |   2 +-
 .../org/apache/hadoop/fs/s3a/Constants.java     |   9 +-
 .../hadoop/fs/s3a/S3AFastOutputStream.java      |   2 +-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  26 +-
 .../apache/hadoop/fs/s3a/S3AOutputStream.java   |  15 +-
 .../src/site/markdown/tools/hadoop-aws/index.md |   6 +
 .../fs/azure/TestWasbUriAndConfiguration.java   |  30 +-
 .../src/test/resources/azure-test.xml           |   4 +
 .../org/apache/hadoop/tools/CopyListing.java    |  15 +-
 .../java/org/apache/hadoop/tools/DiffInfo.java  |  32 +-
 .../java/org/apache/hadoop/tools/DistCp.java    |  27 +-
 .../org/apache/hadoop/tools/DistCpOptions.java  |   4 +-
 .../org/apache/hadoop/tools/DistCpSync.java     | 308 +++++++-
 .../apache/hadoop/tools/SimpleCopyListing.java  | 151 +++-
 .../org/apache/hadoop/tools/TestDistCpSync.java | 345 +++++++-
 .../apache/hadoop/tools/TestOptionsParser.java  |  22 +-
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |   3 +
 .../yarn/sls/scheduler/RMNodeWrapper.java       |   5 +
 hadoop-yarn-project/CHANGES.txt                 | 135 +++-
 .../hadoop-yarn/bin/start-yarn.sh               |   7 +-
 .../hadoop-yarn/bin/stop-yarn.sh                |   7 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn        |   7 +
 .../yarn/api/ApplicationClientProtocol.java     |  18 +
 .../UpdateApplicationPriorityRequest.java       |  80 ++
 .../UpdateApplicationPriorityResponse.java      |  47 ++
 .../yarn/api/records/ApplicationReport.java     |  31 +-
 .../yarn/api/records/LogAggregationContext.java |  95 +++
 .../yarn/api/records/QueueStatistics.java       |  36 +
 .../hadoop/yarn/conf/YarnConfiguration.java     |  61 +-
 .../api/ContainerLogAggregationPolicy.java      |  54 ++
 .../yarn/server/api/ContainerLogContext.java    |  71 ++
 .../main/proto/applicationclient_protocol.proto |   1 +
 .../src/main/proto/yarn_protos.proto            |   7 +
 .../src/main/proto/yarn_service_protos.proto    |   8 +
 .../distributedshell/ApplicationMaster.java     |  31 +-
 .../distributedshell/TestDSAppMaster.java       |  11 +-
 .../hadoop/yarn/client/api/YarnClient.java      |  17 +
 .../yarn/client/api/impl/YarnClientImpl.java    |  11 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  35 +-
 .../apache/hadoop/yarn/client/cli/TopCLI.java   |  24 +-
 .../yarn/client/TestResourceTrackerOnHA.java    |   2 +-
 .../yarn/client/api/impl/TestYarnClient.java    |  10 +-
 .../hadoop/yarn/client/cli/TestYarnCLI.java     |  34 +-
 .../ApplicationClientProtocolPBClientImpl.java  |  20 +
 .../ApplicationClientProtocolPBServiceImpl.java |  22 +
 .../UpdateApplicationPriorityRequestPBImpl.java | 171 ++++
 ...UpdateApplicationPriorityResponsePBImpl.java |  69 ++
 .../impl/pb/ApplicationReportPBImpl.java        |  49 ++
 .../impl/pb/LogAggregationContextPBImpl.java    |  40 +
 .../records/impl/pb/QueueStatisticsPBImpl.java  |  36 +
 .../hadoop/yarn/event/AsyncDispatcher.java      |   9 +-
 .../ContainerLogsRetentionPolicy.java           |  29 -
 .../nodelabels/CommonNodeLabelsManager.java     |  22 +-
 .../nodelabels/FileSystemNodeLabelsStore.java   |   8 +-
 .../hadoop/yarn/util/resource/Resources.java    |  14 +-
 .../src/main/resources/yarn-default.xml         |  99 ++-
 .../hadoop/yarn/api/TestApplicatonReport.java   |   4 +-
 .../hadoop/yarn/event/TestAsyncDispatcher.java  |  25 +-
 .../nodelabels/TestCommonNodeLabelsManager.java |  17 +-
 ...pplicationHistoryManagerOnTimelineStore.java |  28 +-
 .../ApplicationHistoryServer.java               |  14 +-
 ...pplicationHistoryManagerOnTimelineStore.java |   4 +
 .../yarn/server/api/records/NodeStatus.java     |  18 +-
 .../api/records/impl/pb/NodeStatusPBImpl.java   |  22 +
 .../metrics/ApplicationMetricsConstants.java    |   6 +
 .../hadoop/yarn/server/utils/BuilderUtils.java  |   4 +-
 .../hadoop/yarn/server/webapp/AppBlock.java     |  24 +-
 .../hadoop/yarn/server/webapp/AppsBlock.java    |  12 +-
 .../hadoop/yarn/server/webapp/WebPageUtils.java |   4 +-
 .../hadoop/yarn/server/webapp/dao/AppInfo.java  |  15 +
 .../main/proto/yarn_server_common_protos.proto  |   1 +
 .../hadoop/yarn/server/nodemanager/Context.java |   2 +
 .../yarn/server/nodemanager/NodeManager.java    |  73 +-
 .../server/nodemanager/NodeResourceMonitor.java |  10 +-
 .../nodemanager/NodeResourceMonitorImpl.java    | 140 ++++
 .../nodemanager/NodeStatusUpdaterImpl.java      | 280 +++++--
 .../application/ApplicationImpl.java            |   5 +-
 .../AMOnlyLogAggregationPolicy.java             |  31 +
 ...AMOrFailedContainerLogAggregationPolicy.java |  35 +
 .../AbstractContainerLogAggregationPolicy.java  |  31 +
 .../AllContainerLogAggregationPolicy.java       |  30 +
 .../logaggregation/AppLogAggregator.java        |   5 +-
 .../logaggregation/AppLogAggregatorImpl.java    | 131 ++--
 .../FailedContainerLogAggregationPolicy.java    |  33 +
 ...edOrKilledContainerLogAggregationPolicy.java |  30 +
 .../logaggregation/LogAggregationService.java   |  19 +-
 .../NoneContainerLogAggregationPolicy.java      |  30 +
 .../SampleContainerLogAggregationPolicy.java    | 124 +++
 .../event/LogHandlerAppStartedEvent.java        |  15 +-
 .../monitor/ContainersMonitorImpl.java          |  18 +-
 .../nodemanager/metrics/NodeManagerMetrics.java |  13 +-
 .../nodelabels/AbstractNodeLabelsProvider.java  | 146 ++++
 .../ConfigurationNodeLabelsProvider.java        |  81 ++
 .../server/nodemanager/webapp/NodePage.java     |   8 +-
 .../server/nodemanager/webapp/dao/NodeInfo.java |   6 +
 .../impl/container-executor.c                   |  63 +-
 .../server/nodemanager/TestNodeManager.java     |  50 +-
 .../nodemanager/TestNodeResourceMonitor.java    |  35 +
 .../nodemanager/TestNodeStatusUpdater.java      |  34 +
 .../TestNodeStatusUpdaterForLabels.java         |  76 +-
 .../containermanager/TestAuxServices.java       |   1 +
 .../launcher/TestContainerLaunch.java           |   2 +-
 .../TestLogAggregationService.java              | 677 ++++++++++++++--
 .../TestNonAggregatingLogHandler.java           |  12 +-
 .../monitor/TestContainersMonitor.java          |   2 +-
 .../TestConfigurationNodeLabelsProvider.java    | 146 ++++
 .../nodemanager/webapp/TestNMWebServices.java   |   2 +-
 .../ApplicationMasterService.java               |   3 +-
 .../server/resourcemanager/ClientRMService.java |  84 +-
 .../resourcemanager/NodesListManager.java       |  28 +-
 .../resourcemanager/RMActiveServiceContext.java |  30 -
 .../server/resourcemanager/RMAuditLogger.java   |   2 +
 .../server/resourcemanager/RMContextImpl.java   |  23 +-
 .../server/resourcemanager/ResourceManager.java |  43 +-
 .../resourcemanager/ResourceTrackerService.java |   2 +
 .../metrics/ApplicationCreatedEvent.java        |  17 +-
 .../metrics/SystemMetricsPublisher.java         |   9 +-
 .../nodelabels/RMNodeLabelsManager.java         |  14 +-
 .../recovery/FileSystemRMStateStore.java        | 209 +++--
 .../recovery/LeveldbRMStateStore.java           | 114 ++-
 .../recovery/MemoryRMStateStore.java            |  57 ++
 .../recovery/NullRMStateStore.java              |  23 +
 .../resourcemanager/recovery/RMStateStore.java  | 166 +++-
 .../recovery/RMStateStoreEventType.java         |   5 +-
 .../RMStateStoreStoreReservationEvent.java      |  56 ++
 .../recovery/RMStateUpdateAppEvent.java         |  13 +
 .../recovery/ZKRMStateStore.java                | 124 ++-
 .../resourcemanager/reservation/Plan.java       |   3 +-
 .../resourcemanager/reservation/PlanView.java   |  11 +-
 .../ReservationSchedulerConfiguration.java      |  14 +-
 .../reservation/ReservationSystem.java          |  15 +-
 .../reservation/ReservationSystemUtil.java      | 102 ++-
 .../server/resourcemanager/rmapp/RMAppImpl.java |  12 +-
 .../rmapp/attempt/RMAppAttemptImpl.java         |  22 +-
 .../rmcontainer/RMContainerImpl.java            |  70 +-
 .../server/resourcemanager/rmnode/RMNode.java   |   7 +-
 .../resourcemanager/rmnode/RMNodeImpl.java      |  23 +-
 .../scheduler/AbstractYarnScheduler.java        |  17 +-
 .../scheduler/SchedulerApplicationAttempt.java  |   2 +-
 .../scheduler/YarnScheduler.java                |  22 +
 .../scheduler/capacity/AbstractCSQueue.java     |  11 +-
 .../scheduler/capacity/CSAssignment.java        |  11 +-
 .../scheduler/capacity/CapacityScheduler.java   | 115 ++-
 .../capacity/CapacitySchedulerContext.java      |   2 -
 .../scheduler/capacity/LeafQueue.java           |  90 ++-
 .../scheduler/capacity/ParentQueue.java         |  18 +-
 .../capacity/allocator/AllocationState.java     |  28 +
 .../capacity/allocator/ContainerAllocation.java |  89 +++
 .../capacity/allocator/ContainerAllocator.java  | 150 ++++
 .../allocator/RegularContainerAllocator.java    | 698 +++++++++++++++++
 .../scheduler/common/fica/FiCaSchedulerApp.java | 639 +--------------
 .../scheduler/fair/FSAppAttempt.java            |   4 +-
 .../AbstractComparatorOrderingPolicy.java       |   6 +
 .../ClientToAMTokenSecretManagerInRM.java       |   7 +
 .../resourcemanager/webapp/RMAppsBlock.java     |   7 +-
 .../resourcemanager/webapp/dao/AppInfo.java     |  19 +-
 .../webapp/dao/CapacitySchedulerQueueInfo.java  |  18 +
 .../webapp/dao/FairSchedulerQueueInfo.java      |  24 +-
 .../yarn_server_resourcemanager_recovery.proto  |  18 +-
 .../yarn/server/resourcemanager/MockAM.java     |  14 +-
 .../yarn/server/resourcemanager/MockNodes.java  |   4 +
 .../server/resourcemanager/TestAppManager.java  |   5 +-
 .../resourcemanager/TestClientRMService.java    |  64 ++
 .../resourcemanager/TestRMNodeTransitions.java  |  62 +-
 .../resourcemanager/TestResourceManager.java    |   1 +
 .../TestWorkPreservingRMRestart.java            |   2 +-
 .../TestRMAppLogAggregationStatus.java          |   4 +-
 .../metrics/TestSystemMetricsPublisher.java     |  16 +
 ...pacityPreemptionPolicyForNodePartitions.java |  10 +-
 .../nodelabels/TestRMNodeLabelsManager.java     |  48 +-
 .../recovery/RMStateStoreTestBase.java          | 186 +++++
 .../recovery/TestFSRMStateStore.java            |   1 +
 .../recovery/TestLeveldbRMStateStore.java       |   6 +
 .../recovery/TestZKRMStateStore.java            |   1 +
 .../reservation/ReservationSystemTestUtil.java  | 193 ++---
 .../TestCapacityReservationSystem.java          |  94 ---
 .../TestCapacitySchedulerPlanFollower.java      |  11 +-
 .../reservation/TestFairReservationSystem.java  | 127 ---
 .../TestFairSchedulerPlanFollower.java          |  71 +-
 .../TestInMemoryReservationAllocation.java      |  36 +-
 .../reservation/TestReservationSystem.java      | 213 +++++
 .../resourcetracker/TestNMReconnect.java        |  39 +
 .../rmapp/TestNodesListManager.java             | 162 ++++
 .../rmapp/TestRMAppTransitions.java             |   6 +-
 .../attempt/TestRMAppAttemptTransitions.java    |  40 +-
 .../rmcontainer/TestRMContainerImpl.java        |  79 +-
 .../capacity/TestApplicationLimits.java         |  14 +-
 .../capacity/TestApplicationPriority.java       | 329 ++++++--
 .../capacity/TestCapacityScheduler.java         |  83 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   3 +-
 .../capacity/TestContainerAllocation.java       |  12 +-
 .../scheduler/capacity/TestLeafQueue.java       | 356 +++++----
 .../scheduler/capacity/TestParentQueue.java     |   2 -
 .../scheduler/capacity/TestReservations.java    |   8 +-
 .../scheduler/capacity/TestUtils.java           |   9 +-
 .../scheduler/fifo/TestFifoScheduler.java       |  19 +-
 .../webapp/TestRMWebServicesApps.java           |  36 +-
 .../webapp/TestRMWebServicesCapacitySched.java  |   4 +-
 .../yarn/server/webproxy/WebAppProxyServer.java |  32 +-
 .../src/site/markdown/ResourceManagerRest.md    |  20 +-
 .../src/site/markdown/TimelineServer.md         |  32 +-
 614 files changed, 24687 insertions(+), 10842 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index 7131c7d,42934c3..786dd1a
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@@ -19,10 -19,7 +19,9 @@@ package org.apache.hadoop.hdfs.server.b
  
  import org.apache.hadoop.classification.InterfaceAudience;
  import org.apache.hadoop.hdfs.protocol.Block;
- import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
  
 +import java.util.LinkedList;
 +
  /**
   * Subclass of {@link BlockInfo}, used for a block with replication scheme.
   */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 144d8fb,7f02612..4dd2205
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -91,14 -99,6 +99,7 @@@ import com.google.common.base.Precondit
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
- import java.io.IOException;
- import java.io.PrintWriter;
- import java.util.*;
- import java.util.concurrent.ThreadLocalRandom;
- import java.util.concurrent.atomic.AtomicLong;
- 
- import static org.apache.hadoop.util.ExitUtil.terminate;
 +
  /**
   * Keeps information related to the blocks stored in the Hadoop cluster.
   */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index de30e08,7e3c59b..fd1c0f0
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@@ -17,10 -17,22 +17,8 @@@
   */
  package org.apache.hadoop.hdfs.server.blockmanagement;
  
 -import java.util.ArrayList;
 -import java.util.BitSet;
 -import java.util.Collection;
 -import java.util.Collections;
 -import java.util.HashMap;
 -import java.util.HashSet;
 -import java.util.Iterator;
 -import java.util.LinkedList;
 -import java.util.List;
 -import java.util.Map;
 -import java.util.Queue;
 -import java.util.Set;
 -
  import com.google.common.annotations.VisibleForTesting;
 -
  import com.google.common.collect.ImmutableList;
- import org.apache.commons.logging.Log;
- import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.classification.InterfaceAudience;
  import org.apache.hadoop.classification.InterfaceStability;
  import org.apache.hadoop.fs.StorageType;
@@@ -36,9 -49,9 +35,11 @@@ import org.apache.hadoop.hdfs.util.Enum
  import org.apache.hadoop.hdfs.util.LightWeightHashSet;
  import org.apache.hadoop.util.IntrusiveCollection;
  import org.apache.hadoop.util.Time;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
  
 +import java.util.*;
 +
  /**
   * This class extends the DatanodeInfo class with ephemeral information (eg
   * health, capacity, what blocks are associated with the Datanode) that is

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index a325a19,216d6d2..8bd8c51
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@@ -17,9 -17,13 +17,8 @@@
   */
  package org.apache.hadoop.hdfs.server.blockmanagement;
  
 -import java.util.Arrays;
 -import java.util.Iterator;
 -import java.util.List;
 -
  import com.google.common.annotations.VisibleForTesting;
 -
  import org.apache.hadoop.fs.StorageType;
- import org.apache.hadoop.hdfs.protocol.Block;
  import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
  import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
  import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index f393aa0,1cb308f..2468453
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@@ -791,15 -791,15 +794,15 @@@ class BlockReceiver implements Closeabl
          //
          if (syncBehindWrites) {
            if (syncBehindWritesInBackground) {
 -            this.datanode.getFSDataset().submitBackgroundSyncFileRangeRequest(
 +            dataset.submitBackgroundSyncFileRangeRequest(
                  block, outFd, lastCacheManagementOffset,
                  offsetInBlock - lastCacheManagementOffset,
-                 NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
+                 SYNC_FILE_RANGE_WRITE);
            } else {
              NativeIO.POSIX.syncFileRangeIfPossible(outFd,
-                 lastCacheManagementOffset, offsetInBlock
-                     - lastCacheManagementOffset,
-                 NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
+                 lastCacheManagementOffset,
+                 offsetInBlock - lastCacheManagementOffset,
+                 SYNC_FILE_RANGE_WRITE);
            }
          }
          //

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 75b0d26,dfaa525..11ad30a
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@@ -643,14 -628,9 +642,16 @@@ class DataXceiver extends Receiver impl
      final boolean isClient = !isDatanode;
      final boolean isTransfer = stage == BlockConstructionStage.TRANSFER_RBW
          || stage == BlockConstructionStage.TRANSFER_FINALIZED;
+     allowLazyPersist = allowLazyPersist &&
+         (dnConf.getAllowNonLocalLazyPersist() || peer.isLocal());
      long size = 0;
 +    final FsDatasetSpi<?> dataset =
 +        (FsDatasetSpi<?>) datanode.getDataset(block.getBlockPoolId());
 +    if (dataset == null) {
 +      throw new IOException(
 +          "Unknown or unitialized blockpool " + block.getBlockPoolId());
 +    }
 +
      // check single target for transfer-RBW/Finalized 
      if (isTransfer && targets.length > 0) {
        throw new IOException(stage + " does not support multiple targets "
@@@ -1208,10 -1158,10 +1205,10 @@@
          DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto(
              checksumInfo.getChecksum());
          // open a block receiver and check if the block does not exist
-         blockReceiver = new BlockReceiver(block, storageType,
+         blockReceiver = getBlockReceiver(block, storageType,
              proxyReply, proxySock.getRemoteSocketAddress().toString(),
              proxySock.getLocalSocketAddress().toString(),
 -            null, 0, 0, 0, "", null, datanode, remoteChecksum,
 +            null, 0, 0, 0, "", null, datanode, dataset, remoteChecksum,
              CachingStrategy.newDropBehind(), false, false);
          
          // receive a block
@@@ -1263,6 -1213,39 +1260,40 @@@
      datanode.metrics.addReplaceBlockOp(elapsed());
    }
  
+ 
+   /**
+    * Separated for testing.
+    */
+   @VisibleForTesting
+   BlockReceiver getBlockReceiver(
+       final ExtendedBlock block, final StorageType storageType,
+       final DataInputStream in,
+       final String inAddr, final String myAddr,
+       final BlockConstructionStage stage,
+       final long newGs, final long minBytesRcvd, final long maxBytesRcvd,
+       final String clientname, final DatanodeInfo srcDataNode,
 -      final DataNode dn, DataChecksum requestedChecksum,
++      final DataNode dn, final FsDatasetSpi<?> dataset,
++      DataChecksum requestedChecksum,
+       CachingStrategy cachingStrategy,
+       final boolean allowLazyPersist,
+       final boolean pinning) throws IOException {
+     return new BlockReceiver(block, storageType, in,
+         inAddr, myAddr, stage, newGs, minBytesRcvd, maxBytesRcvd,
 -        clientname, srcDataNode, dn, requestedChecksum,
++        clientname, srcDataNode, dn, dataset, requestedChecksum,
+         cachingStrategy, allowLazyPersist, pinning);
+   }
+ 
+   /**
+    * Separated for testing.
+    * @return
+    */
+   @VisibleForTesting
+   DataOutputStream getBufferedOutputStream() {
+     return new DataOutputStream(
+         new BufferedOutputStream(getOutputStream(), smallBufferSize));
+   }
+ 
+ 
    private long elapsed() {
      return monotonicNow() - opStartTime;
    }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 3c39ace,63f0377..9b9e5c4
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@@ -30,14 -30,19 +30,13 @@@ import java.util.ArrayList
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
 -import java.util.Set;
  
  import org.apache.hadoop.classification.InterfaceAudience;
 -import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.StorageType;
 -import org.apache.hadoop.hdfs.DFSConfigKeys;
  import org.apache.hadoop.hdfs.protocol.Block;
 -import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
  import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
  import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
- import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
  import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 -import org.apache.hadoop.hdfs.server.datanode.DataNode;
 -import org.apache.hadoop.hdfs.server.datanode.DataStorage;
  import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
  import org.apache.hadoop.hdfs.server.datanode.Replica;
  import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
@@@ -398,21 -560,37 +397,18 @@@ public interface FsDatasetSpi<V extend
        ) throws IOException;
  
    /**
-    * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in 
-    * <code>blocks</code>.
-    * 
-    * @param bpid pool to query
-    * @param blockIds List of block ids for which to return metadata
-    * @return metadata Metadata for the list of blocks
-    * @throws IOException
++  /**
++   * submit a sync_file_range request to AsyncDiskService
+    * Enable 'trash' for the given dataset. When trash is enabled, files are
+    * moved to a separate trash directory instead of being deleted immediately.
+    * This can be useful for example during rolling upgrades.
     */
-   public HdfsBlocksMetadata getHdfsBlocksMetadata(String bpid,
-       long[] blockIds) throws IOException;
+   void enableTrash(String bpid);
  
    /**
-    * submit a sync_file_range request to AsyncDiskService
+    * Clear trash
     */
 -  void clearTrash(String bpid);
 -
 -  /**
 -   * @return true when trash is enabled
 -   */
 -  boolean trashEnabled(String bpid);
 -
 -  /**
 -   * Create a marker file indicating that a rolling upgrade is in progress.
 -   */
 -  void setRollingUpgradeMarker(String bpid) throws IOException;
 -
 -  /**
 -   * Delete the rolling upgrade marker file if it exists.
 -   * @param bpid
 -   */
 -  void clearRollingUpgradeMarker(String bpid) throws IOException;
 -
 -  /**
 -   * submit a sync_file_range request to AsyncDiskService.
 -   */
 -  void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block,
 +  public void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block,
        final FileDescriptor fd, final long offset, final long nbytes,
        final int flags);
  

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
index 2b7e3ff,0000000..7d1eb66
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
@@@ -1,163 -1,0 +1,162 @@@
 +/**
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.hadoop.storagecontainer;
 +
 +import org.apache.hadoop.hdfs.protocol.Block;
 +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
- import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
 +import org.apache.hadoop.hdfs.server.namenode.CacheManager;
 +import org.apache.hadoop.hdfs.server.namenode.NameNode;
 +import org.apache.hadoop.hdfs.server.namenode.Namesystem;
 +import org.apache.hadoop.ipc.StandbyException;
 +import org.apache.hadoop.security.AccessControlException;
 +
 +import java.util.concurrent.locks.ReentrantReadWriteLock;
 +
 +/**
 + * Namesystem implementation to be used by StorageContainerManager.
 + */
 +public class StorageContainerNameService implements Namesystem {
 +
 +  private ReentrantReadWriteLock coarseLock = new ReentrantReadWriteLock();
 +  private String blockPoolId;
 +  private volatile boolean serviceRunning = true;
 +
 +  public void shutdown() {
 +    serviceRunning = false;
 +  }
 +
 +  @Override
 +  public boolean isRunning() {
 +    return serviceRunning;
 +  }
 +
 +  @Override
 +  public void checkSuperuserPrivilege() throws AccessControlException {
 +    // TBD
 +  }
 +
 +  @Override
 +  public String getBlockPoolId() {
 +    return blockPoolId;
 +  }
 +
 +  public void setBlockPoolId(String id) {
 +    this.blockPoolId = id;
 +  }
 +
 +  @Override
 +  public boolean isInStandbyState() {
 +    // HA mode is not supported
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean isGenStampInFuture(Block block) {
 +    // HA mode is not supported
 +    return false;
 +  }
 +
 +  @Override
 +  public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal) {
 +    // TBD
 +  }
 +
 +  @Override
 +  public void checkOperation(NameNode.OperationCategory read)
 +    throws StandbyException {
 +    // HA mode is not supported
 +  }
 +
 +  @Override
-   public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
++  public boolean isInSnapshot(BlockInfo blockInfo) {
 +    // Snapshots not supported
 +    return false;
 +  }
 +
 +  @Override
 +  public CacheManager getCacheManager() {
 +    // Cache Management is not supported
 +    return null;
 +  }
 +
 +  @Override
 +  public void readLock() {
 +    coarseLock.readLock().lock();
 +  }
 +
 +  @Override
 +  public void readUnlock() {
 +    coarseLock.readLock().unlock();
 +  }
 +
 +  @Override
 +  public boolean hasReadLock() {
 +    return coarseLock.getReadHoldCount() > 0 || hasWriteLock();
 +  }
 +
 +  @Override
 +  public void writeLock() {
 +    coarseLock.writeLock().lock();
 +  }
 +
 +  @Override
 +  public void writeLockInterruptibly() throws InterruptedException {
 +    coarseLock.writeLock().lockInterruptibly();
 +  }
 +
 +  @Override
 +  public void writeUnlock() {
 +    coarseLock.writeLock().unlock();
 +  }
 +
 +  @Override
 +  public boolean hasWriteLock() {
 +    return coarseLock.isWriteLockedByCurrentThread();
 +  }
 +
 +  @Override
 +  public void checkSafeMode() {
 +    // TBD
 +  }
 +
 +  @Override
 +  public boolean isInSafeMode() {
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean isInStartupSafeMode() {
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean isPopulatingReplQueues() {
 +    return false;
 +  }
 +
 +  @Override
 +  public void incrementSafeBlockCount(int replication) {
 +    // Do nothing
 +  }
 +
 +  @Override
 +  public void decrementSafeBlockCount(BlockInfo b) {
 +    // Do nothing
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index b05be39,5d1b31a..4788fcc
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@@ -42,12 -42,7 +42,10 @@@ import org.apache.hadoop.hdfs.protocol.
  import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
  import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
  import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
- import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
- import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
  import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 +import org.apache.hadoop.hdfs.server.common.Storage;
 +import org.apache.hadoop.hdfs.server.datanode.dataset.DatasetSpi;
 +import org.apache.hadoop.hdfs.server.datanode.fsdataset.*;
  import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
  import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
  import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
index 0000000,d8a7188..48f8cef
mode 000000,100644..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
@@@ -1,0 -1,178 +1,180 @@@
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  * <p/>
+  * http://www.apache.org/licenses/LICENSE-2.0
+  * <p/>
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.hadoop.hdfs.server.datanode;
+ 
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.StorageType;
+ import org.apache.hadoop.hdfs.*;
+ import org.apache.hadoop.hdfs.net.*;
+ import org.apache.hadoop.hdfs.protocol.*;
+ import org.apache.hadoop.hdfs.protocol.datatransfer.*;
+ import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
+ import org.apache.hadoop.util.DataChecksum;
+ import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
++import org.apache.hadoop.hdfs.server.datanode.fsdataset.*;
+ 
+ import org.junit.Rule;
+ import org.junit.Test;
+ import org.junit.rules.Timeout;
+ import org.mockito.ArgumentCaptor;
+ 
+ import java.io.DataInputStream;
+ import java.io.DataOutputStream;
+ import java.io.IOException;
+ import java.util.Arrays;
+ 
+ import static org.hamcrest.MatcherAssert.assertThat;
+ import static org.hamcrest.core.Is.is;
+ import static org.mockito.Mockito.*;
+ 
+ 
+ /**
+  * Mock-based unit test to verify that the DataXceiver correctly handles the
+  * LazyPersist hint from clients.
+  */
+ public class TestDataXceiverLazyPersistHint {
+   @Rule
+   public Timeout timeout = new Timeout(300000);
+ 
+   private enum PeerLocality {
+     LOCAL,
+     REMOTE
+   }
+ 
+   private enum NonLocalLazyPersist {
+     ALLOWED,
+     NOT_ALLOWED
+   }
+ 
+   /**
+    * Ensure that the correct hint is passed to the block receiver when
+    * the client is local.
+    */
+   @Test
+   public void testWithLocalClient() throws IOException {
+     ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class);
+     DataXceiver xceiver = makeStubDataXceiver(
+         PeerLocality.LOCAL, NonLocalLazyPersist.NOT_ALLOWED, captor);
+ 
+     for (Boolean lazyPersistSetting : Arrays.asList(true, false)) {
+       issueWriteBlockCall(xceiver, lazyPersistSetting);
+       assertThat(captor.getValue(), is(lazyPersistSetting));
+     }
+   }
+ 
+   /**
+    * Ensure that hint is always false when the client is remote.
+    */
+   @Test
+   public void testWithRemoteClient() throws IOException {
+     ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class);
+     DataXceiver xceiver = makeStubDataXceiver(
+         PeerLocality.REMOTE, NonLocalLazyPersist.NOT_ALLOWED, captor);
+ 
+     for (Boolean lazyPersistSetting : Arrays.asList(true, false)) {
+       issueWriteBlockCall(xceiver, lazyPersistSetting);
+       assertThat(captor.getValue(), is(false));
+     }
+   }
+ 
+   /**
+    * Ensure that the correct hint is passed to the block receiver when
+    * the client is remote AND dfs.datanode.allow.non.local.lazy.persist
+    * is set to true.
+    */
+   @Test
+   public void testOverrideWithRemoteClient() throws IOException {
+     ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class);
+     DataXceiver xceiver = makeStubDataXceiver(
+         PeerLocality.REMOTE, NonLocalLazyPersist.ALLOWED, captor);
+ 
+     for (Boolean lazyPersistSetting : Arrays.asList(true, false)) {
+       issueWriteBlockCall(xceiver, lazyPersistSetting);
+       assertThat(captor.getValue(), is(lazyPersistSetting));
+     }
+   }
+ 
+   /**
+    * Issue a write block call with dummy parameters. The only parameter useful
+    * for this test is the value of lazyPersist.
+    */
+   private void issueWriteBlockCall(DataXceiver xceiver, boolean lazyPersist)
+       throws IOException {
+     xceiver.writeBlock(
+         new ExtendedBlock("Dummy-pool", 0L),
+         StorageType.RAM_DISK,
+         null,
+         "Dummy-Client",
+         new DatanodeInfo[0],
+         new StorageType[0],
+         mock(DatanodeInfo.class),
+         BlockConstructionStage.PIPELINE_SETUP_CREATE,
+         0, 0, 0, 0,
+         DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 0),
+         CachingStrategy.newDefaultStrategy(),
+         lazyPersist,
+         false, null);
+   }
+ 
+   // Helper functions to setup the mock objects.
+ 
+   private static DataXceiver makeStubDataXceiver(
+       PeerLocality locality,
+       NonLocalLazyPersist nonLocalLazyPersist,
+       final ArgumentCaptor<Boolean> captor) throws IOException {
+     DataXceiver xceiverSpy = spy(DataXceiver.create(
+             getMockPeer(locality),
+             getMockDn(nonLocalLazyPersist),
+             mock(DataXceiverServer.class)));
+ 
+     doReturn(mock(BlockReceiver.class)).when(xceiverSpy).getBlockReceiver(
+         any(ExtendedBlock.class), any(StorageType.class),
+         any(DataInputStream.class), anyString(), anyString(),
+         any(BlockConstructionStage.class), anyLong(), anyLong(), anyLong(),
+         anyString(), any(DatanodeInfo.class), any(DataNode.class),
++        any(FsDatasetSpi.class),
+         any(DataChecksum.class), any(CachingStrategy.class),
+         captor.capture(), anyBoolean());
+     doReturn(mock(DataOutputStream.class)).when(xceiverSpy)
+         .getBufferedOutputStream();
+     return xceiverSpy;
+   }
+ 
+   private static Peer getMockPeer(PeerLocality locality) {
+     Peer peer = mock(Peer.class);
+     when(peer.isLocal()).thenReturn(locality == PeerLocality.LOCAL);
+     when(peer.getRemoteAddressString()).thenReturn("1.1.1.1:1000");
+     when(peer.getLocalAddressString()).thenReturn("2.2.2.2:2000");
+     return peer;
+   }
+ 
+   private static DataNode getMockDn(NonLocalLazyPersist nonLocalLazyPersist) {
+     Configuration conf = new HdfsConfiguration();
+     conf.setBoolean(
+         DFS_DATANODE_NON_LOCAL_LAZY_PERSIST,
+         nonLocalLazyPersist == NonLocalLazyPersist.ALLOWED);
+     DNConf dnConf = new DNConf(conf);
+     DataNodeMetrics mockMetrics = mock(DataNodeMetrics.class);
+     DataNode mockDn = mock(DataNode.class);
+     when(mockDn.getDnConf()).thenReturn(dnConf);
+     when(mockDn.getConf()).thenReturn(conf);
+     when(mockDn.getMetrics()).thenReturn(mockMetrics);
+     return mockDn;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index 29bb04b,e3be5fd..c5a1bd9
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@@ -27,9 -27,7 +27,8 @@@ import org.apache.hadoop.hdfs.protocol.
  import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
  import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
  import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
- import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
  import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 +import org.apache.hadoop.hdfs.server.common.Storage;
  import org.apache.hadoop.hdfs.server.datanode.*;
  import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
  import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;


Mime
View raw message