hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Date Fri, 11 Nov 2016 18:58:13 GMT
Merge branch 'trunk' into HDFS-7240

 Conflicts:
	hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e55bdefd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e55bdefd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e55bdefd

Branch: refs/heads/HDFS-7240
Commit: e55bdefdab3c8a7f1e51451bb8280e61f2c6f0db
Parents: e6a6b5b 503e73e
Author: Anu Engineer <aengineer@apache.org>
Authored: Fri Nov 11 10:49:05 2016 -0800
Committer: Anu Engineer <aengineer@apache.org>
Committed: Fri Nov 11 10:49:05 2016 -0800

----------------------------------------------------------------------
 .gitignore                                      |   14 +
 BUILDING.txt                                    |   23 +-
 LICENSE.txt                                     |  546 +++
 dev-support/bin/checkcompatibility.py           |  345 ++
 dev-support/bin/create-release                  |    2 +-
 dev-support/docker/Dockerfile                   |    9 +
 .../resources/assemblies/hadoop-yarn-dist.xml   |    7 +
 .../main/resources/checkstyle/checkstyle.xml    |    2 +-
 hadoop-client/pom.xml                           |   20 +-
 .../hadoop-cloud-storage/pom.xml                |  127 +
 hadoop-cloud-storage-project/pom.xml            |   54 +
 .../hadoop-auth-examples/pom.xml                |    2 +-
 .../examples/RequestLoggerFilter.java           |   12 +
 hadoop-common-project/hadoop-auth/pom.xml       |   13 +-
 .../server/KerberosAuthenticationHandler.java   |    7 +-
 .../client/AuthenticatorTestCase.java           |   29 +-
 hadoop-common-project/hadoop-common/pom.xml     |   45 +-
 .../hadoop-common/src/CMakeLists.txt            |    4 +-
 .../hadoop-common/src/main/bin/hadoop           |    2 +-
 .../hadoop-common/src/main/conf/hadoop-env.sh   |    6 -
 .../src/main/conf/hadoop-metrics2.properties    |   22 +-
 .../org/apache/hadoop/conf/ConfServlet.java     |    8 +-
 .../crypto/key/kms/KMSClientProvider.java       |   56 +-
 .../crypto/key/kms/KMSDelegationToken.java      |   52 +
 .../fs/CommonConfigurationKeysPublic.java       |    2 +
 .../apache/hadoop/fs/FSExceptionMessages.java   |    2 +
 .../java/org/apache/hadoop/fs/FileSystem.java   |    9 +-
 .../java/org/apache/hadoop/fs/FileUtil.java     |    8 +-
 .../org/apache/hadoop/fs/HarFileSystem.java     |    3 +
 .../java/org/apache/hadoop/fs/shell/Count.java  |    9 +-
 .../org/apache/hadoop/fs/viewfs/InodeTree.java  |    6 +
 .../apache/hadoop/fs/viewfs/ViewFileSystem.java |   15 +-
 .../org/apache/hadoop/fs/viewfs/ViewFs.java     |    8 +-
 .../hadoop/http/AdminAuthorizedServlet.java     |   13 +-
 .../org/apache/hadoop/http/HttpRequestLog.java  |    4 +-
 .../org/apache/hadoop/http/HttpServer2.java     |  337 +-
 .../java/org/apache/hadoop/http/JettyUtils.java |   35 +
 .../rawcoder/NativeXORRawDecoder.java           |   59 +
 .../rawcoder/NativeXORRawEncoder.java           |   60 +
 .../NativeXORRawErasureCoderFactory.java        |   39 +
 .../hadoop/ipc/metrics/RpcDetailedMetrics.java  |    4 +-
 .../org/apache/hadoop/jmx/JMXJsonServlet.java   |    8 +-
 .../java/org/apache/hadoop/log/LogLevel.java    |   11 +-
 .../metrics2/lib/MutableMetricsFactory.java     |    5 +
 .../hadoop/metrics2/lib/MutableRates.java       |    6 +
 .../lib/MutableRatesWithAggregation.java        |  148 +
 .../apache/hadoop/metrics2/lib/MutableStat.java |    4 +
 .../apache/hadoop/metrics2/util/SampleStat.java |   19 +-
 .../apache/hadoop/security/SaslInputStream.java |    5 +-
 .../apache/hadoop/security/SaslRpcClient.java   |    3 +
 .../hadoop/security/UserGroupInformation.java   |   84 +-
 .../ssl/SslSelectChannelConnectorSecure.java    |   58 -
 .../AbstractDelegationTokenSecretManager.java   |   69 +-
 .../DelegationTokenAuthenticationHandler.java   |   32 +-
 .../org/apache/hadoop/util/DiskChecker.java     |  159 +-
 .../hadoop/util/GenericOptionsParser.java       |   37 +-
 .../apache/hadoop/util/InstrumentedLock.java    |  197 +
 .../hadoop/util/InstrumentedReadLock.java       |   92 +
 .../hadoop/util/InstrumentedReadWriteLock.java  |   58 +
 .../hadoop/util/InstrumentedWriteLock.java      |   54 +
 .../apache/hadoop/util/LimitInputStream.java    |    3 +
 .../main/java/org/apache/hadoop/util/Shell.java |    8 +-
 .../main/java/org/apache/hadoop/util/Time.java  |   18 +
 .../src/main/native/native.vcxproj              |    6 +-
 .../hadoop/io/erasurecode/jni_xor_decoder.c     |   80 +
 .../hadoop/io/erasurecode/jni_xor_encoder.c     |   82 +
 ...apache.hadoop.security.token.TokenIdentifier |   14 +
 .../src/main/resources/core-default.xml         |  164 +-
 .../src/site/markdown/DeprecatedProperties.md   |   24 -
 .../src/site/markdown/FileSystemShell.md        |    2 +-
 .../site/markdown/filesystem/introduction.md    |   15 +
 .../org/apache/hadoop/conf/TestConfServlet.java |    2 +-
 .../apache/hadoop/conf/TestConfigRedactor.java  |    2 +
 .../conf/TestConfigurationFieldsBase.java       |   47 +
 .../hadoop/fs/FSMainOperationsBaseTest.java     |    4 +-
 .../org/apache/hadoop/fs/TestDefaultUri.java    |  115 +
 .../org/apache/hadoop/fs/TestFsShellCopy.java   |   51 +
 .../fs/contract/AbstractContractCreateTest.java |   18 +-
 .../hadoop/fs/contract/ContractOptions.java     |   18 +
 .../hadoop/fs/contract/ContractTestUtils.java   |    6 +
 .../org/apache/hadoop/fs/shell/TestCount.java   |    9 +-
 .../TestFSMainOperationsLocalFileSystem.java    |    4 +-
 .../fs/viewfs/ViewFileSystemBaseTest.java       |   41 +-
 .../fs/viewfs/ViewFileSystemTestSetup.java      |   10 +-
 .../hadoop/fs/viewfs/ViewFsTestSetup.java       |   10 +-
 .../http/TestAuthenticationSessionCookie.java   |   11 +-
 .../apache/hadoop/http/TestHttpRequestLog.java  |    4 +-
 .../org/apache/hadoop/http/TestHttpServer.java  |   39 +-
 .../apache/hadoop/http/TestServletFilter.java   |    7 +-
 .../hadoop/http/resource/JerseyResource.java    |    5 +-
 .../rawcoder/TestNativeXORRawCoder.java         |   36 +
 .../erasurecode/rawcoder/TestXORRawCoder.java   |   38 +-
 .../rawcoder/TestXORRawCoderBase.java           |   59 +
 .../rawcoder/TestXORRawCoderInteroperable1.java |   36 +
 .../rawcoder/TestXORRawCoderInteroperable2.java |   37 +
 .../org/apache/hadoop/io/file/tfile/Timer.java  |   52 +-
 .../hadoop/metrics2/lib/TestMutableMetrics.java |  165 +-
 .../hadoop/security/TestUGIWithMiniKdc.java     |  144 +
 .../security/TestUserGroupInformation.java      |   93 +
 ...tionTokenAuthenticationHandlerWithMocks.java |   50 +
 .../delegation/web/TestWebDelegationToken.java  |   64 +-
 .../org/apache/hadoop/util/TestDiskChecker.java |  178 +-
 .../hadoop/util/TestInstrumentedLock.java       |  162 +
 .../util/TestInstrumentedReadWriteLock.java     |  234 +
 .../java/org/apache/hadoop/util/TestTime.java   |   50 +
 .../src/test/resources/contract/ftp.xml         |    7 +-
 .../dev-support/findbugsExcludeFile.xml         |    2 +-
 hadoop-common-project/hadoop-kms/pom.xml        |   20 +-
 .../hadoop/crypto/key/kms/server/KMS.java       |   21 +-
 .../key/kms/server/KMSAuthenticationFilter.java |   16 +-
 .../crypto/key/kms/server/KMSJSONWriter.java    |    3 +-
 .../hadoop-kms/src/site/markdown/index.md.vm    |    2 +-
 .../hadoop/crypto/key/kms/server/MiniKMS.java   |   63 +-
 .../hadoop/crypto/key/kms/server/TestKMS.java   |   19 +-
 hadoop-common-project/hadoop-nfs/pom.xml        |    2 +-
 hadoop-dist/pom.xml                             |   45 +
 hadoop-hdfs-project/hadoop-hdfs-client/pom.xml  |    6 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |   30 +-
 .../hadoop/hdfs/DFSOpsCountStatistics.java      |    1 +
 .../hadoop/hdfs/DFSStripedOutputStream.java     |    4 +-
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |   50 +
 .../hadoop/hdfs/protocol/DatanodeInfo.java      |  217 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |   22 +-
 .../server/datanode/DiskBalancerWorkItem.java   |    4 +-
 .../server/datanode/DiskBalancerWorkStatus.java |   13 +-
 .../hdfs/util/CombinedHostsFileReader.java      |    8 +-
 .../hdfs/util/CombinedHostsFileWriter.java      |    2 +-
 .../apache/hadoop/hdfs/web/JsonUtilClient.java  |   46 +-
 .../hadoop/hdfs/web/URLConnectionFactory.java   |    2 +-
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |   28 +-
 ...onfRefreshTokenBasedAccessTokenProvider.java |    4 +-
 .../CredentialBasedAccessTokenProvider.java     |    4 +-
 .../hadoop/hdfs/web/resources/GetOpParam.java   |    1 +
 .../hdfs/web/TestURLConnectionFactory.java      |   17 +
 .../hadoop/hdfs/web/TestWebHDFSOAuth2.java      |    2 +-
 ...ClientCredentialTimeBasedTokenRefresher.java |    2 +-
 ...TestRefreshTokenTimeBasedTokenRefresher.java |    2 +-
 hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml  |   26 +-
 .../hadoop/fs/http/client/HttpFSFileSystem.java |   31 +-
 .../hadoop/fs/http/server/FSOperations.java     |   23 +
 .../http/server/HttpFSParametersProvider.java   |    1 +
 .../hadoop/fs/http/server/HttpFSServer.java     |   19 +-
 .../org/apache/hadoop/lib/server/Server.java    |   15 +-
 .../apache/hadoop/lib/wsrs/JSONMapProvider.java |    3 +-
 .../apache/hadoop/lib/wsrs/JSONProvider.java    |    3 +-
 .../src/site/markdown/index.md                  |    2 +
 .../fs/http/client/BaseTestHttpFSWith.java      |  191 +-
 .../TestHttpFSFWithSWebhdfsFileSystem.java      |    6 +-
 .../hadoop/fs/http/server/TestHttpFSServer.java |   76 +-
 .../fs/http/server/TestHttpFSServerNoACLs.java  |    6 +-
 .../http/server/TestHttpFSServerNoXAttrs.java   |    6 +-
 .../fs/http/server/TestHttpFSWithKerberos.java  |    6 +-
 .../org/apache/hadoop/test/TestHFSTestCase.java |    8 +-
 .../org/apache/hadoop/test/TestHTestCase.java   |    8 +-
 .../org/apache/hadoop/test/TestJettyHelper.java |   56 +-
 .../src/test/resources/default-log4j.properties |    8 +-
 hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml     |   18 +-
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |   41 +-
 .../hadoop-hdfs/src/main/bin/hdfs               |    2 +-
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |    2 +
 .../apache/hadoop/hdfs/InstrumentedLock.java    |  185 -
 .../DatanodeProtocolServerSideTranslatorPB.java |    4 +-
 .../NamenodeProtocolServerSideTranslatorPB.java |    6 +-
 .../hdfs/qjournal/server/JournalNode.java       |    2 +-
 .../hadoop/hdfs/server/balancer/Balancer.java   |    5 +-
 .../hadoop/hdfs/server/balancer/Dispatcher.java |   49 +-
 .../server/blockmanagement/BlockIdManager.java  |    1 +
 .../blockmanagement/BlockManagerSafeMode.java   |   19 +-
 .../BlockPlacementPolicyDefault.java            |    8 +-
 .../CacheReplicationMonitor.java                |   14 +-
 .../server/blockmanagement/DatanodeManager.java |   16 +-
 .../hadoop/hdfs/server/common/Storage.java      |   82 +-
 .../server/datanode/BlockPoolSliceStorage.java  |   48 +-
 .../hdfs/server/datanode/BlockSender.java       |    7 +
 .../hadoop/hdfs/server/datanode/DataNode.java   |   25 +-
 .../hdfs/server/datanode/DataStorage.java       |   77 +-
 .../hdfs/server/datanode/DirectoryScanner.java  |   14 +-
 .../hdfs/server/datanode/LocalReplica.java      |    8 +-
 .../server/datanode/ReportBadBlockAction.java   |    4 +-
 .../hdfs/server/datanode/StorageLocation.java   |  108 +-
 .../server/datanode/checker/AsyncChecker.java   |   63 +
 .../hdfs/server/datanode/checker/Checkable.java |   49 +
 .../datanode/checker/ThrottledAsyncChecker.java |  224 +
 .../server/datanode/checker/package-info.java   |   26 +
 .../erasurecode/ErasureCodingWorker.java        |   23 +-
 .../StripedBlockChecksumReconstructor.java      |    1 +
 .../erasurecode/StripedBlockReconstructor.java  |    3 +
 .../erasurecode/StripedBlockWriter.java         |    4 +-
 .../server/datanode/fsdataset/FsDatasetSpi.java |   14 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |   54 +-
 .../datanode/fsdataset/impl/FsVolumeImpl.java   |   49 +-
 .../datanode/metrics/DataNodeMetrics.java       |   13 +-
 .../diskbalancer/DiskBalancerException.java     |    1 +
 .../server/diskbalancer/command/Command.java    |   23 +-
 .../diskbalancer/command/PlanCommand.java       |   15 +-
 .../connectors/JsonNodeConnector.java           |    4 +-
 .../datamodel/DiskBalancerCluster.java          |    8 +-
 .../datamodel/DiskBalancerVolume.java           |    8 +-
 .../datamodel/DiskBalancerVolumeSet.java        |    6 +-
 .../server/diskbalancer/planner/NodePlan.java   |    6 +-
 .../hdfs/server/namenode/CacheManager.java      |    8 +-
 .../server/namenode/EncryptionZoneManager.java  |    3 +-
 .../hadoop/hdfs/server/namenode/FSDirAclOp.java |   28 +-
 .../hdfs/server/namenode/FSDirAppendOp.java     |    3 +-
 .../hdfs/server/namenode/FSDirAttrOp.java       |   26 +-
 .../hdfs/server/namenode/FSDirConcatOp.java     |    8 +-
 .../hdfs/server/namenode/FSDirDeleteOp.java     |   15 +-
 .../server/namenode/FSDirEncryptionZoneOp.java  |    5 +-
 .../server/namenode/FSDirErasureCodingOp.java   |    5 +-
 .../hdfs/server/namenode/FSDirMkdirOp.java      |   18 +-
 .../hdfs/server/namenode/FSDirRenameOp.java     |   24 +-
 .../hdfs/server/namenode/FSDirSnapshotOp.java   |   17 +-
 .../server/namenode/FSDirStatAndListingOp.java  |   49 +-
 .../hdfs/server/namenode/FSDirSymlinkOp.java    |    3 +-
 .../hdfs/server/namenode/FSDirTruncateOp.java   |    9 +-
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |    3 +-
 .../hdfs/server/namenode/FSDirXAttrOp.java      |   12 +-
 .../hdfs/server/namenode/FSDirectory.java       |  191 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   54 +-
 .../hdfs/server/namenode/FSImageFormat.java     |   17 +-
 .../hdfs/server/namenode/FSNamesystem.java      |   33 +-
 .../server/namenode/FSPermissionChecker.java    |  165 +-
 .../hdfs/server/namenode/INodesInPath.java      |   90 +-
 .../hadoop/hdfs/server/namenode/NNStorage.java  |    4 +-
 .../hadoop/hdfs/server/namenode/NameNode.java   |    1 -
 .../hdfs/server/namenode/NameNodeRpcServer.java |    1 -
 .../hdfs/server/namenode/NamenodeFsck.java      |   15 +-
 .../server/namenode/StartupProgressServlet.java |    6 +-
 .../hdfs/server/namenode/TransferFsImage.java   |    2 +-
 .../namenode/snapshot/SnapshotManager.java      |    5 +-
 .../web/resources/NamenodeWebHdfsMethods.java   |   82 +-
 .../protocol/StorageReceivedDeletedBlocks.java  |    7 +
 .../hadoop/hdfs/tools/DiskBalancerCLI.java      |   69 +-
 .../org/apache/hadoop/hdfs/web/JsonUtil.java    |    2 +-
 .../impl/ContainerLocationManagerImpl.java      |    8 +-
 .../common/impl/ContainerManagerImpl.java       |    2 +-
 .../src/main/resources/hdfs-default.xml         |   39 +-
 .../src/site/markdown/HDFSDiskbalancer.md       |    1 +
 .../src/site/markdown/HDFSErasureCoding.md      |   11 +-
 .../hadoop-hdfs/src/site/markdown/WebHDFS.md    |   37 +-
 .../fs/viewfs/TestViewFsDefaultValue.java       |   12 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |   37 +-
 .../hadoop/hdfs/TestDFSClientSocketSize.java    |    6 +-
 .../apache/hadoop/hdfs/TestDatanodeReport.java  |   38 +-
 .../apache/hadoop/hdfs/TestDecommission.java    |    2 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  102 +-
 .../apache/hadoop/hdfs/TestExtendedAcls.java    |  441 ++
 .../org/apache/hadoop/hdfs/TestFileAppend.java  |   71 +
 .../apache/hadoop/hdfs/TestFileCorruption.java  |    7 +-
 .../org/apache/hadoop/hdfs/TestFileStatus.java  |    4 +-
 .../hadoop/hdfs/TestInstrumentedLock.java       |  166 -
 .../java/org/apache/hadoop/hdfs/TestQuota.java  | 2104 ++++----
 .../hadoop/hdfs/TestReservedRawPaths.java       |    5 +-
 .../hdfs/TestSecureEncryptionZoneWithKMS.java   |    9 +-
 .../client/impl/TestBlockReaderFactory.java     |    6 +-
 .../sasl/SaslDataTransferTestCase.java          |    9 +-
 .../hadoop/hdfs/protocolPB/TestPBHelper.java    |    4 +-
 .../hdfs/qjournal/TestSecureNNWithQJM.java      |   11 +-
 .../qjournal/server/TestJournalNodeMXBean.java  |    2 +-
 .../hdfs/server/balancer/TestBalancer.java      |   16 +-
 .../TestBlockManagerSafeMode.java               |   20 +
 .../blockmanagement/TestBlockStatsMXBean.java   |    2 +-
 .../blockmanagement/TestDatanodeManager.java    |   37 +
 .../TestNameNodePrunesMissingStorages.java      |    7 +-
 .../TestPendingReconstruction.java              |   12 +-
 .../TestSequentialBlockGroupId.java             |    5 +
 .../server/datanode/SimulatedFSDataset.java     |    5 -
 .../hdfs/server/datanode/TestBlockRecovery.java |    6 +-
 .../hdfs/server/datanode/TestBlockScanner.java  |    6 +-
 .../hdfs/server/datanode/TestDataDirs.java      |    3 +-
 .../TestDataNodeErasureCodingMetrics.java       |   43 +-
 .../datanode/TestDataNodeHotSwapVolumes.java    |   24 +-
 .../server/datanode/TestDataNodeMXBean.java     |    6 +-
 .../datanode/TestDataNodeVolumeFailure.java     |  177 +-
 .../TestDataNodeVolumeFailureReporting.java     |   37 +-
 .../hdfs/server/datanode/TestDataStorage.java   |    7 +-
 .../server/datanode/TestDirectoryScanner.java   |    4 +-
 .../hdfs/server/datanode/TestDiskError.java     |    3 +-
 .../datanode/TestIncrementalBrVariations.java   |    4 +-
 .../checker/TestThrottledAsyncChecker.java      |  276 ++
 .../extdataset/ExternalDatasetImpl.java         |    5 -
 .../fsdataset/impl/FsDatasetImplTestUtils.java  |   14 +
 .../fsdataset/impl/FsDatasetTestUtil.java       |    2 +-
 .../impl/TestInterDatanodeProtocol.java         |    4 +-
 .../diskbalancer/DiskBalancerTestUtil.java      |   21 +-
 .../diskbalancer/TestDiskBalancerRPC.java       |    2 +-
 .../command/TestDiskBalancerCommand.java        |  178 +-
 .../hdfs/server/namenode/FSAclBaseTest.java     |    7 +-
 .../hdfs/server/namenode/FSXAttrBaseTest.java   |    3 +-
 .../server/namenode/NNThroughputBenchmark.java  |    5 +-
 .../hdfs/server/namenode/NameNodeAdapter.java   |    5 +-
 .../hdfs/server/namenode/TestAuditLogs.java     |    9 +
 .../hdfs/server/namenode/TestDeadDatanode.java  |    3 +-
 .../namenode/TestDiskspaceQuotaUpdate.java      |   19 +-
 .../hdfs/server/namenode/TestFSDirectory.java   |   37 +-
 .../server/namenode/TestFSNamesystemMBean.java  |    2 +-
 .../namenode/TestFSPermissionChecker.java       |    5 +-
 .../hdfs/server/namenode/TestFileTruncate.java  |    5 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |   12 +-
 .../server/namenode/TestGetBlockLocations.java  |    5 +-
 .../namenode/TestNameNodeConfiguration.java     |   40 +
 .../server/namenode/TestNameNodeMXBean.java     |    4 +-
 .../TestNameNodeMetadataConsistency.java        |   45 +-
 .../TestNameNodeRespectsBindHostKeys.java       |   10 +-
 .../server/namenode/TestQuotaByStorageType.java |  166 +-
 .../server/namenode/TestSnapshotPathINodes.java |    8 +
 .../hdfs/server/namenode/TestStartup.java       |   55 +-
 .../namenode/TestStartupProgressServlet.java    |    2 +-
 .../server/namenode/TestTransferFsImage.java    |    2 +-
 .../namenode/snapshot/SnapshotTestHelper.java   |   10 +-
 .../snapshot/TestSnapshotReplication.java       |    3 +-
 .../shortcircuit/TestShortCircuitCache.java     |   11 +-
 .../apache/hadoop/hdfs/tools/TestDFSAdmin.java  |  145 +-
 .../apache/hadoop/hdfs/web/TestJsonUtil.java    |    4 +-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   43 +-
 .../hadoop/hdfs/web/TestWebHDFSForHA.java       |    2 +-
 .../hadoop/hdfs/web/TestWebHdfsTokens.java      |    5 +-
 .../apache/hadoop/security/TestPermission.java  |   10 +-
 .../hadoop/security/TestPermissionSymlinks.java |    7 +-
 .../hadoop/test/MiniDFSClusterManager.java      |    2 +-
 .../hadoop-mapreduce-client-app/pom.xml         |    4 +
 .../jobhistory/JobHistoryEventHandler.java      |    4 +-
 .../hadoop/mapreduce/v2/app/JobEndNotifier.java |   37 +-
 .../v2/app/rm/RMContainerAllocator.java         |  201 +-
 .../mapreduce/v2/app/webapp/AMWebServices.java  |   49 +-
 .../jobhistory/TestJobHistoryEventHandler.java  |    5 +-
 .../v2/app/rm/TestRMContainerAllocator.java     |  218 +-
 .../v2/app/webapp/TestAMWebServices.java        |   31 +-
 .../v2/app/webapp/TestAMWebServicesAttempt.java |   13 +-
 .../app/webapp/TestAMWebServicesAttempts.java   |   34 +-
 .../v2/app/webapp/TestAMWebServicesJobConf.java |   13 +-
 .../v2/app/webapp/TestAMWebServicesJobs.java    |   64 +-
 .../v2/app/webapp/TestAMWebServicesTasks.java   |   61 +-
 .../hadoop-mapreduce-client-common/pom.xml      |    2 -
 .../hadoop-mapreduce-client-core/pom.xml        |    4 +
 .../apache/hadoop/mapred/IFileOutputStream.java |    8 +-
 .../java/org/apache/hadoop/mapred/JobConf.java  |   11 +-
 .../org/apache/hadoop/mapred/QueueManager.java  |    8 +-
 .../hadoop/mapred/pipes/BinaryProtocol.java     |   14 +-
 .../hadoop/mapreduce/JobSubmissionFiles.java    |   63 +-
 .../apache/hadoop/mapreduce/JobSubmitter.java   |   17 +-
 .../mapreduce/server/jobtracker/JTConfig.java   |   81 +-
 .../hadoop/mapreduce/task/reduce/Fetcher.java   |   36 +
 .../hadoop/mapreduce/task/reduce/MapHost.java   |    4 -
 .../task/reduce/ShuffleSchedulerImpl.java       |   43 +-
 .../hadoop/mapreduce/util/ConfigUtil.java       |   53 -
 .../mapreduce/util/JobHistoryEventUtils.java    |    9 +-
 .../src/main/resources/mapred-default.xml       |   13 +
 .../mapreduce/TestJobSubmissionFiles.java       |  142 +
 .../mapreduce/task/reduce/TestFetcher.java      |   22 +
 .../mapreduce/v2/hs/webapp/HsTasksPage.java     |    3 +-
 .../mapreduce/v2/hs/webapp/HsWebServices.java   |   40 +-
 .../src/site/markdown/HistoryServerRest.md      |    2 +-
 .../mapreduce/v2/hs/TestJobHistoryParsing.java  |    2 +
 .../v2/hs/webapp/TestHsWebServices.java         |   25 +-
 .../v2/hs/webapp/TestHsWebServicesAttempts.java |   34 +-
 .../v2/hs/webapp/TestHsWebServicesJobConf.java  |   13 +-
 .../v2/hs/webapp/TestHsWebServicesJobs.java     |   67 +-
 .../hs/webapp/TestHsWebServicesJobsQuery.java   |   76 +-
 .../v2/hs/webapp/TestHsWebServicesTasks.java    |   61 +-
 .../resources/job_1329348432655_0001_conf.xml   |   15 -
 .../hadoop/ipc/TestMRCJCSocketFactory.java      |    5 +-
 .../hadoop/mapred/NotificationTestCase.java     |   12 +-
 .../hadoop/mapred/TestClientRedirect.java       |    9 +
 .../apache/hadoop/mapred/TestIFileStreams.java  |   20 +
 .../mapreduce/MiniHadoopClusterManager.java     |    2 +-
 .../apache/hadoop/mapreduce/v2/TestMRJobs.java  |    5 +-
 .../hadoop/mapreduce/v2/TestRMNMInfo.java       |    4 +-
 .../hadoop-mapreduce-client-shuffle/pom.xml     |    2 -
 .../apache/hadoop/mapred/ShuffleHandler.java    |   41 +-
 .../hadoop/mapred/TestShuffleHandler.java       |   42 +-
 .../hadoop-mapreduce-client/pom.xml             |    4 +-
 hadoop-mapreduce-project/pom.xml                |    4 +-
 hadoop-maven-plugins/pom.xml                    |    4 +
 .../hadoop/maven/plugin/protoc/ProtocMojo.java  |  228 +-
 .../maven/plugin/protoc/ProtocRunner.java       |  283 ++
 .../maven/plugin/protoc/ProtocTestMojo.java     |   61 +
 .../maven/plugin/protoc/package-info.java       |   98 +
 .../maven/plugin/resourcegz/ResourceGzMojo.java |  131 +
 hadoop-project-dist/pom.xml                     |    2 +-
 hadoop-project/pom.xml                          |   62 +-
 hadoop-project/src/site/site.xml                |    1 -
 .../apache/hadoop/tools/TestHadoopArchives.java |    2 +-
 hadoop-tools/hadoop-aws/pom.xml                 |    7 -
 .../fs/s3a/AWSCredentialProviderList.java       |   11 +
 .../java/org/apache/hadoop/fs/s3a/Listing.java  |    5 +-
 .../org/apache/hadoop/fs/s3a/S3AFileStatus.java |   33 +-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |   46 +-
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  155 +-
 ...haredInstanceProfileCredentialsProvider.java |   67 +
 .../src/site/markdown/tools/hadoop-aws/index.md |  108 +-
 .../fs/contract/s3a/ITestS3AContractDistCp.java |    6 +
 .../hadoop/fs/contract/s3a/S3AContract.java     |    6 +-
 .../hadoop/fs/s3a/AbstractS3ATestBase.java      |   26 +-
 .../fs/s3a/ITestS3AAWSCredentialsProvider.java  |  113 +-
 .../fs/s3a/ITestS3ABlockingThreadPool.java      |   82 -
 .../apache/hadoop/fs/s3a/ITestS3ABlocksize.java |   19 +-
 .../hadoop/fs/s3a/ITestS3AConfiguration.java    |   59 +-
 .../hadoop/fs/s3a/ITestS3AEncryption.java       |    9 +-
 .../ITestS3AEncryptionAlgorithmPropagation.java |    7 -
 .../hadoop/fs/s3a/ITestS3AFailureHandling.java  |   11 +-
 .../fs/s3a/ITestS3AFileOperationCost.java       |   19 +-
 .../fs/s3a/ITestS3AFileSystemContract.java      |   33 +-
 .../fs/s3a/ITestS3ATemporaryCredentials.java    |   14 +-
 .../apache/hadoop/fs/s3a/S3ATestConstants.java  |   27 +-
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |   58 +-
 .../fs/s3a/TestS3AAWSCredentialsProvider.java   |  273 ++
 .../fs/s3a/scale/AbstractSTestS3AHugeFiles.java |   25 +-
 .../s3a/scale/ITestS3ADeleteFilesOneByOne.java  |   12 +-
 .../fs/s3a/scale/ITestS3ADeleteManyFiles.java   |   13 +-
 .../s3a/scale/ITestS3ADirectoryPerformance.java |   16 +-
 .../scale/ITestS3AHugeFilesClassicOutput.java   |    4 +-
 .../scale/ITestS3AInputStreamPerformance.java   |    3 +-
 .../hadoop/fs/s3a/scale/S3AScaleTestBase.java   |  139 +-
 .../org/apache/hadoop/fs/s3a/yarn/ITestS3A.java |    4 +-
 .../fs/s3a/yarn/ITestS3AMiniYarnCluster.java    |   50 +-
 .../src/test/resources/contract/s3a.xml         |    5 +
 .../src/test/resources/contract/s3n.xml         |   10 +
 hadoop-tools/hadoop-azure-datalake/pom.xml      |    2 +-
 hadoop-tools/hadoop-azure/pom.xml               |    7 +-
 .../fs/azure/AzureNativeFileSystemStore.java    |    2 +-
 .../hadoop/fs/azure/BlockBlobAppendStream.java  |    6 +-
 ...stNativeAzureFileSystemOperationsMocked.java |    9 +
 .../hadoop/tools/CopyListingFileStatus.java     |  129 +-
 .../apache/hadoop/tools/mapred/CopyMapper.java  |   27 +-
 .../tools/mapred/RetriableFileCopyCommand.java  |   56 +-
 .../hadoop/tools/util/ThrottledInputStream.java |    3 +
 .../src/site/markdown/DistCp.md.vm              |    3 +-
 .../hadoop/tools/TestCopyListingFileStatus.java |   67 +
 .../mapred/TestRetriableFileCopyCommand.java    |    5 +-
 .../hadoop/mapred/gridmix/GridmixTestUtils.java |    5 +-
 hadoop-tools/hadoop-openstack/pom.xml           |    2 +-
 .../swift/http/HttpInputStreamWithRelease.java  |    3 +
 .../swift/snative/SwiftNativeInputStream.java   |    3 +
 .../fs/swift/TestFSMainOperationsSwift.java     |    6 +
 .../src/test/resources/contract/swift.xml       |   10 +
 hadoop-tools/hadoop-sls/pom.xml                 |   10 +-
 .../src/main/data/2jobs2min-rumen-jh.json       |   52 +-
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |    9 +-
 .../yarn/sls/scheduler/RMNodeWrapper.java       |    9 +-
 .../apache/hadoop/yarn/sls/web/SLSWebApp.java   |   19 +-
 .../dev-support/findbugs-exclude.xml            |    9 +
 .../hadoop-yarn/hadoop-yarn-api/pom.xml         |    2 -
 .../yarn/api/ApplicationClientProtocol.java     |   23 +
 .../UpdateApplicationTimeoutsRequest.java       |   81 +
 .../UpdateApplicationTimeoutsResponse.java      |   46 +
 .../records/ApplicationResourceUsageReport.java |   43 +-
 .../records/ApplicationSubmissionContext.java   |    4 +
 .../yarn/api/records/ResourceRequest.java       |    2 +-
 .../hadoop/yarn/conf/YarnConfiguration.java     |   32 +-
 .../main/proto/applicationclient_protocol.proto |    1 +
 .../src/main/proto/yarn_protos.proto            |    7 +
 .../src/main/proto/yarn_service_protos.proto    |    9 +
 .../yarn/conf/TestYarnConfigurationFields.java  |   15 +-
 .../distributedshell/TestDistributedShell.java  |    9 +-
 .../TestDistributedShellWithNodeLabels.java     |   12 +-
 .../hadoop-yarn/hadoop-yarn-client/pom.xml      |    2 -
 .../yarn/client/api/impl/AMRMClientImpl.java    |   23 +-
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |    7 +
 .../hadoop/yarn/client/cli/RMAdminCLI.java      |  191 +-
 ...stHedgingRequestRMFailoverProxyProvider.java |   31 +-
 .../yarn/client/api/impl/TestAMRMClient.java    |   22 +-
 .../TestOpportunisticContainerAllocation.java   |   71 +-
 .../hadoop/yarn/client/cli/TestRMAdminCLI.java  |   19 +-
 .../hadoop/yarn/client/cli/TestYarnCLI.java     |   18 +-
 .../hadoop-yarn/hadoop-yarn-common/pom.xml      |   23 +-
 .../ApplicationClientProtocolPBClientImpl.java  |   21 +-
 .../ApplicationClientProtocolPBServiceImpl.java |   22 +
 .../UpdateApplicationTimeoutsRequestPBImpl.java |  220 +
 ...UpdateApplicationTimeoutsResponsePBImpl.java |   73 +
 .../ApplicationResourceUsageReportPBImpl.java   |   28 +
 .../RequestHedgingRMFailoverProxyProvider.java  |   90 +-
 .../client/api/impl/TimelineClientImpl.java     |    8 +-
 .../hadoop/yarn/event/AsyncDispatcher.java      |    1 +
 .../logaggregation/AggregatedLogFormat.java     |    7 +-
 .../yarn/logaggregation/LogCLIHelpers.java      |    7 +
 .../yarn/util/AbstractLivelinessMonitor.java    |   17 +-
 .../yarn/util/ResourceCalculatorPlugin.java     |   39 +
 .../java/org/apache/hadoop/yarn/util/Times.java |   33 +
 .../org/apache/hadoop/yarn/webapp/WebApps.java  |    8 +
 .../static/dt-1.9.4/js/jquery.dataTables.min.js |  157 +
 .../dt-1.9.4/js/jquery.dataTables.min.js.gz     |  Bin 21726 -> 0 bytes
 .../webapps/static/jquery/jquery-1.8.2.min.js   |    2 +
 .../static/jquery/jquery-1.8.2.min.js.gz        |  Bin 33430 -> 0 bytes
 .../static/jquery/jquery-ui-1.9.1.custom.min.js |    6 +
 .../jquery/jquery-ui-1.9.1.custom.min.js.gz     |  Bin 62855 -> 0 bytes
 .../webapps/static/jt/jquery.jstree.js          | 4544 ++++++++++++++++++
 .../webapps/static/jt/jquery.jstree.js.gz       |  Bin 37540 -> 0 bytes
 .../src/main/resources/yarn-default.xml         |   44 +-
 .../hadoop/yarn/webapp/MyTestWebService.java    |    3 +-
 .../client/impl/zk/RegistrySecurity.java        |   14 +-
 .../registry/secure/KerberosConfiguration.java  |   39 +-
 .../registry/secure/TestSecureLogins.java       |   99 +-
 .../pom.xml                                     |    4 +-
 ...pplicationHistoryManagerOnTimelineStore.java |    8 +-
 .../ApplicationHistoryServer.java               |   50 +-
 .../webapp/AHSWebServices.java                  |   27 +-
 .../timeline/webapp/TimelineWebServices.java    |   23 +-
 ...pplicationHistoryManagerOnTimelineStore.java |   11 +-
 .../webapp/TestAHSWebServices.java              |   22 +-
 .../TestRollingLevelDBTimelineStore.java        |    6 +-
 ...TimelineAuthenticationFilterInitializer.java |   55 +-
 .../webapp/TestTimelineWebServices.java         |  141 +-
 .../hadoop-yarn-server-common/pom.xml           |    2 -
 .../DistributedSchedulingAllocateResponse.java  |    6 +-
 .../protocolrecords/NodeHeartbeatResponse.java  |   95 +-
 ...RegisterDistributedSchedulingAMResponse.java |    6 +-
 .../RegisterNodeManagerRequest.java             |   24 +
 .../RegisterNodeManagerResponse.java            |   40 +-
 .../server/api/protocolrecords/RemoteNode.java  |   90 +
 ...ributedSchedulingAllocateResponsePBImpl.java |   41 +-
 .../impl/pb/NodeHeartbeatResponsePBImpl.java    |   15 +-
 ...erDistributedSchedulingAMResponsePBImpl.java |   39 +-
 .../pb/RegisterNodeManagerRequestPBImpl.java    |   30 +-
 .../pb/RegisterNodeManagerResponsePBImpl.java   |   15 +-
 .../impl/pb/RemoteNodePBImpl.java               |  135 +
 .../yarn/server/api/records/NodeStatus.java     |    7 +-
 .../records/OpportunisticContainersStatus.java  |  152 +
 .../api/records/QueuedContainersStatus.java     |   45 -
 .../api/records/impl/pb/NodeStatusPBImpl.java   |   38 +-
 .../pb/OpportunisticContainersStatusPBImpl.java |  139 +
 .../impl/pb/QueuedContainersStatusPBImpl.java   |   84 -
 .../metrics/ApplicationMetricsConstants.java    |    6 +
 .../OpportunisticContainerAllocator.java        |   79 +-
 .../OpportunisticContainerContext.java          |   14 +-
 .../hadoop/yarn/server/utils/BuilderUtils.java  |    5 +-
 .../main/proto/yarn_server_common_protos.proto  |   12 +-
 .../yarn_server_common_service_protos.proto     |   10 +-
 .../hadoop/yarn/TestYarnServerApiClasses.java   |    3 +
 .../protocolrecords/TestProtocolRecords.java    |   17 +-
 .../hadoop-yarn-server-nodemanager/pom.xml      |    6 +-
 .../server/nodemanager/ContainerExecutor.java   |    1 -
 .../nodemanager/DefaultContainerExecutor.java   |   24 +-
 .../server/nodemanager/DeletionService.java     |   13 +-
 .../nodemanager/DockerContainerExecutor.java    |  883 ----
 .../nodemanager/LinuxContainerExecutor.java     |    1 +
 .../yarn/server/nodemanager/NodeManager.java    |    4 +-
 .../nodemanager/NodeResourceMonitorImpl.java    |    6 +-
 .../nodemanager/NodeStatusUpdaterImpl.java      |   34 +-
 .../nodemanager/amrmproxy/AMRMProxyService.java |    4 +-
 .../containermanager/ContainerManager.java      |    3 +
 .../containermanager/ContainerManagerImpl.java  |   10 +-
 .../application/ApplicationImpl.java            |    6 +-
 .../container/ContainerImpl.java                |   29 +-
 .../launcher/ContainerLaunch.java               |   36 +-
 .../runtime/DockerLinuxContainerRuntime.java    |   14 +-
 .../localizer/LocalCacheCleaner.java            |  182 +
 .../localizer/LocalResourcesTrackerImpl.java    |    3 +-
 .../localizer/LocalizedResource.java            |    5 +-
 .../localizer/ResourceLocalizationService.java  |   93 +-
 .../localizer/ResourceRetentionSet.java         |   96 -
 .../security/LocalizerTokenSelector.java        |    4 +-
 .../logaggregation/LogAggregationService.java   |   15 +
 .../monitor/ContainersMonitorImpl.java          |  462 +-
 .../queuing/QueuingContainerManagerImpl.java    |   32 +
 .../scheduler/DistributedScheduler.java         |   22 +-
 .../security/NMTokenSecretManagerInNM.java      |   16 +-
 .../nodemanager/util/ProcessIdFileReader.java   |   12 +-
 .../nodemanager/webapp/ContainerPage.java       |    1 +
 .../nodemanager/webapp/NMWebServices.java       |   28 +-
 .../nodemanager/webapp/dao/ContainerInfo.java   |    7 +
 .../container-executor/impl/configuration.c     |   41 +-
 .../container-executor/impl/configuration.h     |    9 +
 .../main/native/container-executor/impl/main.c  |    5 +-
 .../test/test-container-executor.c              |   77 +-
 .../TestDefaultContainerExecutor.java           |   34 +
 .../TestDockerContainerExecutor.java            |  244 -
 .../TestDockerContainerExecutorWithMocks.java   |  305 --
 .../nodemanager/TestNodeStatusUpdater.java      |    6 +-
 .../amrmproxy/MockResourceManagerFacade.java    |   20 +-
 .../BaseContainerManagerTest.java               |   38 +-
 .../localizer/TestLocalCacheCleanup.java        |  235 +
 .../TestLocalResourcesTrackerImpl.java          |    4 +-
 .../localizer/TestResourceRetention.java        |  106 -
 .../TestLogAggregationService.java              |   28 +-
 .../queuing/TestQueuingContainerManager.java    |    6 +-
 .../scheduler/TestDistributedScheduler.java     |   20 +-
 .../nodemanager/webapp/TestNMWebServices.java   |   27 +-
 .../webapp/TestNMWebServicesApps.java           |   49 +-
 .../webapp/TestNMWebServicesContainers.java     |   27 +-
 .../conf/capacity-scheduler.xml                 |   12 +
 .../hadoop-yarn-server-resourcemanager/pom.xml  |   16 +-
 .../server/resourcemanager/AdminService.java    |    6 +-
 .../ApplicationMasterService.java               |    3 +-
 .../server/resourcemanager/ClientRMService.java |  137 +-
 .../resourcemanager/NodesListManager.java       |    4 +-
 ...pportunisticContainerAllocatorAMService.java |   58 +-
 .../server/resourcemanager/RMAppManager.java    |   39 +
 .../server/resourcemanager/RMAuditLogger.java   |    4 +-
 .../yarn/server/resourcemanager/RMNMInfo.java   |    2 +-
 .../server/resourcemanager/RMServerUtils.java   |   50 +-
 .../server/resourcemanager/ResourceManager.java |   90 +-
 .../resourcemanager/ResourceTrackerService.java |    3 +-
 .../metrics/TimelineServiceV1Publisher.java     |    4 +
 .../metrics/TimelineServiceV2Publisher.java     |    6 +
 .../AbstractPreemptableResourceCalculator.java  |  244 +
 .../capacity/AbstractPreemptionEntity.java      |   98 +
 .../CapacitySchedulerPreemptionContext.java     |   14 +
 .../CapacitySchedulerPreemptionUtils.java       |  119 +-
 .../capacity/FifoCandidatesSelector.java        |  127 +-
 .../FifoIntraQueuePreemptionPlugin.java         |  459 ++
 .../capacity/IntraQueueCandidatesSelector.java  |  238 +
 .../IntraQueuePreemptionComputePlugin.java      |   39 +
 .../capacity/PreemptableResourceCalculator.java |  183 +-
 .../capacity/PreemptionCandidatesSelector.java  |   29 +-
 .../ProportionalCapacityPreemptionPolicy.java   |   86 +-
 .../monitor/capacity/TempAppPerPartition.java   |  101 +
 .../monitor/capacity/TempQueuePerPartition.java |  142 +-
 .../resourcemanager/recovery/RMStateStore.java  |   38 +-
 .../recovery/RMStateUpdateAppEvent.java         |   15 +-
 .../records/ApplicationAttemptStateData.java    |   39 +-
 .../recovery/records/ApplicationStateData.java  |   27 +
 .../pb/ApplicationAttemptStateDataPBImpl.java   |   24 +
 .../impl/pb/ApplicationStateDataPBImpl.java     |   86 +
 .../resourcemanager/resource/Priority.java      |   31 -
 .../server/resourcemanager/rmapp/RMApp.java     |    3 +
 .../server/resourcemanager/rmapp/RMAppImpl.java |   77 +-
 .../resourcemanager/rmapp/RMAppMetrics.java     |   16 +-
 .../rmapp/attempt/RMAppAttemptImpl.java         |   11 +-
 .../rmapp/attempt/RMAppAttemptMetrics.java      |   16 +
 .../rmapp/monitor/RMAppLifetimeMonitor.java     |   72 +-
 .../rmcontainer/RMContainer.java                |   13 +
 .../rmcontainer/RMContainerImpl.java            |   94 +-
 .../server/resourcemanager/rmnode/RMNode.java   |   10 +-
 .../resourcemanager/rmnode/RMNodeImpl.java      |   72 +-
 .../rmnode/RMNodeStatusEvent.java               |    6 +-
 .../scheduler/AbstractYarnScheduler.java        |   24 +-
 .../scheduler/AppSchedulingInfo.java            |  168 +-
 .../scheduler/SchedulerApplicationAttempt.java  |   75 +-
 .../scheduler/activities/ActivitiesLogger.java  |   17 +-
 .../scheduler/activities/ActivitiesManager.java |    7 +-
 .../scheduler/capacity/AbstractCSQueue.java     |   71 +
 ...CSAMContainerLaunchDiagnosticsConstants.java |    2 +
 .../scheduler/capacity/CSAssignment.java        |   33 +
 .../scheduler/capacity/CSQueue.java             |   19 +-
 .../scheduler/capacity/CapacityScheduler.java   |  891 +++-
 .../CapacitySchedulerConfiguration.java         |   71 +
 .../scheduler/capacity/LeafQueue.java           |  540 ++-
 .../scheduler/capacity/ParentQueue.java         |  416 +-
 .../allocator/AbstractContainerAllocator.java   |   39 +-
 .../capacity/allocator/ContainerAllocation.java |   12 +-
 .../capacity/allocator/ContainerAllocator.java  |   15 +-
 .../allocator/IncreaseContainerAllocator.java   |   89 +-
 .../allocator/RegularContainerAllocator.java    |  215 +-
 .../scheduler/common/AssignmentInformation.java |   44 +-
 .../common/ContainerAllocationProposal.java     |  111 +
 .../common/ResourceAllocationCommitter.java     |   29 +
 .../scheduler/common/ResourceCommitRequest.java |  164 +
 .../scheduler/common/SchedulerContainer.java    |   80 +
 .../scheduler/common/fica/FiCaSchedulerApp.java |  640 ++-
 .../distributed/NodeQueueLoadMonitor.java       |   10 +-
 .../scheduler/fair/FSAppAttempt.java            |  132 +-
 .../scheduler/fair/FSLeafQueue.java             |   15 +-
 .../scheduler/fair/FSParentQueue.java           |    6 +-
 .../scheduler/fair/FairScheduler.java           |    4 +-
 .../scheduler/fair/MaxRunningAppsEnforcer.java  |   50 +-
 .../fair/policies/FairSharePolicy.java          |   31 +-
 .../scheduler/fair/policies/FifoPolicy.java     |    5 +-
 .../scheduler/fifo/FifoAppAttempt.java          |  110 +
 .../scheduler/fifo/FifoScheduler.java           |   55 +-
 .../scheduler/placement/PlacementSet.java       |   65 +
 .../scheduler/placement/PlacementSetUtils.java  |   36 +
 .../placement/ResourceRequestUpdateResult.java  |   43 +
 .../placement/SchedulingPlacementSet.java       |   90 +
 .../scheduler/placement/SimplePlacementSet.java |   70 +
 .../AbstractComparatorOrderingPolicy.java       |    4 +-
 .../scheduler/policy/FairOrderingPolicy.java    |    3 +-
 .../scheduler/policy/FifoOrderingPolicy.java    |    4 +-
 .../FifoOrderingPolicyForPendingApps.java       |    3 +-
 .../resourcemanager/webapp/NodesPage.java       |   75 +-
 .../resourcemanager/webapp/RMAppBlock.java      |    8 +-
 .../resourcemanager/webapp/RMWebServices.java   |  121 +-
 .../resourcemanager/webapp/dao/AppInfo.java     |   34 +-
 .../webapp/dao/ClusterMetricsInfo.java          |   11 +-
 .../resourcemanager/webapp/dao/NodeInfo.java    |   41 +-
 .../yarn_server_resourcemanager_recovery.proto  |    3 +
 .../yarn/server/resourcemanager/MockNM.java     |    8 +-
 .../yarn/server/resourcemanager/MockNodes.java  |   21 +-
 .../yarn/server/resourcemanager/MockRM.java     |   91 +-
 .../ReservationACLsTestBase.java                |    7 +-
 .../server/resourcemanager/TestAppManager.java  |   13 +-
 .../resourcemanager/TestClientRMService.java    |   23 +-
 ...pportunisticContainerAllocatorAMService.java |    6 +-
 .../yarn/server/resourcemanager/TestRM.java     |    3 +-
 .../resourcemanager/TestRMAdminService.java     |   40 +-
 .../yarn/server/resourcemanager/TestRMHA.java   |    4 +-
 .../resourcemanager/TestRMNodeTransitions.java  |   40 +
 .../resourcemanager/TestRMProxyUsersConf.java   |   49 +-
 .../server/resourcemanager/TestRMRestart.java   |   19 +
 .../resourcemanager/TestResourceManager.java    |    6 +-
 .../TestResourceTrackerService.java             |   55 +
 .../TestWorkPreservingRMRestart.java            |    3 +-
 .../ahs/TestRMApplicationHistoryWriter.java     |    5 +-
 .../applicationsmanager/MockAsm.java            |   10 +-
 .../metrics/TestSystemMetricsPublisher.java     |   13 +-
 .../TestSystemMetricsPublisherForV2.java        |    4 +-
 ...alCapacityPreemptionPolicyMockFramework.java |  126 +-
 ...estProportionalCapacityPreemptionPolicy.java |    7 +-
 ...ionalCapacityPreemptionPolicyIntraQueue.java |  868 ++++
 .../recovery/RMStateStoreTestBase.java          |    4 +-
 .../recovery/TestZKRMStateStore.java            |    2 +-
 .../planning/TestAlignedPlanner.java            |    4 +-
 .../planning/TestGreedyReservationAgent.java    |    4 +-
 .../server/resourcemanager/rmapp/MockRMApp.java |    6 +
 .../rmapp/TestApplicationLifetimeMonitor.java   |  150 +-
 .../scheduler/TestSchedulerHealth.java          |   22 +-
 .../capacity/TestApplicationLimits.java         |  185 +-
 .../capacity/TestApplicationPriority.java       |  101 +-
 .../capacity/TestCapacityScheduler.java         |  103 +-
 .../TestCapacitySchedulerAsyncScheduling.java   |  143 +
 .../TestCapacitySchedulerNodeLabelUpdate.java   |   74 +-
 .../TestCapacitySchedulerQueueACLs.java         |    3 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   21 +-
 .../capacity/TestContainerAllocation.java       |   45 +-
 .../capacity/TestContainerResizing.java         |   10 +-
 .../scheduler/capacity/TestLeafQueue.java       |  653 ++-
 .../TestNodeLabelContainerAllocation.java       |    4 +-
 .../scheduler/capacity/TestParentQueue.java     |  191 +-
 .../scheduler/capacity/TestReservations.java    |  283 +-
 .../scheduler/capacity/TestUtils.java           |   26 +
 .../distributed/TestNodeQueueLoadMonitor.java   |    8 +-
 .../fair/TestContinuousScheduling.java          |  189 +-
 .../scheduler/fair/TestFairScheduler.java       |  253 +-
 .../fair/TestFairSchedulerEventLog.java         |    2 +-
 .../fair/TestFairSchedulerQueueACLs.java        |    2 +-
 .../fair/TestMaxRunningAppsEnforcer.java        |    2 +-
 .../scheduler/fair/TestSchedulingPolicy.java    |  228 +
 .../scheduler/fifo/TestFifoScheduler.java       |   12 +-
 .../security/TestAMRMTokens.java                |    4 +
 .../resourcemanager/webapp/TestAppPage.java     |    3 +-
 .../resourcemanager/webapp/TestNodesPage.java   |   33 +
 .../resourcemanager/webapp/TestRMWebApp.java    |    1 +
 .../webapp/TestRMWebAppFairScheduler.java       |    3 +-
 .../webapp/TestRMWebServiceAppsNodelabel.java   |  230 +
 .../webapp/TestRMWebServices.java               |   46 +-
 .../webapp/TestRMWebServicesApps.java           |  153 +-
 .../TestRMWebServicesAppsModification.java      |   45 +-
 .../webapp/TestRMWebServicesCapacitySched.java  |   19 +-
 ...ebServicesDelegationTokenAuthentication.java |    6 +-
 .../TestRMWebServicesDelegationTokens.java      |    4 +-
 .../webapp/TestRMWebServicesFairScheduler.java  |   10 +-
 .../TestRMWebServicesForCSWithPartitions.java   |   13 +-
 ...tRMWebServicesHttpStaticUserPermissions.java |    4 +-
 .../webapp/TestRMWebServicesNodeLabels.java     |   55 +-
 .../webapp/TestRMWebServicesNodes.java          |   99 +-
 .../webapp/TestRMWebServicesReservation.java    |   18 +-
 .../TestRMWebServicesSchedulerActivities.java   |   99 +-
 .../yarn/webapp/TestRMWithCSRFFilter.java       |    7 +-
 .../hadoop-yarn-server-tests/pom.xml            |    6 +-
 .../hadoop/yarn/server/TestDiskFailures.java    |    4 +-
 .../pom.xml                                     |    2 +-
 .../pom.xml                                     |   24 +
 ...stTimelineReaderWebServicesHBaseStorage.java |   50 +-
 .../storage/DataGeneratorForTest.java           |  381 ++
 .../storage/TestHBaseTimelineStorage.java       | 3751 ---------------
 .../storage/TestHBaseTimelineStorageApps.java   | 1849 +++++++
 .../TestHBaseTimelineStorageEntities.java       | 1675 +++++++
 .../hadoop-yarn-server-timelineservice/pom.xml  |   22 +-
 .../collector/AppLevelTimelineCollector.java    |   14 +-
 .../collector/TimelineCollectorWebService.java  |    4 +-
 .../reader/TimelineReaderServer.java            |   25 +-
 .../reader/TimelineReaderWebServices.java       |   47 +-
 .../reader/TestTimelineReaderWebServices.java   |   73 +-
 .../hadoop-yarn-server-web-proxy/pom.xml        |    6 +-
 .../server/webproxy/TestWebAppProxyServlet.java |   39 +-
 .../site/markdown/DockerContainerExecutor.md.vm |  154 -
 .../src/site/markdown/NodeLabel.md              |    2 +-
 .../src/site/markdown/NodeManagerRest.md        |    2 +-
 .../src/site/markdown/ResourceManagerRest.md    |  609 ++-
 .../src/site/markdown/TimelineServer.md         |   21 +-
 .../src/site/markdown/YarnCommands.md           |   21 +-
 .../src/site/markdown/YarnUI2.md                |   55 +
 .../hadoop-yarn/hadoop-yarn-ui/pom.xml          |  193 +
 .../hadoop-yarn-ui/public/crossdomain.xml       |   15 +
 .../hadoop-yarn-ui/src/main/webapp/.bowerrc     |    4 +
 .../hadoop-yarn-ui/src/main/webapp/.ember-cli   |    9 +
 .../hadoop-yarn-ui/src/main/webapp/.jshintrc    |   32 +
 .../src/main/webapp/.watchmanconfig             |    3 +
 .../src/main/webapp/WEB-INF/web.xml             |   25 +
 .../src/main/webapp/app/adapters/abstract.js    |   48 +
 .../main/webapp/app/adapters/cluster-info.js    |   30 +
 .../main/webapp/app/adapters/cluster-metric.js  |   30 +
 .../webapp/app/adapters/yarn-app-attempt.js     |   38 +
 .../src/main/webapp/app/adapters/yarn-app.js    |   43 +
 .../webapp/app/adapters/yarn-container-log.js   |   80 +
 .../main/webapp/app/adapters/yarn-container.js  |   63 +
 .../main/webapp/app/adapters/yarn-node-app.js   |   53 +
 .../webapp/app/adapters/yarn-node-container.js  |   54 +
 .../src/main/webapp/app/adapters/yarn-node.js   |   33 +
 .../src/main/webapp/app/adapters/yarn-queue.js  |   30 +
 .../main/webapp/app/adapters/yarn-rm-node.js    |   36 +
 .../hadoop-yarn-ui/src/main/webapp/app/app.js   |   38 +
 .../webapp/app/components/app-attempt-table.js  |   22 +
 .../src/main/webapp/app/components/app-table.js |   22 +
 .../app/components/app-usage-donut-chart.js     |   62 +
 .../src/main/webapp/app/components/bar-chart.js |  127 +
 .../app/components/base-chart-component.js      |  144 +
 .../app/components/base-usage-donut-chart.js    |   43 +
 .../webapp/app/components/breadcrumb-bar.js     |   31 +
 .../webapp/app/components/container-table.js    |   22 +
 .../main/webapp/app/components/donut-chart.js   |  193 +
 .../main/webapp/app/components/item-selector.js |   39 +
 .../main/webapp/app/components/nodes-heatmap.js |  209 +
 ...er-app-memusage-by-nodes-stacked-barchart.js |   88 +
 ...app-ncontainers-by-nodes-stacked-barchart.js |   67 +
 .../app/components/queue-configuration-table.js |   22 +
 .../webapp/app/components/queue-navigator.js    |   22 +
 .../app/components/queue-usage-donut-chart.js   |   69 +
 .../main/webapp/app/components/queue-view.js    |  289 ++
 .../main/webapp/app/components/simple-table.js  |   81 +
 .../webapp/app/components/stacked-barchart.js   |  198 +
 .../main/webapp/app/components/timeline-view.js |  277 ++
 .../main/webapp/app/components/tree-selector.js |  298 ++
 .../src/main/webapp/app/config.js               |   24 +
 .../src/main/webapp/app/constants.js            |   24 +
 .../main/webapp/app/controllers/application.js  |   55 +
 .../webapp/app/controllers/cluster-overview.js  |   32 +
 .../webapp/app/controllers/yarn-app-attempt.js  |   40 +
 .../webapp/app/controllers/yarn-app-attempts.js |   40 +
 .../src/main/webapp/app/controllers/yarn-app.js |   45 +
 .../main/webapp/app/controllers/yarn-apps.js    |   31 +
 .../app/controllers/yarn-container-log.js       |   40 +
 .../webapp/app/controllers/yarn-node-app.js     |   36 +
 .../webapp/app/controllers/yarn-node-apps.js    |   39 +
 .../app/controllers/yarn-node-containers.js     |   39 +
 .../main/webapp/app/controllers/yarn-node.js    |   37 +
 .../app/controllers/yarn-nodes-heatmap.js       |   36 +
 .../main/webapp/app/controllers/yarn-nodes.js   |   33 +
 .../webapp/app/controllers/yarn-queue-apps.js   |   46 +
 .../main/webapp/app/controllers/yarn-queue.js   |   44 +
 .../main/webapp/app/controllers/yarn-queues.js  |   34 +
 .../webapp/app/controllers/yarn-services.js     |   34 +
 .../src/main/webapp/app/helpers/divide.js       |   31 +
 .../main/webapp/app/helpers/log-files-comma.js  |   48 +
 .../src/main/webapp/app/helpers/node-link.js    |   37 +
 .../src/main/webapp/app/helpers/node-menu.js    |   66 +
 .../src/main/webapp/app/helpers/node-name.js    |   46 +
 .../src/main/webapp/app/index.html              |   44 +
 .../src/main/webapp/app/initializers/env.js     |   29 +
 .../src/main/webapp/app/initializers/hosts.js   |   28 +
 .../src/main/webapp/app/initializers/loader.js  |   86 +
 .../src/main/webapp/app/models/cluster-info.js  |   31 +
 .../main/webapp/app/models/cluster-metric.js    |  133 +
 .../main/webapp/app/models/yarn-app-attempt.js  |  143 +
 .../src/main/webapp/app/models/yarn-app.js      |  104 +
 .../webapp/app/models/yarn-container-log.js     |   25 +
 .../main/webapp/app/models/yarn-container.js    |   64 +
 .../src/main/webapp/app/models/yarn-node-app.js |   44 +
 .../webapp/app/models/yarn-node-container.js    |   57 +
 .../src/main/webapp/app/models/yarn-node.js     |   33 +
 .../src/main/webapp/app/models/yarn-queue.js    |   94 +
 .../src/main/webapp/app/models/yarn-rm-node.js  |   99 +
 .../src/main/webapp/app/models/yarn-user.js     |   26 +
 .../src/main/webapp/app/router.js               |   58 +
 .../src/main/webapp/app/routes/abstract.js      |   32 +
 .../src/main/webapp/app/routes/application.js   |   40 +
 .../main/webapp/app/routes/cluster-overview.js  |   44 +
 .../src/main/webapp/app/routes/index.js         |   29 +
 .../main/webapp/app/routes/yarn-app-attempt.js  |   50 +
 .../main/webapp/app/routes/yarn-app-attempts.js |   36 +
 .../src/main/webapp/app/routes/yarn-app.js      |   52 +
 .../src/main/webapp/app/routes/yarn-apps.js     |   35 +
 .../main/webapp/app/routes/yarn-apps/apps.js    |   22 +
 .../webapp/app/routes/yarn-apps/services.js     |   22 +
 .../webapp/app/routes/yarn-container-log.js     |   63 +
 .../src/main/webapp/app/routes/yarn-node-app.js |   35 +
 .../main/webapp/app/routes/yarn-node-apps.js    |   35 +
 .../webapp/app/routes/yarn-node-container.js    |   36 +
 .../webapp/app/routes/yarn-node-containers.js   |   34 +
 .../src/main/webapp/app/routes/yarn-node.js     |   37 +
 .../src/main/webapp/app/routes/yarn-nodes.js    |   35 +
 .../webapp/app/routes/yarn-nodes/heatmap.js     |   22 +
 .../main/webapp/app/routes/yarn-nodes/table.js  |   22 +
 .../main/webapp/app/routes/yarn-queue-apps.js   |   42 +
 .../src/main/webapp/app/routes/yarn-queue.js    |   42 +
 .../src/main/webapp/app/routes/yarn-queues.js   |   42 +
 .../main/webapp/app/routes/yarn-queues/index.js |   23 +
 .../app/routes/yarn-queues/queues-selector.js   |   25 +
 .../main/webapp/app/serializers/cluster-info.js |   47 +
 .../webapp/app/serializers/cluster-metric.js    |   47 +
 .../webapp/app/serializers/yarn-app-attempt.js  |   75 +
 .../src/main/webapp/app/serializers/yarn-app.js |   93 +
 .../app/serializers/yarn-container-log.js       |   39 +
 .../webapp/app/serializers/yarn-container.js    |   79 +
 .../webapp/app/serializers/yarn-node-app.js     |   83 +
 .../app/serializers/yarn-node-container.js      |   71 +
 .../main/webapp/app/serializers/yarn-node.js    |   56 +
 .../main/webapp/app/serializers/yarn-queue.js   |  145 +
 .../main/webapp/app/serializers/yarn-rm-node.js |   74 +
 .../src/main/webapp/app/services/env.js         |   59 +
 .../src/main/webapp/app/services/hosts.js       |   74 +
 .../src/main/webapp/app/styles/app.css          |  279 ++
 .../main/webapp/app/templates/application.hbs   |   85 +
 .../webapp/app/templates/cluster-overview.hbs   |  150 +
 .../templates/components/app-attempt-table.hbs  |   62 +
 .../app/templates/components/app-table.hbs      |   86 +
 .../app/templates/components/breadcrumb-bar.hbs |   22 +
 .../templates/components/container-table.hbs    |   54 +
 .../templates/components/node-menu-panel.hbs    |   44 +
 .../app/templates/components/nodes-heatmap.hbs  |   27 +
 .../components/queue-configuration-table.hbs    |   54 +
 .../templates/components/queue-navigator.hbs    |   28 +
 .../app/templates/components/timeline-view.hbs  |   54 +
 .../src/main/webapp/app/templates/error.hbs     |   19 +
 .../src/main/webapp/app/templates/notfound.hbs  |   20 +
 .../webapp/app/templates/yarn-app-attempt.hbs   |   43 +
 .../webapp/app/templates/yarn-app-attempts.hbs  |   55 +
 .../src/main/webapp/app/templates/yarn-app.hbs  |  253 +
 .../src/main/webapp/app/templates/yarn-apps.hbs |   90 +
 .../webapp/app/templates/yarn-apps/apps.hbs     |   24 +
 .../webapp/app/templates/yarn-apps/services.hbs |   27 +
 .../webapp/app/templates/yarn-container-log.hbs |   38 +
 .../main/webapp/app/templates/yarn-node-app.hbs |   62 +
 .../webapp/app/templates/yarn-node-apps.hbs     |   53 +
 .../app/templates/yarn-node-container.hbs       |   72 +
 .../app/templates/yarn-node-containers.hbs      |   60 +
 .../src/main/webapp/app/templates/yarn-node.hbs |  118 +
 .../main/webapp/app/templates/yarn-nodes.hbs    |   70 +
 .../webapp/app/templates/yarn-nodes/heatmap.hbs |   30 +
 .../webapp/app/templates/yarn-nodes/table.hbs   |   67 +
 .../webapp/app/templates/yarn-queue-apps.hbs    |   64 +
 .../main/webapp/app/templates/yarn-queue.hbs    |  108 +
 .../main/webapp/app/templates/yarn-queues.hbs   |   70 +
 .../src/main/webapp/app/utils/color-utils.js    |   67 +
 .../src/main/webapp/app/utils/converter.js      |  126 +
 .../main/webapp/app/utils/href-address-utils.js |   29 +
 .../src/main/webapp/app/utils/mock.js           |   36 +
 .../src/main/webapp/app/utils/sorter.js         |   73 +
 .../hadoop-yarn-ui/src/main/webapp/bower.json   |   24 +
 .../src/main/webapp/config/configs.env          |   48 +
 .../src/main/webapp/config/default-config.js    |   32 +
 .../src/main/webapp/config/environment.js       |   70 +
 .../src/main/webapp/ember-cli-build.js          |   58 +
 .../src/main/webapp/jsconfig.json               |    6 +
 .../hadoop-yarn-ui/src/main/webapp/package.json |   52 +
 .../assets/images/datatables/Sorting icons.psd  |  Bin 0 -> 27490 bytes
 .../public/assets/images/datatables/favicon.ico |  Bin 0 -> 894 bytes
 .../assets/images/datatables/sort_asc.png       |  Bin 0 -> 160 bytes
 .../images/datatables/sort_asc_disabled.png     |  Bin 0 -> 148 bytes
 .../assets/images/datatables/sort_both.png      |  Bin 0 -> 201 bytes
 .../assets/images/datatables/sort_desc.png      |  Bin 0 -> 158 bytes
 .../images/datatables/sort_desc_disabled.png    |  Bin 0 -> 146 bytes
 .../webapp/public/assets/images/hadoop_logo.png |  Bin 0 -> 31716 bytes
 .../src/main/webapp/public/crossdomain.xml      |   34 +
 .../hadoop-yarn-ui/src/main/webapp/testem.json  |   12 +
 .../src/main/webapp/tests/helpers/resolver.js   |   29 +
 .../src/main/webapp/tests/helpers/start-app.js  |   36 +
 .../src/main/webapp/tests/index.html            |   51 +
 .../components/breadcrumb-bar-test.js           |   43 +
 .../src/main/webapp/tests/test-helper.js        |   24 +
 .../webapp/tests/unit/adapters/yarn-app-test.js |   30 +
 .../unit/adapters/yarn-container-log-test.js    |   73 +
 .../tests/unit/adapters/yarn-node-app-test.js   |   93 +
 .../unit/adapters/yarn-node-container-test.js   |   93 +
 .../tests/unit/adapters/yarn-node-test.js       |   42 +
 .../tests/unit/adapters/yarn-rm-node-test.js    |   44 +
 .../unit/controllers/yarn-app-attempt-test.js   |   30 +
 .../unit/controllers/yarn-app-attempts-test.js  |   30 +
 .../tests/unit/controllers/yarn-app-test.js     |   30 +
 .../tests/unit/controllers/yarn-apps-test.js    |   30 +
 .../unit/controllers/yarn-container-log-test.js |   30 +
 .../unit/controllers/yarn-node-app-test.js      |   30 +
 .../unit/controllers/yarn-node-apps-test.js     |   30 +
 .../controllers/yarn-node-containers-test.js    |   30 +
 .../tests/unit/controllers/yarn-node-test.js    |   30 +
 .../unit/controllers/yarn-nodes-heatmap-test.js |   30 +
 .../tests/unit/controllers/yarn-nodes-test.js   |   30 +
 .../unit/controllers/yarn-queue-apps-test.js    |   30 +
 .../tests/unit/controllers/yarn-queues-test.js  |   30 +
 .../unit/controllers/yarn-services-test.js      |   30 +
 .../webapp/tests/unit/helpers/node-name-test.js |   28 +
 .../webapp/tests/unit/initializers/env-test.js  |   41 +
 .../tests/unit/initializers/hosts-test.js       |   41 +
 .../tests/unit/initializers/jquery-test.js      |   41 +
 .../tests/unit/initializers/loader-test.js      |   40 +
 .../webapp/tests/unit/mixins/charts-test.js     |   30 +
 .../webapp/tests/unit/models/yarn-app-test.js   |   30 +
 .../unit/models/yarn-container-log-test.js      |   48 +
 .../tests/unit/models/yarn-node-app-test.js     |   65 +
 .../unit/models/yarn-node-container-test.js     |   78 +
 .../webapp/tests/unit/models/yarn-node-test.js  |   58 +
 .../tests/unit/models/yarn-rm-node-test.js      |   95 +
 .../tests/unit/routes/yarn-app-attempts-test.js |   29 +
 .../webapp/tests/unit/routes/yarn-apps-test.js  |   29 +
 .../unit/routes/yarn-container-log-test.js      |  120 +
 .../tests/unit/routes/yarn-node-app-test.js     |   56 +
 .../tests/unit/routes/yarn-node-apps-test.js    |   60 +
 .../unit/routes/yarn-node-container-test.js     |   61 +
 .../unit/routes/yarn-node-containers-test.js    |   68 +
 .../webapp/tests/unit/routes/yarn-node-test.js  |   84 +
 .../webapp/tests/unit/routes/yarn-nodes-test.js |   74 +
 .../tests/unit/routes/yarn-queue-apps-test.js   |   29 +
 .../tests/unit/routes/yarn-queues-test.js       |   29 +
 .../tests/unit/serializers/yarn-app-test.js     |   33 +
 .../unit/serializers/yarn-container-log-test.js |   49 +
 .../unit/serializers/yarn-node-app-test.js      |  102 +
 .../serializers/yarn-node-container-test.js     |  128 +
 .../tests/unit/serializers/yarn-node-test.js    |   69 +
 .../tests/unit/serializers/yarn-rm-node-test.js |  153 +
 .../main/webapp/tests/unit/services/env-test.js |   30 +
 .../webapp/tests/unit/services/hosts-test.js    |   30 +
 .../webapp/tests/unit/utils/converter-test.js   |   52 +
 .../main/webapp/tests/unit/utils/sorter-test.js |   26 +
 hadoop-yarn-project/hadoop-yarn/pom.xml         |    1 +
 pom.xml                                         |    8 +-
 1005 files changed, 49211 insertions(+), 14325 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 47692e2,8ce5f3f..9c8dd1b
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@@ -111,10 -111,10 +111,14 @@@ http://maven.apache.org/xsd/maven-4.0.0
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
        </dependency>
 +      <dependency>
 +          <groupId>io.netty</groupId>
 +          <artifactId>netty-all</artifactId>
 +      </dependency>
+     <dependency>
+       <groupId>com.fasterxml.jackson.core</groupId>
+       <artifactId>jackson-databind</artifactId>
+     </dependency>
    </dependencies>
  
    <build>
@@@ -162,10 -161,8 +165,9 @@@
                    <include>inotify.proto</include>
                    <include>erasurecoding.proto</include>
                    <include>ReconfigurationProtocol.proto</include>
 +                  <include>DatanodeContainerProtocol.proto</include>
                  </includes>
                </source>
-               <output>${project.build.directory}/generated-sources/java</output>
              </configuration>
            </execution>
          </executions>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 3693a73,5427db8..2bbf253
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@@ -352,12 -349,20 +350,23 @@@ http://maven.apache.org/xsd/maven-4.0.0
                    <include>QJournalProtocol.proto</include>
                    <include>editlog.proto</include>
                    <include>fsimage.proto</include>
 +                  <include>StorageContainerLocationProtocol.proto</include>
 +                  <include>StorageContainerDatanodeProtocol.proto</include>
 +                  <include>CBlockServiceProtocol.proto</include>
                  </includes>
                </source>
-               <output>${project.build.directory}/generated-sources/java</output>
+             </configuration>
+           </execution>
+           <execution>
+             <id>resource-gz</id>
+             <phase>generate-resources</phase>
+             <goals>
+               <goal>resource-gz</goal>
+             </goals>
+             <configuration>
+               <inputDirectory>${basedir}/src/main/webapps/static</inputDirectory>
+               <outputDirectory>${basedir}/target/webapps/static</outputDirectory>
+               <extensions>js,css</extensions>
              </configuration>
            </execution>
          </executions>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
index e4c1989,0000000..c251966
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
@@@ -1,92 -1,0 +1,94 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + *  with the License.  You may obtain a copy of the License at
 + *
 + *      http://www.apache.org/licenses/LICENSE-2.0
 + *
 + *  Unless required by applicable law or agreed to in writing, software
 + *  distributed under the License is distributed on an "AS IS" BASIS,
 + *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + *  See the License for the specific language governing permissions and
 + *  limitations under the License.
 + */
 +
 +package org.apache.hadoop.ozone.container.common.impl;
 +
 +import com.google.common.base.Preconditions;
 +import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 +import org.apache.hadoop.ozone.OzoneConsts;
 +import org.apache.hadoop.ozone.container.common.interfaces
 +    .ContainerLocationManager;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import java.io.IOException;
 +import java.nio.file.Path;
++import java.nio.file.Paths;
 +import java.util.List;
 +
 +/**
 + * A class that tells the ContainerManager where to place the containers.
 + * Please note : There is *no* one-to-one correlation between metadata
 + * metadataLocations and data metadataLocations.
 + *
 + *  For example : A user could map all container files to a
 + *  SSD but leave data/metadata on bunch of other disks.
 + */
 +public class ContainerLocationManagerImpl implements ContainerLocationManager {
 +  private static final Logger LOG =
 +      LoggerFactory.getLogger(ContainerLocationManagerImpl.class);
 +
 +  private final List<StorageLocation> dataLocations;
 +  private int currentIndex;
 +  private final List<StorageLocation> metadataLocations;
 +
 +  /**
 +   * Constructs a Location Manager.
 +   * @param metadataLocations  - Refers to the metadataLocations
 +   * where we store the container metadata.
 +   * @param dataDirs - metadataLocations where we store the actual
 +   * data or chunk files.
 +   * @throws IOException
 +   */
 +  public ContainerLocationManagerImpl(List<StorageLocation> metadataLocations,
 +      List<StorageLocation> dataDirs)
 +      throws IOException {
 +    dataLocations = dataDirs;
 +    this.metadataLocations = metadataLocations;
 +  }
 +  /**
 +   * Returns the path where the container should be placed from a set of
 +   * metadataLocations.
 +   *
 +   * @return A path where we should place this container and metadata.
 +   * @throws IOException
 +   */
 +  @Override
 +  public Path getContainerPath()
 +      throws IOException {
 +    Preconditions.checkState(metadataLocations.size() > 0);
 +    int index = currentIndex % metadataLocations.size();
-     Path path = metadataLocations.get(index).getFile().toPath();
++    Path path = Paths.get(metadataLocations.get(index).getNormalizedUri());
 +    return path.resolve(OzoneConsts.CONTAINER_ROOT_PREFIX);
 +  }
 +
 +  /**
 +   * Returns the path where the container Data file are stored.
 +   *
 +   * @return  a path where we place the LevelDB and data files of a container.
 +   * @throws IOException
 +   */
 +  @Override
 +  public Path getDataPath(String containerName) throws IOException {
-     Path currentPath = dataLocations.get(currentIndex++ % dataLocations.size())
-         .getFile().toPath();
++    Path currentPath = Paths.get(
++        dataLocations.get(currentIndex++ % dataLocations.size())
++            .getNormalizedUri());
 +    currentPath = currentPath.resolve(OzoneConsts.CONTAINER_PREFIX);
 +    return currentPath.resolve(containerName);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index 88bdebc,0000000..cb571ea
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@@ -1,565 -1,0 +1,565 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + *  with the License.  You may obtain a copy of the License at
 + *
 + *      http://www.apache.org/licenses/LICENSE-2.0
 + *
 + *  Unless required by applicable law or agreed to in writing, software
 + *  distributed under the License is distributed on an "AS IS" BASIS,
 + *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + *  See the License for the specific language governing permissions and
 + *  limitations under the License.
 + */
 +
 +package org.apache.hadoop.ozone.container.common.impl;
 +
 +import com.google.common.annotations.VisibleForTesting;
 +import com.google.common.base.Preconditions;
 +import org.apache.commons.codec.digest.DigestUtils;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileAlreadyExistsException;
 +import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
 +import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 +import org.apache.hadoop.io.IOUtils;
 +import org.apache.hadoop.ozone.OzoneConsts;
 +import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 +import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 +import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
 +import org.apache.hadoop.ozone.container.common.interfaces
 +    .ContainerLocationManager;
 +import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
 +import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
 +import org.apache.hadoop.scm.container.common.helpers.Pipeline;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import java.io.File;
 +import java.io.FileInputStream;
 +import java.io.FileOutputStream;
 +import java.io.FilenameFilter;
 +import java.io.IOException;
 +import java.nio.file.Path;
 +import java.nio.file.Paths;
 +import java.security.DigestInputStream;
 +import java.security.DigestOutputStream;
 +import java.security.MessageDigest;
 +import java.security.NoSuchAlgorithmException;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.concurrent.ConcurrentNavigableMap;
 +import java.util.concurrent.ConcurrentSkipListMap;
 +import java.util.concurrent.locks.ReentrantReadWriteLock;
 +
 +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
 +import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
 +import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
 +
 +/**
 + * A Generic ContainerManagerImpl that will be called from Ozone
 + * ContainerManagerImpl. This allows us to support delta changes to ozone
 + * version without having to rewrite the containerManager.
 + */
 +public class ContainerManagerImpl implements ContainerManager {
 +  static final Logger LOG =
 +      LoggerFactory.getLogger(ContainerManagerImpl.class);
 +
 +  private final ConcurrentSkipListMap<String, ContainerStatus>
 +      containerMap = new ConcurrentSkipListMap<>();
 +
 +  // This lock follows fair locking policy of first-come first-serve
 +  // for waiting threads.
 +  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
 +  private ContainerLocationManager locationManager;
 +  private ChunkManager chunkManager;
 +  private KeyManager keyManager;
 +
 +  /**
 +   * Init call that sets up a container Manager.
 +   *
 +   * @param config        - Configuration.
 +   * @param containerDirs - List of Metadata Container locations.
 +   * @throws IOException
 +   */
 +  @Override
 +  public void init(
 +      Configuration config, List<StorageLocation> containerDirs)
 +      throws IOException {
 +    Preconditions.checkNotNull(config);
 +    Preconditions.checkNotNull(containerDirs);
 +    Preconditions.checkState(containerDirs.size() > 0);
 +
 +    readLock();
 +    try {
 +      for (StorageLocation path : containerDirs) {
-         File directory = path.getFile();
++        File directory = Paths.get(path.getNormalizedUri()).toFile();
 +        if (!directory.isDirectory()) {
 +          LOG.error("Invalid path to container metadata directory. path: {}",
 +              path.toString());
 +          throw new IOException("Invalid path to container metadata directory" +
 +              ". " + path);
 +        }
 +        File[] files = directory.listFiles(new ContainerFilter());
 +        if (files != null) {
 +          for (File containerFile : files) {
 +            String containerPath =
 +                ContainerUtils.getContainerNameFromFile(containerFile);
 +            Preconditions.checkNotNull(containerPath);
 +            readContainerInfo(containerPath);
 +          }
 +        }
 +      }
 +
 +      List<StorageLocation> dataDirs = new LinkedList<>();
 +      for (String dir : config.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
 +        StorageLocation location = StorageLocation.parse(dir);
 +        dataDirs.add(location);
 +      }
 +      this.locationManager =
 +          new ContainerLocationManagerImpl(containerDirs, dataDirs);
 +
 +    } finally {
 +      readUnlock();
 +    }
 +  }
 +
 +  /**
 +   * Reads the Container Info from a file and verifies that checksum match. If
 +   * the checksums match, then that file is added to containerMap.
 +   *
 +   * @param containerName - Name which points to the persisted container.
 +   */
 +  private void readContainerInfo(String containerName)
 +      throws IOException {
 +    Preconditions.checkState(containerName.length() > 0);
 +    FileInputStream containerStream = null;
 +    DigestInputStream dis = null;
 +    FileInputStream metaStream = null;
 +    Path cPath = Paths.get(containerName).getFileName();
 +    String keyName = null;
 +    if (cPath != null) {
 +      keyName = cPath.toString();
 +    }
 +    Preconditions.checkNotNull(keyName);
 +
 +    try {
 +      String containerFileName = containerName.concat(CONTAINER_EXTENSION);
 +      String metaFileName = containerName.concat(CONTAINER_META);
 +
 +      containerStream = new FileInputStream(containerFileName);
 +
 +      metaStream = new FileInputStream(metaFileName);
 +
 +      MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
 +
 +      dis = new DigestInputStream(containerStream, sha);
 +
 +      ContainerData containerData = ContainerData.getFromProtBuf(
 +          ContainerProtos.ContainerData.parseDelimitedFrom(dis));
 +
 +
 +      ContainerProtos.ContainerMeta meta = ContainerProtos.ContainerMeta
 +          .parseDelimitedFrom(metaStream);
 +
 +      if (meta != null && !DigestUtils.sha256Hex(sha.digest()).equals(meta
 +          .getHash())) {
 +        throw new IOException("Invalid SHA found for file.");
 +      }
 +
 +      containerMap.put(keyName, new ContainerStatus(containerData, true));
 +
 +    } catch (IOException | NoSuchAlgorithmException ex) {
 +      LOG.error("read failed for file: {} ex: {}",
 +          containerName, ex.getMessage());
 +
 +      // TODO : Add this file to a recovery Queue.
 +
 +      // Remember that this container is busted and we cannot use it.
 +      containerMap.put(keyName, new ContainerStatus(null, false));
 +    } finally {
 +      IOUtils.closeStream(dis);
 +      IOUtils.closeStream(containerStream);
 +      IOUtils.closeStream(metaStream);
 +    }
 +  }
 +
 +  /**
 +   * Creates a container with the given name.
 +   *
 +   * @param pipeline      -- Nodes which make up this container.
 +   * @param containerData - Container Name and metadata.
 +   * @throws IOException
 +   */
 +  @Override
 +  public void createContainer(Pipeline pipeline, ContainerData containerData)
 +      throws IOException {
 +    Preconditions.checkNotNull(containerData);
 +
 +    writeLock();
 +    try {
 +      if (containerMap.containsKey(containerData.getName())) {
 +        throw new FileAlreadyExistsException("container already exists.");
 +      }
 +
 +      // This is by design. We first write and close the
 +      // container Info and metadata to a directory.
 +      // Then read back and put that info into the containerMap.
 +      // This allows us to make sure that our write is consistent.
 +
 +      writeContainerInfo(containerData);
 +      File cFile = new File(containerData.getContainerPath());
 +      readContainerInfo(ContainerUtils.getContainerNameFromFile(cFile));
 +    } catch (NoSuchAlgorithmException ex) {
 +      throw new IOException("failed to create container", ex);
 +
 +    } finally {
 +      writeUnlock();
 +    }
 +
 +  }
 +
 +  /**
 +   * Writes a container to a chosen location and updates the container Map.
 +   *
 +   * The file formats of ContainerData and Container Meta is the following.
 +   *
 +   * message ContainerData {
 +   * required string name = 1;
 +   * repeated KeyValue metadata = 2;
 +   * optional string dbPath = 3;
 +   * optional string containerPath = 4;
 +   * }
 +   *
 +   * message ContainerMeta {
 +   * required string fileName = 1;
 +   * required string hash = 2;
 +   * }
 +   *
 +   * @param containerData - container Data
 +   */
 +  private void writeContainerInfo(ContainerData containerData)
 +      throws IOException, NoSuchAlgorithmException {
 +
 +    Preconditions.checkNotNull(this.locationManager);
 +
 +    FileOutputStream containerStream = null;
 +    DigestOutputStream dos = null;
 +    FileOutputStream metaStream = null;
 +    Path location = locationManager.getContainerPath();
 +
 +    File containerFile = ContainerUtils.getContainerFile(containerData,
 +        location);
 +    File metadataFile = ContainerUtils.getMetadataFile(containerData, location);
 +
 +    try {
 +      ContainerUtils.verifyIsNewContainer(containerFile, metadataFile);
 +
 +      Path metadataPath = this.locationManager.getDataPath(
 +          containerData.getContainerName());
 +      metadataPath = ContainerUtils.createMetadata(metadataPath);
 +
 +      containerStream = new FileOutputStream(containerFile);
 +      metaStream = new FileOutputStream(metadataFile);
 +      MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
 +
 +      dos = new DigestOutputStream(containerStream, sha);
 +      containerData.setDBPath(metadataPath.resolve(OzoneConsts.CONTAINER_DB)
 +          .toString());
 +      containerData.setContainerPath(containerFile.toString());
 +
 +      ContainerProtos.ContainerData protoData = containerData
 +          .getProtoBufMessage();
 +      protoData.writeDelimitedTo(dos);
 +
 +      ContainerProtos.ContainerMeta protoMeta = ContainerProtos
 +          .ContainerMeta.newBuilder()
 +          .setFileName(containerFile.toString())
 +          .setHash(DigestUtils.sha256Hex(sha.digest()))
 +          .build();
 +      protoMeta.writeDelimitedTo(metaStream);
 +
 +    } catch (IOException ex) {
 +
 +      // TODO : we need to clean up partially constructed files
 +      // The proper way to do would be for a thread
 +      // to read all these 3 artifacts and make sure they are
 +      // sane. That info needs to come from the replication
 +      // pipeline, and if not consistent delete these file.
 +
 +      // In case of ozone this is *not* a deal breaker since
 +      // SCM is guaranteed to generate unique container names.
 +
 +      LOG.error("creation of container failed. Name: {} ",
 +          containerData.getContainerName());
 +      throw ex;
 +    } finally {
 +      IOUtils.closeStream(dos);
 +      IOUtils.closeStream(containerStream);
 +      IOUtils.closeStream(metaStream);
 +    }
 +  }
 +
 +
 +
 +  /**
 +   * Deletes an existing container.
 +   *
 +   * @param pipeline      - nodes that make this container.
 +   * @param containerName - name of the container.
 +   * @throws IOException
 +   */
 +  @Override
 +  public void deleteContainer(Pipeline pipeline, String containerName) throws
 +      IOException {
 +    Preconditions.checkState(containerName.length() > 0);
 +    writeLock();
 +    try {
 +      ContainerStatus status = containerMap.get(containerName);
 +      if (status == null) {
 +        LOG.info("No such container. Name: {}", containerName);
 +        throw new IOException("No such container. Name : " + containerName);
 +      }
 +      ContainerUtils.removeContainer(status.containerData);
 +      containerMap.remove(containerName);
 +    } finally {
 +      writeUnlock();
 +    }
 +
 +  }
 +
 +  /**
 +   * A simple interface for container Iterations.
 +   * <p/>
 +   * This call make no guarantees about consistency of the data between
 +   * different list calls. It just returns the best known data at that point of
 +   * time. It is possible that using this iteration you can miss certain
 +   * container from the listing.
 +   *
 +   * @param prefix  -  Return keys that match this prefix.
 +   * @param count   - how many to return
 +   * @param prevKey - Previous Key Value or empty String.
 +   * @param data    - Actual containerData
 +   * @throws IOException
 +   */
 +  @Override
 +  public void listContainer(String prefix, long count, String prevKey,
 +                            List<ContainerData> data) throws IOException {
 +    // TODO : Support list with Prefix and PrevKey
 +    Preconditions.checkNotNull(data);
 +    readLock();
 +    try {
 +      ConcurrentNavigableMap<String, ContainerStatus> map = null;
 +      if (prevKey == null || prevKey.isEmpty()) {
 +        map = containerMap.tailMap(containerMap.firstKey(), true);
 +      } else {
 +        map = containerMap.tailMap(prevKey, false);
 +      }
 +
 +      int currentCount = 0;
 +      for (ContainerStatus entry : map.values()) {
 +        if (currentCount < count) {
 +          data.add(entry.getContainer());
 +          currentCount++;
 +        } else {
 +          return;
 +        }
 +      }
 +    } finally {
 +      readUnlock();
 +    }
 +  }
 +
 +  /**
 +   * Get metadata about a specific container.
 +   *
 +   * @param containerName - Name of the container
 +   * @return ContainerData - Container Data.
 +   * @throws IOException
 +   */
 +  @Override
 +  public ContainerData readContainer(String containerName) throws IOException {
 +    if(!containerMap.containsKey(containerName)) {
 +      throw new IOException("Unable to find the container. Name: "
 +          + containerName);
 +    }
 +    return containerMap.get(containerName).getContainer();
 +  }
 +
 +  /**
 +   * Supports clean shutdown of container.
 +   *
 +   * @throws IOException
 +   */
 +  @Override
 +  public void shutdown() {
 +    Preconditions.checkState(this.hasWriteLock());
 +    this.containerMap.clear();
 +  }
 +
 +
 +  @VisibleForTesting
 +  ConcurrentSkipListMap<String, ContainerStatus> getContainerMap() {
 +    return containerMap;
 +  }
 +
 +  /**
 +   * Acquire read lock.
 +   */
 +  @Override
 +  public void readLock() {
 +    this.lock.readLock().lock();
 +
 +  }
 +
 +  /**
 +   * Release read lock.
 +   */
 +  @Override
 +  public void readUnlock() {
 +    this.lock.readLock().unlock();
 +  }
 +
 +  /**
 +   * Check if the current thread holds read lock.
 +   */
 +  @Override
 +  public boolean hasReadLock() {
 +    return this.lock.readLock().tryLock();
 +  }
 +
 +  /**
 +   * Acquire write lock.
 +   */
 +  @Override
 +  public void writeLock() {
 +    this.lock.writeLock().lock();
 +  }
 +
 +  /**
 +   * Acquire write lock, unless interrupted while waiting.
 +   */
 +  @Override
 +  public void writeLockInterruptibly() throws InterruptedException {
 +    this.lock.writeLock().lockInterruptibly();
 +
 +  }
 +
 +  /**
 +   * Release write lock.
 +   */
 +  @Override
 +  public void writeUnlock() {
 +    this.lock.writeLock().unlock();
 +
 +  }
 +
 +  /**
 +   * Check if the current thread holds write lock.
 +   */
 +  @Override
 +  public boolean hasWriteLock() {
 +    return this.lock.writeLock().isHeldByCurrentThread();
 +  }
 +
 +  /**
 +   * Sets the chunk Manager.
 +   * @param chunkManager
 +   */
 +  public void setChunkManager(ChunkManager chunkManager) {
 +    this.chunkManager = chunkManager;
 +  }
 +
 +  public ChunkManager getChunkManager() {
 +    return this.chunkManager;
 +  }
 +
 +  /**
 +   * Sets the Key Manager.
 +   *
 +   * @param keyManager - Key Manager.
 +   */
 +  @Override
 +  public void setKeyManager(KeyManager keyManager) {
 +    this.keyManager = keyManager;
 +  }
 +
 +  /**
 +   * Gets the Key Manager.
 +   *
 +   * @return KeyManager.
 +   */
 +  @Override
 +  public KeyManager getKeyManager() {
 +    return this.keyManager;
 +  }
 +
 +  /**
 +   * Filter out only container files from the container metadata dir.
 +   */
 +  private static class ContainerFilter implements FilenameFilter {
 +    /**
 +     * Tests if a specified file should be included in a file list.
 +     *
 +     * @param dir  the directory in which the file was found.
 +     * @param name the name of the file.
 +     * @return <code>true</code> if and only if the name should be included in
 +     * the file list; <code>false</code> otherwise.
 +     */
 +    @Override
 +    public boolean accept(File dir, String name) {
 +      return name.endsWith(CONTAINER_EXTENSION);
 +    }
 +  }
 +
 +  /**
 +   * This is an immutable class that represents the state of a container. if the
 +   * container reading encountered an error when we boot up we will post that
 +   * info to a recovery queue and keep the info in the containerMap.
 +   * <p/>
 +   * if and when the issue is fixed, the expectation is that this entry will be
 +   * deleted by the recovery thread from the containerMap and will insert entry
 +   * instead of modifying this class.
 +   */
 +  @VisibleForTesting
 +  static class ContainerStatus {
 +    private final ContainerData containerData;
 +    private final boolean active;
 +
 +    /**
 +     * Creates a Container Status class.
 +     *
 +     * @param containerData - ContainerData.
 +     * @param active        - Active or not active.
 +     */
 +    ContainerStatus(ContainerData containerData, boolean active) {
 +      this.containerData = containerData;
 +      this.active = active;
 +    }
 +
 +    /**
 +     * Returns container if it is active. It is not active if we have had an
 +     * error and we are waiting for the background threads to fix the issue.
 +     *
 +     * @return ContainerData.
 +     */
 +    public ContainerData getContainer() {
 +      if (active) {
 +        return containerData;
 +      }
 +      return null;
 +    }
 +
 +    /**
 +     * Indicates if a container is Active.
 +     *
 +     * @return
 +     */
 +    public boolean isActive() {
 +      return active;
 +    }
 +  }
 +}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message