Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C76C8200C2C for ; Fri, 3 Mar 2017 23:05:13 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id C5E7E160B80; Fri, 3 Mar 2017 22:05:13 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D2F07160B5E for ; Fri, 3 Mar 2017 23:05:10 +0100 (CET) Received: (qmail 9717 invoked by uid 500); 3 Mar 2017 22:04:55 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 8129 invoked by uid 99); 3 Mar 2017 22:04:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 Mar 2017 22:04:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5F808DFE34; Fri, 3 Mar 2017 22:04:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Fri, 03 Mar 2017 22:05:42 -0000 Message-Id: <7b4ba5d07a3d4d0b963426a7b9327a1c@git.apache.org> In-Reply-To: <275bda3b88a54b959537eb82f6111bf2@git.apache.org> References: <275bda3b88a54b959537eb82f6111bf2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240 archived-at: Fri, 03 Mar 2017 22:05:14 -0000 Merge branch 'trunk' into HDFS-7240 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09517262 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09517262 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09517262 Branch: refs/heads/HDFS-7240 Commit: 0951726233270fe1f30a303305a2f6f354a93d14 Parents: 3ab6bc5 490abfb Author: Anu Engineer Authored: Fri Mar 3 14:03:35 2017 -0800 Committer: Anu Engineer Committed: Fri Mar 3 14:03:35 2017 -0800 ---------------------------------------------------------------------- .gitignore | 13 +- BUILDING.txt | 38 +- LICENSE.txt | 1324 +- NOTICE.txt | 129 +- dev-support/bin/create-release | 5 +- dev-support/bin/dist-copynativelibs | 13 + dev-support/bin/dist-layout-stitching | 8 +- dev-support/bin/qbt | 0 dev-support/docker/Dockerfile | 1 + hadoop-assemblies/pom.xml | 5 +- .../main/resources/assemblies/hadoop-dist.xml | 14 - .../resources/assemblies/hadoop-httpfs-dist.xml | 25 +- .../resources/assemblies/hadoop-kms-dist.xml | 25 +- .../assemblies/hadoop-mapreduce-dist.xml | 14 - .../resources/assemblies/hadoop-yarn-dist.xml | 14 - hadoop-build-tools/pom.xml | 4 +- .../main/resources/checkstyle/checkstyle.xml | 4 +- hadoop-client-modules/hadoop-client-api/pom.xml | 254 + .../hadoop-client-check-invariants/pom.xml | 124 + .../hadoop-client-check-test-invariants/pom.xml | 143 + .../hadoop-client-integration-tests/pom.xml | 159 + .../apache/hadoop/example/ITUseMiniCluster.java | 113 + .../src/test/resources/hdfs-site.xml | 34 + .../src/test/resources/log4j.properties | 24 + .../hadoop-client-minicluster/pom.xml | 792 + .../hadoop-client-runtime/pom.xml | 359 + hadoop-client-modules/hadoop-client/pom.xml | 333 + hadoop-client-modules/pom.xml | 45 + hadoop-client/pom.xml | 320 - .../hadoop-cloud-storage/pom.xml | 9 +- hadoop-cloud-storage-project/pom.xml | 5 +- .../hadoop-annotations/pom.xml | 5 +- .../hadoop-auth-examples/pom.xml | 5 +- hadoop-common-project/hadoop-auth/pom.xml | 5 +- .../server/AuthenticationFilter.java | 6 +- .../server/KerberosAuthenticationHandler.java | 254 +- .../authentication/util/KerberosName.java | 4 +- .../util/ZKSignerSecretProvider.java | 2 +- .../authentication/util/TestKerberosName.java | 22 + .../dev-support/findbugsExcludeFile.xml | 6 + hadoop-common-project/hadoop-common/pom.xml | 60 +- .../hadoop-common/src/CMakeLists.txt | 29 + .../hadoop-common/src/config.h.cmake | 1 + .../hadoop-common/src/main/bin/hadoop | 13 +- .../src/main/bin/hadoop-functions.sh | 266 +- .../hadoop-common/src/main/bin/start-all.sh | 23 +- .../hadoop-common/src/main/bin/stop-all.sh | 19 +- .../org/apache/hadoop/conf/Configuration.java | 20 +- .../hadoop/conf/ConfigurationWithLogging.java | 113 + .../hadoop/crypto/key/CachingKeyProvider.java | 14 +- .../hadoop/crypto/key/JavaKeyStoreProvider.java | 31 +- .../apache/hadoop/crypto/key/KeyProvider.java | 40 + .../crypto/key/KeyProviderCryptoExtension.java | 94 +- .../hadoop/crypto/key/KeyProviderExtension.java | 5 + .../org/apache/hadoop/crypto/key/KeyShell.java | 67 +- .../crypto/key/kms/KMSClientProvider.java | 153 +- .../hadoop/crypto/key/kms/KMSRESTConstants.java | 2 + .../key/kms/LoadBalancingKMSClientProvider.java | 38 +- .../hadoop/crypto/key/kms/ValueQueue.java | 117 +- .../org/apache/hadoop/fs/BlockLocation.java | 7 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 2 + .../hadoop/fs/CommonConfigurationKeys.java | 27 + .../fs/CommonConfigurationKeysPublic.java | 44 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 2 +- .../apache/hadoop/fs/FSExceptionMessages.java | 3 + .../java/org/apache/hadoop/fs/FileStatus.java | 21 +- .../java/org/apache/hadoop/fs/FileSystem.java | 1473 +- .../java/org/apache/hadoop/fs/FileUtil.java | 19 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 7 + .../main/java/org/apache/hadoop/fs/FsShell.java | 7 +- .../java/org/apache/hadoop/fs/GlobPattern.java | 2 +- .../org/apache/hadoop/fs/LocalDirAllocator.java | 2 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 12 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 5 +- .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 71 +- .../hadoop/fs/permission/FsCreateModes.java | 1 + .../hadoop/fs/permission/FsPermission.java | 22 +- .../org/apache/hadoop/fs/shell/Command.java | 14 +- .../apache/hadoop/fs/shell/CommandFormat.java | 2 +- .../org/apache/hadoop/fs/shell/Display.java | 11 +- .../org/apache/hadoop/fs/shell/FsUsage.java | 124 +- .../java/org/apache/hadoop/fs/shell/Stat.java | 4 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 11 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 136 +- .../hadoop/fs/viewfs/ViewFileSystemUtil.java | 164 + .../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 37 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 47 +- .../apache/hadoop/ha/ZKFailoverController.java | 7 +- .../org/apache/hadoop/http/HttpServer2.java | 282 +- .../java/org/apache/hadoop/io/SequenceFile.java | 94 +- .../apache/hadoop/io/compress/Decompressor.java | 2 +- .../apache/hadoop/io/compress/GzipCodec.java | 78 +- .../hadoop/io/compress/ZStandardCodec.java | 242 + .../io/compress/zstd/ZStandardCompressor.java | 305 + .../io/compress/zstd/ZStandardDecompressor.java | 323 + .../hadoop/io/compress/zstd/package-info.java | 22 + .../apache/hadoop/io/erasurecode/CodecUtil.java | 28 +- .../io/erasurecode/ErasureCodeConstants.java | 12 +- .../erasurecode/coder/HHXORErasureDecoder.java | 2 +- .../erasurecode/coder/HHXORErasureEncoder.java | 2 +- .../io/erasurecode/coder/RSErasureDecoder.java | 2 +- .../io/erasurecode/coder/RSErasureEncoder.java | 2 +- .../io/erasurecode/rawcoder/util/GF256.java | 2 +- .../org/apache/hadoop/io/file/tfile/TFile.java | 2 +- .../org/apache/hadoop/io/file/tfile/Utils.java | 2 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 40 +- .../hadoop/io/retry/RetryInvocationHandler.java | 8 +- .../apache/hadoop/io/retry/RetryPolicies.java | 3 +- .../main/java/org/apache/hadoop/ipc/Client.java | 6 +- .../apache/hadoop/ipc/DecayRpcScheduler.java | 2 +- .../org/apache/hadoop/ipc/FairCallQueue.java | 165 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 75 +- .../hadoop/ipc/ProtobufRpcEngineCallback.java | 29 + .../main/java/org/apache/hadoop/ipc/RPC.java | 5 +- .../main/java/org/apache/hadoop/ipc/Server.java | 464 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 3 +- .../hadoop/ipc/metrics/RpcDetailedMetrics.java | 6 + .../apache/hadoop/ipc/metrics/RpcMetrics.java | 38 + .../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +- .../java/org/apache/hadoop/log/Log4Json.java | 14 +- .../hadoop/metrics2/MetricsJsonBuilder.java | 125 + .../apache/hadoop/metrics2/MetricsPlugin.java | 2 +- .../metrics2/filter/AbstractPatternFilter.java | 2 +- .../hadoop/metrics2/impl/MetricsConfig.java | 38 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 8 +- .../hadoop/metrics2/lib/MetricsRegistry.java | 8 + .../metrics2/lib/MutableMetricsFactory.java | 4 +- .../hadoop/metrics2/lib/MutableQuantiles.java | 10 + .../lib/MutableRatesWithAggregation.java | 40 +- .../hadoop/metrics2/lib/RollingAverages.java | 286 + .../apache/hadoop/metrics2/sink/FileSink.java | 2 +- .../hadoop/metrics2/sink/GraphiteSink.java | 2 +- .../metrics2/sink/RollingFileSystemSink.java | 2 +- .../apache/hadoop/metrics2/sink/StatsDSink.java | 2 +- .../sink/ganglia/AbstractGangliaSink.java | 2 +- .../metrics2/sink/ganglia/GangliaSink30.java | 26 +- .../java/org/apache/hadoop/net/InnerNode.java | 67 + .../org/apache/hadoop/net/InnerNodeImpl.java | 304 + .../org/apache/hadoop/net/NetworkTopology.java | 501 +- .../net/NetworkTopologyWithNodeGroup.java | 43 +- .../hadoop/security/AccessControlException.java | 2 +- .../AuthenticationFilterInitializer.java | 9 +- .../AuthenticationWithProxyUserFilter.java | 124 + .../org/apache/hadoop/security/Credentials.java | 2 +- .../java/org/apache/hadoop/security/Groups.java | 11 +- .../java/org/apache/hadoop/security/KDiag.java | 69 +- .../apache/hadoop/security/SecurityUtil.java | 10 +- .../security/ShellBasedUnixGroupsMapping.java | 114 +- .../hadoop/security/UserGroupInformation.java | 186 +- .../alias/LocalJavaKeyStoreProvider.java | 2 +- .../authorize/AuthorizationException.java | 2 +- .../security/ssl/FileBasedKeyStoresFactory.java | 8 +- .../security/ssl/ReloadingX509TrustManager.java | 2 +- .../apache/hadoop/security/ssl/SSLFactory.java | 74 +- .../org/apache/hadoop/security/token/Token.java | 2 +- .../AbstractDelegationTokenIdentifier.java | 2 +- .../AbstractDelegationTokenSecretManager.java | 8 +- .../ZKDelegationTokenSecretManager.java | 33 +- .../DelegationTokenAuthenticationHandler.java | 6 +- .../web/DelegationTokenAuthenticator.java | 2 +- .../hadoop/util/DiskValidatorFactory.java | 5 +- .../hadoop/util/GenericOptionsParser.java | 31 +- .../apache/hadoop/util/HttpExceptionUtils.java | 2 +- .../apache/hadoop/util/NativeCodeLoader.java | 5 + .../hadoop/util/NativeLibraryChecker.java | 13 +- .../hadoop/util/ReadWriteDiskValidator.java | 95 + .../util/ReadWriteDiskValidatorMetrics.java | 170 + .../main/java/org/apache/hadoop/util/Shell.java | 65 +- .../org/apache/hadoop/util/StringUtils.java | 11 + .../hadoop-common/src/main/java/overview.html | 274 - .../src/main/native/gtest/gtest-all.cc | 10403 ++++++++ .../src/main/native/gtest/include/gtest/gtest.h | 21192 +++++++++++++++++ .../io/compress/zstd/ZStandardCompressor.c | 259 + .../io/compress/zstd/ZStandardDecompressor.c | 218 + .../zstd/org_apache_hadoop_io_compress_zstd.h | 34 + .../org/apache/hadoop/util/NativeCodeLoader.c | 11 + ...g.apache.hadoop.io.compress.CompressionCodec | 1 + .../resources/common-version-info.properties | 4 +- .../src/main/resources/core-default.xml | 152 +- .../src/site/markdown/ClusterSetup.md | 3 +- .../src/site/markdown/CommandsManual.md | 18 +- .../src/site/markdown/CredentialProviderAPI.md | 3 + .../src/site/markdown/DeprecatedProperties.md | 7 +- .../src/site/markdown/FileSystemShell.md | 326 +- .../hadoop-common/src/site/markdown/Metrics.md | 96 +- .../src/site/markdown/NativeLibraries.md.vm | 1 + .../src/site/markdown/SecureMode.md | 2 +- .../src/site/markdown/SingleCluster.md.vm | 12 +- .../hadoop-common/src/site/markdown/Tracing.md | 2 +- .../src/site/markdown/UnixShellGuide.md | 9 + .../src/site/markdown/filesystem/filesystem.md | 19 +- .../site/markdown/filesystem/introduction.md | 4 +- .../3.0.0-alpha2/CHANGES.3.0.0-alpha2.md | 927 + .../3.0.0-alpha2/RELEASENOTES.3.0.0-alpha2.md | 618 + .../org/apache/hadoop/cli/CLITestHelper.java | 15 +- .../conf/TestCommonConfigurationFields.java | 11 +- .../apache/hadoop/conf/TestConfiguration.java | 18 +- .../conf/TestConfigurationFieldsBase.java | 20 +- .../key/TestKeyProviderCryptoExtension.java | 114 + .../apache/hadoop/crypto/key/TestKeyShell.java | 9 + .../fs/FileContextMainOperationsBaseTest.java | 21 +- .../hadoop/fs/FileSystemContractBaseTest.java | 17 +- .../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +- .../org/apache/hadoop/fs/TestDefaultUri.java | 40 +- .../hadoop/fs/TestDelegateToFsCheckPath.java | 35 +- .../org/apache/hadoop/fs/TestFileStatus.java | 19 + .../apache/hadoop/fs/TestFileSystemCaching.java | 8 +- .../apache/hadoop/fs/TestFilterFileSystem.java | 12 +- .../java/org/apache/hadoop/fs/TestFsShell.java | 67 +- .../org/apache/hadoop/fs/TestGlobPattern.java | 8 +- .../fs/contract/AbstractContractCreateTest.java | 96 +- .../AbstractContractGetFileStatusTest.java | 6 + .../hadoop/fs/contract/ContractTestUtils.java | 31 + .../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 56 +- .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 9 +- .../fs/viewfs/ViewFileSystemBaseTest.java | 264 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 22 + .../hadoop/ha/TestActiveStandbyElector.java | 31 + .../org/apache/hadoop/http/TestHttpServer.java | 40 +- .../hadoop/http/TestHttpServerWithSpengo.java | 252 + .../apache/hadoop/http/TestSSLHttpServer.java | 2 +- .../org/apache/hadoop/io/TestSequenceFile.java | 68 +- .../apache/hadoop/io/TestSequenceFileSync.java | 113 +- .../apache/hadoop/io/compress/TestCodec.java | 12 + .../io/compress/TestCompressionStreamReuse.java | 8 + .../TestZStandardCompressorDecompressor.java | 485 + .../erasurecode/TestCodecRawCoderMapping.java | 10 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../apache/hadoop/ipc/TestFairCallQueue.java | 57 +- .../ipc/TestProtoBufRpcServerHandoff.java | 167 + .../java/org/apache/hadoop/ipc/TestRPC.java | 148 +- .../apache/hadoop/ipc/TestRpcServerHandoff.java | 218 + .../org/apache/hadoop/log/TestLog4Json.java | 13 +- .../metrics2/filter/TestPatternFilter.java | 2 +- .../hadoop/metrics2/impl/ConfigBuilder.java | 11 +- .../apache/hadoop/metrics2/impl/ConfigUtil.java | 14 +- .../hadoop/metrics2/impl/MetricsRecords.java | 16 + .../metrics2/impl/TestMetricsCollectorImpl.java | 2 +- .../hadoop/metrics2/impl/TestMetricsConfig.java | 2 +- .../metrics2/impl/TestMetricsSystemImpl.java | 2 +- .../metrics2/lib/TestRollingAverages.java | 124 + .../sink/RollingFileSystemSinkTestBase.java | 2 +- .../sink/TestRollingFileSystemSink.java | 2 +- .../metrics2/sink/ganglia/TestGangliaSink.java | 2 +- .../apache/hadoop/net/TestClusterTopology.java | 5 +- .../security/TestAuthenticationFilter.java | 13 +- .../TestAuthenticationWithProxyUserFilter.java | 78 + .../security/TestFixKerberosTicketOrder.java | 158 + .../hadoop/security/TestGroupsCaching.java | 19 +- .../apache/hadoop/security/TestKDiagNoKDC.java | 8 + .../hadoop/security/TestRaceWhenRelogin.java | 162 + .../TestShellBasedUnixGroupsMapping.java | 135 +- .../hadoop/security/TestUGIWithMiniKdc.java | 1 + .../security/TestUserGroupInformation.java | 16 +- .../ssl/TestReloadingX509TrustManager.java | 18 + ...tionTokenAuthenticationHandlerWithMocks.java | 3 +- .../delegation/web/TestWebDelegationToken.java | 8 +- .../apache/hadoop/test/GenericTestUtils.java | 72 +- .../org/apache/hadoop/test/MetricsAsserts.java | 14 + .../hadoop/util/TestHttpExceptionUtils.java | 2 +- .../hadoop/util/TestReadWriteDiskValidator.java | 161 + .../java/org/apache/hadoop/util/TestShell.java | 52 + .../hadoop-common/src/test/proto/test.proto | 11 +- .../src/test/proto/test_rpc_service.proto | 6 +- .../src/test/resources/testConf.xml | 2 +- .../src/test/resources/zstd/test_file.txt | 71 + .../src/test/resources/zstd/test_file.txt.zst | Bin 0 -> 3690 bytes .../scripts/hadoop-functions_test_helper.bash | 6 +- .../src/test/scripts/hadoop_abs.bats | 65 + .../test/scripts/hadoop_get_verify_uservar.bats | 21 + .../src/test/scripts/hadoop_mkdir.bats | 42 + .../test/scripts/hadoop_privilege_check.bats | 26 + .../src/test/scripts/hadoop_using_envvar.bats | 33 + .../dev-support/findbugsExcludeFile.xml | 2 +- hadoop-common-project/hadoop-kms/pom.xml | 157 +- .../hadoop-kms/src/main/conf/kms-env.sh | 53 +- .../src/main/conf/kms-log4j.properties | 3 +- .../hadoop-kms/src/main/conf/kms-site.xml | 167 +- ...rKeyGeneratorKeyProviderCryptoExtension.java | 13 +- .../hadoop/crypto/key/kms/server/KMS.java | 71 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 36 +- .../hadoop/crypto/key/kms/server/KMSAudit.java | 34 +- .../crypto/key/kms/server/KMSAuditLogger.java | 21 +- .../crypto/key/kms/server/KMSConfiguration.java | 18 + .../crypto/key/kms/server/KMSJMXServlet.java | 36 - .../crypto/key/kms/server/KMSJSONReader.java | 3 +- .../crypto/key/kms/server/KMSJSONWriter.java | 3 +- .../hadoop/crypto/key/kms/server/KMSWebApp.java | 38 +- .../crypto/key/kms/server/KMSWebServer.java | 155 + .../kms/server/KeyAuthorizationKeyProvider.java | 24 + .../hadoop-kms/src/main/libexec/kms-config.sh | 77 - .../main/libexec/shellprofile.d/hadoop-kms.sh | 57 + .../src/main/resources/kms-default.xml | 262 + .../main/resources/webapps/kms/WEB-INF/web.xml | 68 + .../main/resources/webapps/static/index.html | 35 + .../hadoop-kms/src/main/sbin/kms.sh | 123 +- .../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 - .../hadoop-kms/src/main/tomcat/ROOT/index.html | 27 - .../src/main/tomcat/logging.properties | 67 - .../hadoop-kms/src/main/tomcat/server.xml | 155 - .../src/main/tomcat/ssl-server.xml.conf | 137 - .../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 78 - .../hadoop-kms/src/site/configuration.xsl | 49 + .../hadoop-kms/src/site/markdown/index.md.vm | 218 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 121 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 287 +- .../crypto/key/kms/server/TestKMSAudit.java | 13 +- hadoop-common-project/hadoop-minikdc/pom.xml | 5 +- hadoop-common-project/hadoop-nfs/pom.xml | 5 +- .../org/apache/hadoop/oncrpc/RpcProgram.java | 30 +- .../apache/hadoop/oncrpc/SimpleUdpClient.java | 11 +- .../apache/hadoop/oncrpc/SimpleUdpServer.java | 1 + hadoop-common-project/pom.xml | 5 +- hadoop-dist/pom.xml | 32 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 5 +- .../org/apache/hadoop/fs/HdfsBlockLocation.java | 17 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 22 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 48 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 5 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 105 +- .../hadoop/hdfs/DistributedFileSystem.java | 45 +- .../apache/hadoop/hdfs/HdfsConfiguration.java | 10 +- .../hadoop/hdfs/PositionStripeReader.java | 1 - .../apache/hadoop/hdfs/StripedDataStreamer.java | 8 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 29 +- .../hdfs/client/HdfsClientConfigKeys.java | 12 +- .../apache/hadoop/hdfs/protocol/BlockType.java | 57 + .../hadoop/hdfs/protocol/ClientProtocol.java | 14 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 8 +- .../hdfs/protocol/FsPermissionExtension.java | 2 + .../hadoop/hdfs/protocol/HdfsConstants.java | 2 + .../hadoop/hdfs/protocol/HdfsFileStatus.java | 2 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 4 + .../datatransfer/BlockPinningException.java | 33 + .../datatransfer/DataTransferProtoUtil.java | 17 +- .../ClientNamenodeProtocolTranslatorPB.java | 21 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 71 + .../token/block/BlockTokenIdentifier.java | 89 +- .../server/datanode/BlockMetadataHeader.java | 29 +- .../server/datanode/DiskBalancerWorkItem.java | 2 +- .../server/datanode/DiskBalancerWorkStatus.java | 4 +- .../hdfs/server/protocol/SlowPeerReports.java | 107 + .../hdfs/util/CombinedHostsFileReader.java | 2 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 104 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 125 +- ...onfRefreshTokenBasedAccessTokenProvider.java | 2 +- .../CredentialBasedAccessTokenProvider.java | 2 +- .../hdfs/web/resources/AclPermissionParam.java | 17 +- .../hdfs/web/resources/CreateParentParam.java | 2 +- .../hadoop/hdfs/web/resources/GetOpParam.java | 15 +- .../hdfs/web/resources/OverwriteParam.java | 2 +- .../hadoop/hdfs/web/resources/PostOpParam.java | 2 + .../hadoop/hdfs/web/resources/PutOpParam.java | 1 + .../hdfs/web/resources/StoragePolicyParam.java | 43 + .../src/main/proto/ClientNamenodeProtocol.proto | 2 + .../src/main/proto/datatransfer.proto | 1 + .../src/main/proto/erasurecoding.proto | 9 +- .../src/main/proto/hdfs.proto | 43 +- .../hadoop/hdfs/protocol/TestBlockType.java | 61 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 116 +- .../src/main/conf/httpfs-env.sh | 47 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 92 +- .../hadoop/fs/http/server/FSOperations.java | 130 + .../http/server/HttpFSAuthenticationFilter.java | 2 +- .../http/server/HttpFSParametersProvider.java | 23 + .../hadoop/fs/http/server/HttpFSServer.java | 35 + .../fs/http/server/HttpFSServerWebServer.java | 170 + .../apache/hadoop/lib/servlet/MDCFilter.java | 4 +- .../src/main/libexec/httpfs-config.sh | 76 - .../libexec/shellprofile.d/hadoop-httpfs.sh | 67 + .../src/main/resources/httpfs-default.xml | 72 + .../main/resources/webapps/static/index.html | 36 + .../resources/webapps/webhdfs/WEB-INF/web.xml | 98 + .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 126 +- .../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 - .../src/main/tomcat/ROOT/index.html | 21 - .../src/main/tomcat/logging.properties | 67 - .../src/main/tomcat/server.xml | 151 - .../src/main/tomcat/ssl-server.xml.conf | 136 - .../src/site/markdown/ServerSetup.md.vm | 146 +- .../src/site/markdown/index.md | 8 +- .../fs/http/client/BaseTestHttpFSWith.java | 52 +- .../http/server/TestHttpFSServerWebServer.java | 106 + .../hadoop-hdfs-native-client/pom.xml | 5 +- .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 2 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 +- .../hadoop/hdfs/nfs/conf/NfsConfigKeys.java | 6 + .../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 4 +- .../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 19 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 4 +- .../org/apache/hadoop/hdfs/nfs/TestMountd.java | 13 +- .../dev-support/findbugsExcludeFile.xml | 60 + .../jdiff/Apache_Hadoop_HDFS_3.0.0-alpha2.xml | 326 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 13 +- .../hadoop-hdfs/src/main/bin/hdfs | 17 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 50 +- .../src/main/bin/start-secure-dns.sh | 23 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 34 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 22 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 122 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 18 +- .../DatanodeProtocolClientSideTranslatorPB.java | 9 +- .../DatanodeProtocolServerSideTranslatorPB.java | 3 +- .../NamenodeProtocolServerSideTranslatorPB.java | 16 + .../NamenodeProtocolTranslatorPB.java | 15 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 154 +- .../hadoop/hdfs/qjournal/client/QuorumCall.java | 26 +- .../qjournal/client/QuorumJournalManager.java | 38 +- .../hadoop/hdfs/qjournal/server/JNStorage.java | 16 +- .../hadoop/hdfs/qjournal/server/Journal.java | 19 + .../hdfs/qjournal/server/JournalNode.java | 23 +- .../hdfs/qjournal/server/JournalNodeSyncer.java | 413 + .../token/block/BlockTokenSecretManager.java | 18 +- .../hadoop/hdfs/server/balancer/Balancer.java | 14 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 102 +- .../hadoop/hdfs/server/balancer/KeyManager.java | 6 +- .../server/blockmanagement/BlockIdManager.java | 16 +- .../hdfs/server/blockmanagement/BlockInfo.java | 7 +- .../blockmanagement/BlockInfoContiguous.java | 10 + .../blockmanagement/BlockInfoStriped.java | 27 +- .../server/blockmanagement/BlockManager.java | 237 +- .../blockmanagement/BlockPlacementPolicies.java | 13 +- .../BlockPlacementPolicyDefault.java | 10 +- .../BlockUnderConstructionFeature.java | 34 +- .../blockmanagement/DatanodeDescriptor.java | 28 +- .../server/blockmanagement/DatanodeManager.java | 64 +- .../blockmanagement/DatanodeStorageInfo.java | 5 + .../blockmanagement/DecommissionManager.java | 9 +- .../server/blockmanagement/NumberReplicas.java | 2 +- .../PendingReconstructionBlocks.java | 4 +- .../server/blockmanagement/SlowPeerTracker.java | 273 + .../hdfs/server/common/HdfsServerConstants.java | 9 +- .../server/common/HttpGetFailedException.java | 39 + .../server/common/HttpPutFailedException.java | 37 + .../hadoop/hdfs/server/common/Storage.java | 25 +- .../apache/hadoop/hdfs/server/common/Util.java | 296 +- .../hdfs/server/datanode/BPOfferService.java | 10 +- .../hdfs/server/datanode/BPServiceActor.java | 52 +- .../server/datanode/BlockPoolSliceStorage.java | 8 +- .../hdfs/server/datanode/BlockReceiver.java | 142 +- .../hdfs/server/datanode/BlockScanner.java | 10 +- .../hdfs/server/datanode/BlockSender.java | 131 +- .../hadoop/hdfs/server/datanode/DNConf.java | 31 +- .../hadoop/hdfs/server/datanode/DataNode.java | 493 +- .../server/datanode/DataNodeFaultInjector.java | 25 +- .../hdfs/server/datanode/DataNodeMXBean.java | 18 +- .../hdfs/server/datanode/DataStorage.java | 21 +- .../hdfs/server/datanode/DataXceiver.java | 19 +- .../hdfs/server/datanode/DatanodeUtil.java | 19 +- .../hdfs/server/datanode/DirectoryScanner.java | 12 +- .../datanode/FaultInjectorFileIoEvents.java | 55 + .../hdfs/server/datanode/FileIoProvider.java | 1051 + .../hdfs/server/datanode/FinalizedReplica.java | 29 + .../hdfs/server/datanode/LocalReplica.java | 198 +- .../server/datanode/LocalReplicaInPipeline.java | 53 +- .../server/datanode/ProfilingFileIoEvents.java | 141 + .../hdfs/server/datanode/ReplicaInfo.java | 17 +- .../server/datanode/ReplicaUnderRecovery.java | 2 +- .../hdfs/server/datanode/StorageLocation.java | 35 +- .../hdfs/server/datanode/VolumeScanner.java | 9 +- .../server/datanode/checker/AsyncChecker.java | 5 +- .../datanode/checker/DatasetVolumeChecker.java | 436 + .../checker/StorageLocationChecker.java | 240 + .../datanode/checker/ThrottledAsyncChecker.java | 19 +- .../datanode/checker/VolumeCheckResult.java | 43 + .../erasurecode/StripedBlockReader.java | 10 +- .../erasurecode/StripedBlockReconstructor.java | 7 +- .../erasurecode/StripedBlockWriter.java | 1 + .../datanode/erasurecode/StripedReader.java | 4 + .../erasurecode/StripedReconstructor.java | 31 + .../datanode/erasurecode/StripedWriter.java | 4 + .../fsdataset/DataNodeVolumeMetrics.java | 289 + .../server/datanode/fsdataset/FsDatasetSpi.java | 13 +- .../server/datanode/fsdataset/FsVolumeSpi.java | 28 +- .../datanode/fsdataset/ReplicaInputStreams.java | 105 +- .../fsdataset/ReplicaOutputStreams.java | 87 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 206 +- .../impl/FsDatasetAsyncDiskService.java | 13 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 63 +- .../datanode/fsdataset/impl/FsDatasetUtil.java | 5 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 140 +- .../fsdataset/impl/FsVolumeImplBuilder.java | 12 +- .../datanode/fsdataset/impl/FsVolumeList.java | 25 +- .../datanode/metrics/DataNodeDiskMetrics.java | 181 + .../datanode/metrics/DataNodeMetrics.java | 24 +- .../datanode/metrics/DataNodePeerMetrics.java | 148 + .../datanode/metrics/OutlierDetector.java | 182 + .../server/datanode/web/DatanodeHttpServer.java | 17 +- .../web/PortUnificationServerHandler.java | 106 - .../datanode/web/SimpleHttpProxyHandler.java | 18 +- .../datanode/web/dtp/DtpHttp2FrameListener.java | 52 - .../datanode/web/dtp/DtpHttp2Handler.java | 34 - .../datanode/web/webhdfs/ExceptionHandler.java | 29 +- .../server/datanode/web/webhdfs/HdfsWriter.java | 29 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 62 +- .../server/diskbalancer/command/Command.java | 2 +- .../connectors/JsonNodeConnector.java | 2 +- .../datamodel/DiskBalancerCluster.java | 2 +- .../datamodel/DiskBalancerVolume.java | 2 +- .../server/diskbalancer/planner/NodePlan.java | 2 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 35 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 4 +- .../hdfs/server/namenode/CacheManager.java | 2 +- .../hdfs/server/namenode/Checkpointer.java | 6 +- .../ContentSummaryComputationContext.java | 2 +- .../server/namenode/EditLogFileInputStream.java | 3 +- .../hdfs/server/namenode/EditsDoubleBuffer.java | 34 + .../server/namenode/EncryptionZoneManager.java | 8 +- .../namenode/ErasureCodingPolicyManager.java | 26 +- .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 6 +- .../hdfs/server/namenode/FSDirAttrOp.java | 16 +- .../server/namenode/FSDirErasureCodingOp.java | 164 +- .../hdfs/server/namenode/FSDirTruncateOp.java | 4 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 75 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 96 +- .../hdfs/server/namenode/FSEditLogLoader.java | 7 +- .../hdfs/server/namenode/FSEditLogOp.java | 29 +- .../hadoop/hdfs/server/namenode/FSImage.java | 22 +- .../hdfs/server/namenode/FSImageFormat.java | 6 +- .../server/namenode/FSImageFormatPBINode.java | 24 +- .../server/namenode/FSImageFormatProtobuf.java | 4 +- ...FSImagePreTransactionalStorageInspector.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 629 +- .../hdfs/server/namenode/FSNamesystemLock.java | 59 +- .../server/namenode/FSPermissionChecker.java | 87 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 115 +- .../server/namenode/INodeFileAttributes.java | 17 +- .../hdfs/server/namenode/ImageServlet.java | 15 +- .../hdfs/server/namenode/LeaseManager.java | 2 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 19 +- .../hadoop/hdfs/server/namenode/NameNode.java | 123 +- .../server/namenode/NameNodeHttpServer.java | 4 + .../hdfs/server/namenode/NameNodeMXBean.java | 15 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 42 +- .../server/namenode/NameNodeStatusMXBean.java | 14 +- .../hdfs/server/namenode/NamenodeFsck.java | 110 +- .../hdfs/server/namenode/SecondaryNameNode.java | 6 +- .../hdfs/server/namenode/TransferFsImage.java | 296 +- .../server/namenode/ha/BootstrapStandby.java | 10 +- .../server/namenode/ha/StandbyCheckpointer.java | 4 +- .../snapshot/FSImageFormatPBSnapshot.java | 11 +- .../web/resources/NamenodeWebHdfsMethods.java | 55 +- .../protocol/BalancerBandwidthCommand.java | 2 +- .../hdfs/server/protocol/DatanodeProtocol.java | 8 +- .../hdfs/server/protocol/NamenodeProtocol.java | 8 + .../hdfs/server/protocol/NamespaceInfo.java | 52 +- .../apache/hadoop/hdfs/tools/AdminHelper.java | 12 + .../apache/hadoop/hdfs/tools/CacheAdmin.java | 16 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 26 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 59 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 10 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 296 + .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 33 +- .../hadoop/hdfs/tools/erasurecode/ECCli.java | 62 - .../hdfs/tools/erasurecode/ECCommand.java | 214 - .../offlineImageViewer/FSImageHandler.java | 41 +- .../tools/offlineImageViewer/FSImageLoader.java | 7 +- .../OfflineImageReconstructor.java | 101 +- .../offlineImageViewer/PBImageXmlWriter.java | 37 +- .../hdfs/tools/snapshot/SnapshotDiff.java | 19 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 60 + .../hadoop-hdfs/src/main/java/overview.html | 274 - .../src/main/proto/DatanodeProtocol.proto | 30 +- .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 15 + .../src/main/proto/NamenodeProtocol.proto | 16 + .../hadoop-hdfs/src/main/proto/fsimage.proto | 5 +- .../src/main/resources/hdfs-default.xml | 153 +- .../src/main/webapps/datanode/datanode.html | 2 + .../src/main/webapps/hdfs/dfshealth.html | 51 +- .../src/main/webapps/hdfs/dfshealth.js | 88 +- .../src/main/webapps/hdfs/explorer.html | 13 +- .../src/main/webapps/hdfs/explorer.js | 37 +- .../src/main/webapps/static/d3-v4.1.1.min.js | 8 + .../src/main/webapps/static/hadoop.css | 18 +- .../src/site/markdown/ArchivalStorage.md | 14 +- .../site/markdown/CentralizedCacheManagement.md | 25 +- .../src/site/markdown/ExtendedAttributes.md | 8 +- .../hadoop-hdfs/src/site/markdown/Federation.md | 15 +- .../src/site/markdown/HDFSCommands.md | 68 +- .../src/site/markdown/HDFSDiskbalancer.md | 6 +- .../src/site/markdown/HDFSErasureCoding.md | 35 +- .../markdown/HDFSHighAvailabilityWithNFS.md | 30 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 31 +- .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 36 +- .../src/site/markdown/HdfsEditsViewer.md | 78 +- .../src/site/markdown/HdfsImageViewer.md | 74 +- .../src/site/markdown/HdfsMultihoming.md | 9 +- .../src/site/markdown/HdfsNfsGateway.md | 9 +- .../src/site/markdown/HdfsPermissionsGuide.md | 15 +- .../src/site/markdown/HdfsQuotaAdminGuide.md | 10 +- .../src/site/markdown/HdfsRollingUpgrade.md | 26 +- .../src/site/markdown/HdfsSnapshots.md | 16 +- .../src/site/markdown/HdfsUserGuide.md | 23 +- .../hadoop-hdfs/src/site/markdown/LibHdfs.md | 10 +- .../src/site/markdown/MemoryStorage.md | 14 +- .../src/site/markdown/SLGUserGuide.md | 7 +- .../src/site/markdown/ShortCircuitLocalReads.md | 7 +- .../src/site/markdown/TransparentEncryption.md | 29 +- .../hadoop-hdfs/src/site/markdown/ViewFs.md | 15 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 553 +- .../hadoop/cli/CLITestCmdErasureCoding.java | 4 +- .../apache/hadoop/cli/TestCryptoAdminCLI.java | 3 +- .../cli/util/ErasureCodingCliCmdExecutor.java | 6 +- .../hadoop/fs/permission/TestStickyBit.java | 63 + .../fs/viewfs/TestViewFileSystemHdfs.java | 110 + .../apache/hadoop/hdfs/AdminStatesBaseTest.java | 155 +- .../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 39 +- .../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 + .../hdfs/ErasureCodeBenchmarkThroughput.java | 3 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 23 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 140 +- .../hadoop/hdfs/TestAppendSnapshotTruncate.java | 2 - .../hadoop/hdfs/TestBlockStoragePolicy.java | 4 +- .../TestClientProtocolForPipelineRecovery.java | 3 +- .../hadoop/hdfs/TestDFSNetworkTopology.java | 260 + .../apache/hadoop/hdfs/TestDFSOutputStream.java | 22 +- .../TestDFSRSDefault10x4StripedInputStream.java | 35 + ...TestDFSRSDefault10x4StripedOutputStream.java | 36 + ...fault10x4StripedOutputStreamWithFailure.java | 36 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 213 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 34 +- .../TestDFSStripedOutputStreamWithFailure.java | 278 +- ...estDFSStripedOutputStreamWithFailure000.java | 8 +- ...estDFSStripedOutputStreamWithFailure010.java | 8 +- ...estDFSStripedOutputStreamWithFailure020.java | 8 +- ...estDFSStripedOutputStreamWithFailure030.java | 8 +- ...estDFSStripedOutputStreamWithFailure040.java | 8 +- ...estDFSStripedOutputStreamWithFailure050.java | 8 +- ...estDFSStripedOutputStreamWithFailure060.java | 8 +- ...estDFSStripedOutputStreamWithFailure070.java | 8 +- ...estDFSStripedOutputStreamWithFailure080.java | 8 +- ...estDFSStripedOutputStreamWithFailure090.java | 8 +- ...estDFSStripedOutputStreamWithFailure100.java | 8 +- ...estDFSStripedOutputStreamWithFailure110.java | 8 +- ...estDFSStripedOutputStreamWithFailure120.java | 8 +- ...estDFSStripedOutputStreamWithFailure130.java | 8 +- ...estDFSStripedOutputStreamWithFailure140.java | 8 +- ...estDFSStripedOutputStreamWithFailure150.java | 8 +- ...estDFSStripedOutputStreamWithFailure160.java | 8 +- ...estDFSStripedOutputStreamWithFailure170.java | 8 +- ...estDFSStripedOutputStreamWithFailure180.java | 8 +- ...estDFSStripedOutputStreamWithFailure190.java | 8 +- ...estDFSStripedOutputStreamWithFailure200.java | 8 +- ...estDFSStripedOutputStreamWithFailure210.java | 9 +- .../hdfs/TestDFSXORStripedInputStream.java | 33 + .../hdfs/TestDFSXORStripedOutputStream.java | 35 + ...estDFSXORStripedOutputStreamWithFailure.java | 36 + .../apache/hadoop/hdfs/TestDecommission.java | 43 + .../hdfs/TestDecommissionWithStriped.java | 59 +- .../apache/hadoop/hdfs/TestDeprecatedKeys.java | 22 +- .../hadoop/hdfs/TestEncryptedTransfer.java | 2 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 24 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 47 +- .../TestErasureCodingPolicyWithSnapshot.java | 19 +- .../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 16 +- .../hadoop/hdfs/TestFileCreationDelete.java | 11 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 2 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 76 +- .../hadoop/hdfs/TestMaintenanceState.java | 165 +- .../hadoop/hdfs/TestMissingBlocksAlert.java | 3 +- .../java/org/apache/hadoop/hdfs/TestPread.java | 51 + .../java/org/apache/hadoop/hdfs/TestQuota.java | 26 + .../hdfs/TestReadStripedFileWithDecoding.java | 35 +- .../TestReadStripedFileWithMissingBlocks.java | 29 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 24 +- .../apache/hadoop/hdfs/TestRenameWhileOpen.java | 45 +- .../hdfs/TestReplaceDatanodeOnFailure.java | 3 +- .../org/apache/hadoop/hdfs/TestReplication.java | 3 +- .../org/apache/hadoop/hdfs/TestSafeMode.java | 1 + .../hdfs/TestSafeModeWithStripedFile.java | 23 +- .../hadoop/hdfs/TestSnapshotCommands.java | 34 + .../TestUnsetAndChangeDirectoryEcPolicy.java | 366 + .../hadoop/hdfs/TestWriteReadStripedFile.java | 29 +- .../hdfs/TestWriteStripedFileWithFailure.java | 19 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 40 +- .../hdfs/qjournal/MiniJournalCluster.java | 8 + .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 1 + .../hdfs/qjournal/TestJournalNodeSync.java | 264 + .../hdfs/qjournal/client/TestQuorumCall.java | 17 + .../hdfs/qjournal/server/TestJournal.java | 17 + .../security/token/block/TestBlockToken.java | 297 +- .../hdfs/server/balancer/TestBalancer.java | 58 +- .../blockmanagement/BlockManagerTestUtil.java | 25 +- .../server/blockmanagement/TestBlockInfo.java | 20 +- .../blockmanagement/TestBlockInfoStriped.java | 95 +- .../blockmanagement/TestBlockStatsMXBean.java | 4 + .../TestBlockTokenWithDFSStriped.java | 20 +- .../TestBlocksWithNotEnoughRacks.java | 4 +- .../blockmanagement/TestHeartbeatHandling.java | 10 + .../TestNameNodePrunesMissingStorages.java | 4 +- .../server/blockmanagement/TestNodeCount.java | 2 +- .../TestPendingInvalidateBlock.java | 2 +- .../TestPendingReconstruction.java | 2 +- .../TestRBWBlockInvalidation.java | 45 +- ...constructStripedBlocksWithRackAwareness.java | 75 +- .../blockmanagement/TestReplicationPolicy.java | 37 +- .../TestReplicationPolicyConsiderLoad.java | 2 +- .../TestReplicationPolicyWithNodeGroup.java | 23 +- .../TestSequentialBlockGroupId.java | 15 +- .../blockmanagement/TestSlowPeerTracker.java | 226 + .../TestSortLocatedStripedBlock.java | 45 +- .../hdfs/server/common/StorageAdapter.java | 2 +- .../hdfs/server/datanode/DataNodeTestUtils.java | 108 + .../datanode/InternalDataNodeTestUtils.java | 6 +- .../server/datanode/SimulatedFSDataset.java | 67 +- .../server/datanode/TestBPOfferService.java | 38 +- .../datanode/TestBlockPoolSliceStorage.java | 2 +- .../hdfs/server/datanode/TestBlockRecovery.java | 63 +- .../server/datanode/TestBlockReplacement.java | 70 +- .../hdfs/server/datanode/TestBlockScanner.java | 100 + .../datanode/TestBpServiceActorScheduler.java | 30 +- .../hdfs/server/datanode/TestDataDirs.java | 27 - .../TestDataNodeErasureCodingMetrics.java | 194 +- .../datanode/TestDataNodeFSDataSetSink.java | 2 +- .../datanode/TestDataNodeFaultInjector.java | 173 + .../datanode/TestDataNodeHotSwapVolumes.java | 184 +- .../server/datanode/TestDataNodeLifeline.java | 40 +- .../datanode/TestDataNodePeerMetrics.java | 94 + .../hdfs/server/datanode/TestDataNodeUUID.java | 2 +- .../datanode/TestDataNodeVolumeFailure.java | 120 +- .../TestDataNodeVolumeFailureReporting.java | 78 +- .../TestDataNodeVolumeFailureToleration.java | 9 + .../datanode/TestDataNodeVolumeMetrics.java | 182 + .../TestDatanodeProtocolRetryPolicy.java | 6 +- .../server/datanode/TestDirectoryScanner.java | 35 +- .../hdfs/server/datanode/TestDiskError.java | 24 +- .../server/datanode/TestFsDatasetCache.java | 3 +- .../hdfs/server/datanode/TestStorageReport.java | 4 +- .../checker/TestDatasetVolumeChecker.java | 226 + .../TestDatasetVolumeCheckerFailures.java | 177 + .../checker/TestStorageLocationChecker.java | 217 + .../checker/TestThrottledAsyncChecker.java | 128 +- .../extdataset/ExternalDatasetImpl.java | 9 +- .../extdataset/ExternalReplicaInPipeline.java | 6 +- .../datanode/extdataset/ExternalVolumeImpl.java | 26 + .../fsdataset/impl/TestFsDatasetImpl.java | 84 +- .../fsdataset/impl/TestFsVolumeList.java | 37 - .../fsdataset/impl/TestLazyPersistFiles.java | 12 +- .../TestDataNodeOutlierDetectionViaMetrics.java | 142 + .../datanode/metrics/TestSlowNodeDetector.java | 335 + .../datanode/web/dtp/Http2ResponseHandler.java | 65 - .../server/datanode/web/dtp/TestDtpHttp2.java | 147 - .../hadoop/hdfs/server/mover/TestMover.java | 197 +- .../hdfs/server/mover/TestStorageMover.java | 4 +- .../hdfs/server/namenode/CreateEditsLog.java | 2 +- .../server/namenode/NNThroughputBenchmark.java | 17 +- .../hdfs/server/namenode/NameNodeAdapter.java | 4 +- .../TestAddOverReplicatedStripedBlocks.java | 76 +- .../namenode/TestAddStripedBlockInFBR.java | 57 +- .../server/namenode/TestAddStripedBlocks.java | 75 +- .../hdfs/server/namenode/TestAuditLogger.java | 73 + .../hdfs/server/namenode/TestDeadDatanode.java | 7 +- .../namenode/TestDecommissioningStatus.java | 5 +- .../server/namenode/TestEditsDoubleBuffer.java | 54 + .../namenode/TestEncryptionZoneManager.java | 138 + .../hdfs/server/namenode/TestFSDirAttrOp.java | 76 + .../server/namenode/TestFSEditLogLoader.java | 19 +- .../hdfs/server/namenode/TestFSImage.java | 134 +- .../hdfs/server/namenode/TestFSNamesystem.java | 21 + .../server/namenode/TestFSNamesystemLock.java | 59 +- .../server/namenode/TestFSNamesystemMBean.java | 34 + .../hdfs/server/namenode/TestFileTruncate.java | 31 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 433 +- .../hdfs/server/namenode/TestHostsFiles.java | 4 +- .../hdfs/server/namenode/TestINodeFile.java | 80 +- .../hdfs/server/namenode/TestMetaSave.java | 5 +- .../server/namenode/TestNameNodeMXBean.java | 228 +- .../namenode/TestNameNodeMetricsLogger.java | 11 +- .../namenode/TestQuotaWithStripedBlocks.java | 23 +- .../namenode/TestReconstructStripedBlocks.java | 68 +- .../server/namenode/TestStripedINodeFile.java | 90 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 11 +- .../ha/TestDNFencingWithReplication.java | 2 +- .../namenode/ha/TestPipelinesFailover.java | 5 +- .../TestNNMetricFilesInGetListingOps.java | 2 +- .../namenode/metrics/TestNameNodeMetrics.java | 14 +- .../shortcircuit/TestShortCircuitLocalRead.java | 2 +- .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 16 +- .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 13 +- .../hdfs/tools/TestDelegationTokenFetcher.java | 7 +- .../hdfs/tools/TestStoragePolicyCommands.java | 14 +- .../TestOfflineImageViewer.java | 81 +- ...TestOfflineImageViewerWithStripedBlocks.java | 32 +- .../hadoop/hdfs/util/TestStripedBlockUtil.java | 138 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 8 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 412 +- .../hdfs/web/TestWebHdfsFileSystemContract.java | 40 + .../hadoop/hdfs/web/resources/TestParam.java | 42 + .../apache/hadoop/net/TestNetworkTopology.java | 37 +- .../hadoop/tools/TestHdfsConfigFields.java | 17 +- .../src/test/resources/testCryptoConf.xml | 98 +- .../test/resources/testErasureCodingConf.xml | 147 +- hadoop-hdfs-project/pom.xml | 5 +- hadoop-mapreduce-project/bin/mapred | 10 + .../hadoop-mapreduce-client-app/pom.xml | 5 +- .../hadoop/mapred/TaskAttemptListenerImpl.java | 17 +- .../org/apache/hadoop/mapred/YarnChild.java | 2 + .../jobhistory/JobHistoryCopyService.java | 5 +- .../jobhistory/JobHistoryEventHandler.java | 65 +- .../hadoop/mapreduce/v2/app/AppContext.java | 4 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 30 +- .../v2/app/client/MRClientService.java | 4 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 4 +- .../mapreduce/v2/app/job/impl/TaskImpl.java | 1 + .../v2/app/rm/RMContainerAllocator.java | 24 +- .../v2/app/speculate/DefaultSpeculator.java | 3 +- .../mapreduce/v2/app/webapp/AppController.java | 41 +- .../mapreduce/v2/app/webapp/ConfBlock.java | 2 +- .../mapreduce/v2/app/webapp/TaskPage.java | 2 +- .../mapreduce/v2/app/webapp/TasksBlock.java | 2 +- .../mapreduce/v2/app/webapp/dao/JobInfo.java | 4 +- .../mapred/TestLocalContainerLauncher.java | 2 +- .../mapred/TestTaskAttemptFinishingMonitor.java | 2 +- .../mapred/TestTaskAttemptListenerImpl.java | 17 +- .../jobhistory/TestJobHistoryEventHandler.java | 18 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 14 - .../hadoop/mapreduce/v2/app/MockAppContext.java | 4 +- .../mapreduce/v2/app/MockEventHandler.java | 6 +- .../v2/app/TestCheckpointPreemptionPolicy.java | 4 +- .../mapreduce/v2/app/TestFetchFailure.java | 17 +- .../hadoop/mapreduce/v2/app/TestKill.java | 11 +- .../v2/app/TestKillAMPreemptionPolicy.java | 4 +- .../mapreduce/v2/app/TestRuntimeEstimators.java | 3 +- .../app/commit/TestCommitterEventHandler.java | 2 +- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 14 +- .../app/launcher/TestContainerLauncherImpl.java | 12 +- .../app/local/TestLocalContainerAllocator.java | 4 +- .../v2/app/rm/TestRMContainerAllocator.java | 12 +- .../v2/app/webapp/TestAppController.java | 14 + .../hadoop-mapreduce-client-common/pom.xml | 5 +- .../apache/hadoop/mapreduce/TypeConverter.java | 7 +- .../hadoop-mapreduce-client-core/pom.xml | 21 +- .../org/apache/hadoop/mapred/CleanupQueue.java | 7 +- .../java/org/apache/hadoop/mapred/JobConf.java | 2 +- .../java/org/apache/hadoop/mapred/MapTask.java | 16 +- .../java/org/apache/hadoop/mapred/Mapper.java | 2 +- .../java/org/apache/hadoop/mapred/Reducer.java | 2 +- .../java/org/apache/hadoop/mapred/Task.java | 24 +- .../java/org/apache/hadoop/mapreduce/Job.java | 2 +- .../hadoop/mapreduce/JobResourceUploader.java | 14 +- .../org/apache/hadoop/mapreduce/JobStatus.java | 18 +- .../hadoop/mapreduce/JobSubmissionFiles.java | 5 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 2 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 20 +- .../apache/hadoop/mapreduce/TaskCounter.java | 8 +- .../counters/FrameworkCounterGroup.java | 6 +- .../lib/output/FileOutputCommitter.java | 12 +- .../lib/output/PartialFileOutputCommitter.java | 2 +- .../mapreduce/lib/output/TextOutputFormat.java | 15 +- .../mapreduce/lib/partition/InputSampler.java | 6 +- .../mapreduce/task/reduce/MergeManagerImpl.java | 3 +- .../task/reduce/ShuffleSchedulerImpl.java | 40 +- .../src/main/resources/mapred-default.xml | 26 + .../hadoop/mapreduce/TaskCounter.properties | 4 + .../site/markdown/DistributedCacheDeploy.md.vm | 2 + .../src/site/markdown/EncryptedShuffle.md | 2 + .../src/site/markdown/MapReduceTutorial.md | 47 +- .../MapReduce_Compatibility_Hadoop1_Hadoop2.md | 2 + .../src/site/markdown/MapredAppMasterRest.md | 16 +- .../src/site/markdown/MapredCommands.md | 20 +- .../PluggableShuffleAndPluggableSort.md | 2 + .../org/apache/hadoop/mapred/TestCounters.java | 31 +- .../TestPreemptableFileOutputCommitter.java | 2 +- .../mapreduce/task/reduce/TestMergeManager.java | 7 + .../hadoop-mapreduce-client-hs-plugins/pom.xml | 5 +- .../hadoop-mapreduce-client-hs/pom.xml | 5 +- .../mapreduce/v2/hs/HistoryFileManager.java | 48 +- ...istoryServerFileSystemStateStoreService.java | 2 +- .../hadoop/mapreduce/v2/hs/JobHistory.java | 3 +- .../mapreduce/v2/hs/webapp/HsJobsBlock.java | 2 +- .../hadoop/mapreduce/v2/hs/webapp/HsWebApp.java | 2 + .../mapreduce/v2/hs/webapp/dao/JobInfo.java | 15 +- .../src/site/markdown/HistoryServerRest.md | 22 +- .../mapreduce/v2/hs/webapp/dao/TestJobInfo.java | 29 +- .../resources/job_1329348432655_0001_conf.xml | 2 - .../hadoop-mapreduce-client-jobclient/pom.xml | 5 +- .../hadoop/mapred/ClientServiceDelegate.java | 4 +- .../hadoop/mapred/ResourceMgrDelegate.java | 6 +- .../org/apache/hadoop/mapred/YARNRunner.java | 177 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 2 +- .../apache/hadoop/mapred/JobClientUnitTest.java | 6 +- .../java/org/apache/hadoop/mapred/MRBench.java | 24 +- .../apache/hadoop/mapred/TestJobCounters.java | 188 + .../hadoop/mapred/TestMROpportunisticMaps.java | 20 +- .../apache/hadoop/mapred/TestYARNRunner.java | 39 + .../mapred/pipes/TestPipeApplication.java | 37 +- .../mapreduce/security/CredentialsTestJob.java | 2 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 8 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 25 +- .../src/CMakeLists.txt | 10 +- .../src/main/native/gtest/gtest-all.cc | 9592 -------- .../src/main/native/gtest/include/gtest/gtest.h | 20061 ---------------- .../src/main/native/src/lib/Log.h | 2 +- .../hadoop-mapreduce-client-shuffle/pom.xml | 5 +- .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- .../hadoop-mapreduce-client/pom.xml | 5 +- .../hadoop-mapreduce-examples/pom.xml | 5 +- .../hadoop/examples/dancing/Pentomino.java | 5 + .../org/apache/hadoop/examples/pi/package.html | 50 +- .../examples/terasort/TeraOutputFormat.java | 4 +- .../hadoop/examples/terasort/TeraScheduler.java | 16 +- hadoop-mapreduce-project/pom.xml | 5 +- hadoop-maven-plugins/pom.xml | 33 +- .../maven/plugin/protoc/ProtocRunner.java | 5 +- .../resource/ServicesResourceTransformer.java | 164 + .../plugin/shade/resource/package-info.java | 23 + hadoop-minicluster/pom.xml | 5 +- hadoop-project-dist/pom.xml | 13 +- hadoop-project/pom.xml | 199 +- hadoop-project/src/site/markdown/index.md.vm | 40 +- hadoop-project/src/site/site.xml | 5 +- hadoop-tools/hadoop-aliyun/pom.xml | 2 +- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 5 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 47 +- .../fs/aliyun/oss/AliyunOSSInputStream.java | 4 +- .../oss/TestAliyunOSSFileSystemContract.java | 7 + .../fs/aliyun/oss/TestAliyunOSSInputStream.java | 9 + .../oss/contract/TestAliyunOSSContractSeek.java | 26 + .../src/test/resources/contract/aliyun-oss.xml | 5 + hadoop-tools/hadoop-archive-logs/pom.xml | 5 +- .../apache/hadoop/tools/HadoopArchiveLogs.java | 4 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 12 +- .../tools/TestHadoopArchiveLogsRunner.java | 11 +- hadoop-tools/hadoop-archives/pom.xml | 5 +- .../org/apache/hadoop/tools/HadoopArchives.java | 17 +- hadoop-tools/hadoop-aws/pom.xml | 22 +- .../s3a/BlockingThreadPoolExecutorService.java | 2 +- .../org/apache/hadoop/fs/s3a/Constants.java | 36 +- .../hadoop/fs/s3a/RenameFailedException.java | 70 + .../hadoop/fs/s3a/S3ABlockOutputStream.java | 69 +- .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 540 +- .../hadoop/fs/s3a/S3AEncryptionMethods.java | 61 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 335 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 57 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 40 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 159 +- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 6 +- .../hadoop/fs/s3a/S3ObjectAttributes.java | 59 + .../org/apache/hadoop/fs/s3a/Statistic.java | 4 +- .../hadoop/fs/s3native/NativeS3FileSystem.java | 7 +- .../hadoop/fs/s3native/S3xLoginHelper.java | 15 +- .../services/org.apache.hadoop.fs.FileSystem | 1 - .../src/site/markdown/tools/hadoop-aws/index.md | 698 +- .../site/markdown/tools/hadoop-aws/testing.md | 814 + .../tools/hadoop-aws/troubleshooting_s3a.md | 52 + .../fs/s3a/AbstractTestS3AEncryption.java | 132 + .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 4 +- .../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 76 +- .../fs/s3a/ITestS3ABlockOutputByteBuffer.java | 5 +- .../hadoop/fs/s3a/ITestS3ABlockOutputDisk.java | 12 + .../hadoop/fs/s3a/ITestS3AConfiguration.java | 135 +- .../hadoop/fs/s3a/ITestS3AEncryption.java | 97 - .../ITestS3AEncryptionAlgorithmPropagation.java | 76 - .../ITestS3AEncryptionAlgorithmValidation.java | 152 + .../ITestS3AEncryptionBlockOutputStream.java | 36 - .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 90 + ...ITestS3AEncryptionSSECBlockOutputStream.java | 46 + .../s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 57 + .../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 48 + ...onSSEKMSUserDefinedKeyBlockOutputStream.java | 52 + .../hadoop/fs/s3a/ITestS3AEncryptionSSES3.java | 43 + ...TestS3AEncryptionSSES3BlockOutputStream.java | 45 + .../fs/s3a/ITestS3AFileSystemContract.java | 15 - .../apache/hadoop/fs/s3a/S3ATestConstants.java | 12 - .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 76 +- .../apache/hadoop/fs/s3a/TestDataBlocks.java | 45 +- .../fs/s3a/TestS3AExceptionTranslation.java | 38 + .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 57 +- .../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 28 +- .../fs/s3a/scale/ITestS3AConcurrentOps.java | 167 + .../scale/ITestS3AInputStreamPerformance.java | 57 +- .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 28 +- .../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 14 +- .../NativeS3FileSystemContractBaseTest.java | 2 +- .../hadoop/fs/s3native/TestS3xLoginHelper.java | 28 + .../src/test/resources/contract/s3a.xml | 2 +- .../dev-support/findbugs-exclude.xml | 24 - hadoop-tools/hadoop-azure-datalake/pom.xml | 42 +- .../main/java/org/apache/hadoop/fs/adl/Adl.java | 4 + .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 96 + .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 950 +- .../apache/hadoop/fs/adl/AdlFsInputStream.java | 149 + .../apache/hadoop/fs/adl/AdlFsOutputStream.java | 82 + .../org/apache/hadoop/fs/adl/AdlPermission.java | 69 + .../hadoop/fs/adl/SdkTokenProviderAdapter.java | 41 + .../apache/hadoop/fs/adl/TokenProviderType.java | 25 + .../fs/adl/oauth2/AzureADTokenProvider.java | 70 + ...hedRefreshTokenBasedAccessTokenProvider.java | 135 - .../hadoop/fs/adl/oauth2/package-info.java | 2 +- .../org/apache/hadoop/fs/adl/package-info.java | 2 +- .../org/apache/hadoop/hdfs/web/ADLConfKeys.java | 61 - .../apache/hadoop/hdfs/web/BufferManager.java | 180 - .../web/PrivateAzureDataLakeFileSystem.java | 1116 - ...ClientCredentialBasedAccesTokenProvider.java | 156 - ...hedRefreshTokenBasedAccessTokenProvider.java | 37 - .../hadoop/hdfs/web/oauth2/package-info.java | 24 - .../apache/hadoop/hdfs/web/package-info.java | 25 - .../hadoop/hdfs/web/resources/ADLFlush.java | 49 - .../hdfs/web/resources/ADLGetOpParam.java | 96 - .../hdfs/web/resources/ADLPostOpParam.java | 97 - .../hdfs/web/resources/ADLPutOpParam.java | 94 - .../hdfs/web/resources/ADLVersionInfo.java | 51 - .../web/resources/AppendADLNoRedirectParam.java | 45 - .../web/resources/CreateADLNoRedirectParam.java | 44 - .../hadoop/hdfs/web/resources/LeaseParam.java | 53 - .../web/resources/ReadADLNoRedirectParam.java | 44 - .../hadoop/hdfs/web/resources/package-info.java | 27 - .../META-INF/org.apache.hadoop.fs.FileSystem | 16 + .../src/site/markdown/index.md | 319 +- .../apache/hadoop/fs/adl/AdlMockWebServer.java | 99 + .../apache/hadoop/fs/adl/TestACLFeatures.java | 262 + .../hadoop/fs/adl/TestADLResponseData.java | 86 +- .../org/apache/hadoop/fs/adl/TestAdlRead.java | 196 + .../hadoop/fs/adl/TestAzureADTokenProvider.java | 285 + .../adl/TestConcurrentDataReadOperations.java | 299 + .../hadoop/fs/adl/TestCustomTokenProvider.java | 136 + .../apache/hadoop/fs/adl/TestGetFileStatus.java | 58 +- .../apache/hadoop/fs/adl/TestListStatus.java | 72 +- .../fs/adl/TestRelativePathFormation.java | 61 + .../fs/adl/TestValidateConfiguration.java | 112 + .../hadoop/fs/adl/TestableAdlFileSystem.java | 3 +- .../fs/adl/common/CustomMockTokenProvider.java | 61 + .../hadoop/fs/adl/common/ExpectedResponse.java | 71 + .../hadoop/fs/adl/common/Parallelized.java | 60 + .../hadoop/fs/adl/common/TestDataForRead.java | 122 + .../fs/adl/live/AdlStorageConfiguration.java | 54 +- .../hadoop/fs/adl/live/AdlStorageContract.java | 19 +- .../fs/adl/live/TestAdlContractAppendLive.java | 11 +- .../fs/adl/live/TestAdlContractConcatLive.java | 23 +- .../fs/adl/live/TestAdlContractCreateLive.java | 19 +- .../fs/adl/live/TestAdlContractDeleteLive.java | 11 +- .../live/TestAdlContractGetFileStatusLive.java | 36 + .../fs/adl/live/TestAdlContractMkdirLive.java | 25 +- .../fs/adl/live/TestAdlContractOpenLive.java | 11 +- .../fs/adl/live/TestAdlContractRenameLive.java | 30 +- .../fs/adl/live/TestAdlContractRootDirLive.java | 19 +- .../fs/adl/live/TestAdlContractSeekLive.java | 11 +- .../live/TestAdlDifferentSizeWritesLive.java | 71 +- .../live/TestAdlFileContextCreateMkdirLive.java | 67 + .../TestAdlFileContextMainOperationsLive.java | 99 + .../adl/live/TestAdlFileSystemContractLive.java | 59 +- .../live/TestAdlInternalCreateNonRecursive.java | 134 + .../fs/adl/live/TestAdlPermissionLive.java | 116 + .../hadoop/fs/adl/live/TestAdlReadLive.java | 342 - .../adl/live/TestAdlSupportedCharsetInPath.java | 334 + ...estAdlWebHdfsFileContextCreateMkdirLive.java | 79 - ...AdlWebHdfsFileContextMainOperationsLive.java | 104 - .../apache/hadoop/fs/adl/live/TestMetadata.java | 144 + ...hedRefreshTokenBasedAccessTokenProvider.java | 149 - .../hadoop/fs/common/AdlMockWebServer.java | 116 - .../hadoop/fs/common/ExpectedResponse.java | 72 - .../hadoop/fs/common/TestDataForRead.java | 120 - .../org/apache/hadoop/hdfs/web/TestAdlRead.java | 205 - .../web/TestConcurrentDataReadOperations.java | 306 - .../hdfs/web/TestConfigurationSetting.java | 138 - .../hdfs/web/TestSplitSizeCalculation.java | 123 - .../src/test/resources/adls.xml | 22 +- .../test/resources/contract-test-options.xml | 57 - .../src/test/resources/log4j.properties | 30 + hadoop-tools/hadoop-azure/pom.xml | 8 +- .../fs/azure/AzureNativeFileSystemStore.java | 122 +- .../org/apache/hadoop/fs/azure/KeyProvider.java | 3 +- .../fs/azure/LocalSASKeyGeneratorImpl.java | 263 + .../hadoop/fs/azure/NativeAzureFileSystem.java | 56 +- .../hadoop/fs/azure/NativeFileSystemStore.java | 28 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 296 + .../fs/azure/SASKeyGenerationException.java | 40 + .../hadoop/fs/azure/SASKeyGeneratorImpl.java | 61 + .../fs/azure/SASKeyGeneratorInterface.java | 66 + .../hadoop/fs/azure/SecureModeException.java | 40 + .../fs/azure/SecureStorageInterfaceImpl.java | 565 + .../hadoop/fs/azure/SelfRenewingLease.java | 2 +- .../hadoop/fs/azure/SendRequestIntercept.java | 3 +- .../hadoop/fs/azure/StorageInterface.java | 28 +- .../org/apache/hadoop/fs/azure/WasbFsck.java | 8 +- .../fs/azure/WasbRemoteCallException.java | 41 + .../hadoop/fs/azure/WasbRemoteCallHelper.java | 93 + .../java/org/apache/hadoop/fs/azure/Wasbs.java | 47 + .../metrics/AzureFileSystemInstrumentation.java | 2 + .../hadoop-azure/src/site/markdown/index.md | 48 + .../fs/azure/AzureBlobStorageTestAccount.java | 2 +- .../hadoop/fs/azure/TestContainerChecks.java | 12 +- ...estNativeAzureFileSystemConcurrencyLive.java | 119 + .../fs/azure/TestWasbUriAndConfiguration.java | 74 +- .../src/test/resources/azure-test.xml | 15 +- hadoop-tools/hadoop-datajoin/pom.xml | 5 +- hadoop-tools/hadoop-distcp/pom.xml | 5 +- .../org/apache/hadoop/tools/DistCpSync.java | 4 +- .../apache/hadoop/tools/SimpleCopyListing.java | 16 +- .../hadoop/tools/mapred/CopyCommitter.java | 4 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 5 +- .../tools/mapred/RetriableFileCopyCommand.java | 9 +- .../apache/hadoop/tools/util/DistCpUtils.java | 4 +- .../src/site/markdown/DistCp.md.vm | 132 +- .../src/test/resources/log4j.properties | 22 + hadoop-tools/hadoop-extras/pom.xml | 5 +- .../java/org/apache/hadoop/tools/DistTool.java | 10 +- hadoop-tools/hadoop-gridmix/pom.xml | 5 +- .../gridmix/DummyResourceCalculatorPlugin.java | 4 + hadoop-tools/hadoop-kafka/pom.xml | 4 +- .../apache/hadoop/metrics2/sink/KafkaSink.java | 2 +- .../hadoop/metrics2/impl/TestKafkaMetrics.java | 2 +- hadoop-tools/hadoop-openstack/pom.xml | 4 +- .../fs/swift/auth/PasswordCredentials.java | 3 +- .../fs/swift/snative/SwiftNativeFileSystem.java | 2 + .../hadoop/fs/swift/util/SwiftTestUtils.java | 9 +- hadoop-tools/hadoop-pipes/pom.xml | 5 +- hadoop-tools/hadoop-rumen/pom.xml | 10 +- .../hadoop/tools/rumen/state/StatePool.java | 25 +- hadoop-tools/hadoop-sls/pom.xml | 5 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 4 +- .../src/main/data/2jobs2min-rumen-jh.json | 6 - .../org/apache/hadoop/yarn/sls/SLSRunner.java | 20 +- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 89 +- .../yarn/sls/appmaster/MRAMSimulator.java | 218 +- .../sls/resourcemanager/MockAMLauncher.java | 115 + .../sls/scheduler/ResourceSchedulerWrapper.java | 19 +- .../sls/scheduler/SLSCapacityScheduler.java | 77 +- hadoop-tools/hadoop-streaming/pom.xml | 5 +- .../java/org/apache/hadoop/record/package.html | 74 +- .../org/apache/hadoop/typedbytes/package.html | 13 +- hadoop-tools/hadoop-tools-dist/pom.xml | 11 +- hadoop-tools/pom.xml | 5 +- .../hadoop-yarn/bin/start-yarn.sh | 25 +- .../hadoop-yarn/bin/stop-yarn.sh | 31 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 11 + .../dev-support/findbugs-exclude.xml | 4 - .../jdiff/Apache_Hadoop_YARN_API_2.6.0.xml | 2 +- .../jdiff/Apache_Hadoop_YARN_API_2.7.2.xml | 2 +- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 9 +- .../hadoop/yarn/api/ApplicationConstants.java | 7 - .../api/protocolrecords/AllocateRequest.java | 138 +- .../api/protocolrecords/AllocateResponse.java | 283 +- .../GetClusterNodeLabelsResponse.java | 50 +- .../GetLabelsToNodesResponse.java | 6 +- .../GetNodesToLabelsResponse.java | 7 +- .../yarn/api/records/ApplicationReport.java | 10 + .../yarn/api/records/ApplicationTimeout.java | 109 + .../yarn/api/records/ContainerExitStatus.java | 6 + .../api/records/ContainerLaunchContext.java | 16 + .../hadoop/yarn/api/records/ContainerState.java | 6 +- .../yarn/api/records/ContainerStatus.java | 40 +- .../yarn/api/records/ContainerUpdateType.java | 9 +- .../hadoop/yarn/api/records/NodeReport.java | 12 +- .../hadoop/yarn/api/records/QueueState.java | 11 +- .../yarn/api/records/ResourceRequest.java | 209 +- .../yarn/api/records/UpdateContainerError.java | 25 +- .../api/records/UpdateContainerRequest.java | 45 +- .../records/timelineservice/FlowRunEntity.java | 4 +- .../records/timelineservice/TimelineEntity.java | 3 +- .../org/apache/hadoop/yarn/conf/HAUtil.java | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 193 +- .../AddToClusterNodeLabelsRequest.java | 11 +- ..._server_resourcemanager_service_protos.proto | 5 +- .../src/main/proto/yarn_protos.proto | 23 +- .../src/main/proto/yarn_service_protos.proto | 11 +- .../yarn/conf/TestYarnConfigurationFields.java | 24 - .../pom.xml | 5 +- .../distributedshell/ApplicationMaster.java | 100 +- .../pom.xml | 5 +- .../UnmanagedAMLauncher.java | 10 +- .../hadoop-yarn-applications/pom.xml | 5 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 5 +- .../hadoop/yarn/client/api/AMRMClient.java | 180 +- .../yarn/client/api/SharedCacheClient.java | 23 +- .../hadoop/yarn/client/api/YarnClient.java | 43 +- .../yarn/client/api/async/AMRMClientAsync.java | 52 +- .../api/async/impl/AMRMClientAsyncImpl.java | 45 +- .../yarn/client/api/impl/AMRMClientImpl.java | 140 +- .../client/api/impl/SharedCacheClientImpl.java | 31 +- .../yarn/client/api/impl/YarnClientImpl.java | 84 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 62 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 106 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 44 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 15 +- .../hadoop/yarn/client/TestGetGroups.java | 36 +- ...stHedgingRequestRMFailoverProxyProvider.java | 57 +- .../hadoop/yarn/client/TestRMFailover.java | 100 +- ...nagerAdministrationProtocolPBClientImpl.java | 33 +- .../api/async/impl/TestAMRMClientAsync.java | 2 +- .../yarn/client/api/impl/TestAMRMClient.java | 62 +- .../api/impl/TestAMRMClientOnRMRestart.java | 19 +- .../yarn/client/api/impl/TestAMRMProxy.java | 48 +- .../api/impl/TestDistributedScheduling.java | 4 +- .../yarn/client/api/impl/TestNMClient.java | 9 +- .../TestOpportunisticContainerAllocation.java | 402 +- .../api/impl/TestSharedCacheClientImpl.java | 37 +- .../yarn/client/api/impl/TestYarnClient.java | 205 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 15 + .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 47 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 118 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 38 +- .../FileSystemBasedConfigurationProvider.java | 4 +- .../impl/pb/AllocateResponsePBImpl.java | 8 +- .../pb/GetClusterNodeLabelsResponsePBImpl.java | 45 +- .../impl/pb/GetLabelsToNodesResponsePBImpl.java | 20 +- .../impl/pb/GetNodesToLabelsResponsePBImpl.java | 52 +- .../impl/pb/ApplicationReportPBImpl.java | 96 + .../impl/pb/ApplicationTimeoutPBImpl.java | 130 + .../impl/pb/ContainerLaunchContextPBImpl.java | 26 + .../impl/pb/UpdateContainerErrorPBImpl.java | 16 + .../RequestHedgingRMFailoverProxyProvider.java | 6 +- .../hadoop/yarn/client/api/TimelineClient.java | 94 +- .../yarn/client/api/TimelineV2Client.java | 92 + .../api/impl/FileSystemTimelineWriter.java | 44 +- .../client/api/impl/TimelineClientImpl.java | 827 +- .../yarn/client/api/impl/TimelineConnector.java | 440 + .../client/api/impl/TimelineV2ClientImpl.java | 459 + .../hadoop/yarn/event/AsyncDispatcher.java | 26 +- .../apache/hadoop/yarn/event/Dispatcher.java | 2 +- .../logaggregation/AggregatedLogFormat.java | 2 +- .../ContainerLogAggregationType.java | 31 + .../yarn/logaggregation/ContainerLogMeta.java | 61 + .../logaggregation/LogAggregationUtils.java | 64 +- .../yarn/logaggregation/LogCLIHelpers.java | 119 +- .../yarn/logaggregation/LogToolUtils.java | 276 + .../logaggregation/PerContainerLogFileInfo.java | 93 + .../nodelabels/CommonNodeLabelsManager.java | 2 +- .../nodelabels/FileSystemNodeLabelsStore.java | 26 +- .../NonAppendableFSNodeLabelStore.java | 8 +- .../apache/hadoop/yarn/security/AccessType.java | 2 + .../yarn/security/ContainerTokenIdentifier.java | 25 + .../security/YarnAuthorizationProvider.java | 15 + .../pb/AddToClusterNodeLabelsRequestPBImpl.java | 47 +- .../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +- .../org/apache/hadoop/yarn/state/Graph.java | 2 +- .../state/InvalidStateTransitionException.java | 22 +- .../state/InvalidStateTransitonException.java | 19 +- .../yarn/state/VisualizeStateMachine.java | 4 +- .../org/apache/hadoop/yarn/util/FSDownload.java | 2 +- .../yarn/util/ProcfsBasedProcessTree.java | 5 +- .../yarn/util/ResourceCalculatorPlugin.java | 3 + .../java/org/apache/hadoop/yarn/util/Times.java | 3 + .../resource/DefaultResourceCalculator.java | 22 +- .../resource/DominantResourceCalculator.java | 38 +- .../yarn/util/resource/ResourceCalculator.java | 49 +- .../hadoop/yarn/util/resource/Resources.java | 28 + .../yarn/util/timeline/TimelineUtils.java | 6 +- .../apache/hadoop/yarn/webapp/Controller.java | 2 +- .../org/apache/hadoop/yarn/webapp/Router.java | 23 +- .../org/apache/hadoop/yarn/webapp/WebApp.java | 13 + .../org/apache/hadoop/yarn/webapp/WebApps.java | 38 +- .../webapp/YarnJacksonJaxbJsonProvider.java | 16 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 31 +- .../yarn/webapp/util/YarnWebServiceUtils.java | 62 + .../main/resources/webapps/sharedcache/.keep | 0 .../src/main/resources/yarn-default.xml | 202 +- .../main/resources/yarn-version-info.properties | 2 +- .../hadoop/yarn/api/BasePBImplRecordsTest.java | 2 +- .../hadoop/yarn/api/TestPBImplRecords.java | 2 + .../client/api/impl/TestTimelineClient.java | 39 +- .../api/impl/TestTimelineClientV2Impl.java | 4 +- .../hadoop/yarn/event/DrainDispatcher.java | 30 +- .../hadoop/yarn/event/InlineDispatcher.java | 2 +- .../hadoop/yarn/event/TestAsyncDispatcher.java | 42 + .../logaggregation/TestContainerLogsUtils.java | 122 + .../nodelabels/TestCommonNodeLabelsManager.java | 2 +- .../TestFileSystemNodeLabelsStore.java | 4 +- .../util/resource/TestResourceCalculator.java | 79 + .../apache/hadoop/yarn/webapp/TestWebApp.java | 77 +- .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 16 +- .../registry/client/binding/JsonSerDeser.java | 32 +- .../client/impl/zk/RegistrySecurity.java | 13 +- .../hadoop/registry/client/types/Endpoint.java | 4 +- .../client/types/RegistryPathStatus.java | 4 +- .../registry/client/types/ServiceRecord.java | 7 +- .../registry/secure/TestSecureRegistry.java | 22 + .../pom.xml | 5 +- ...pplicationHistoryManagerOnTimelineStore.java | 2 +- .../FileSystemApplicationHistoryStore.java | 16 +- .../webapp/AHSWebServices.java | 462 +- .../server/timeline/GenericObjectMapper.java | 6 +- .../server/timeline/LeveldbTimelineStore.java | 33 +- .../timeline/webapp/TimelineWebServices.java | 4 + .../TestFileSystemApplicationHistoryStore.java | 17 +- .../webapp/TestAHSWebServices.java | 377 +- .../timeline/TestLeveldbTimelineStore.java | 39 + .../hadoop-yarn-server-common/pom.xml | 5 +- .../server/api/protocolrecords/RemoteNode.java | 7 + .../OpportunisticContainerAllocator.java | 77 +- .../OpportunisticContainerContext.java | 40 +- .../server/scheduler/SchedulerRequestKey.java | 172 + .../hadoop/yarn/server/utils/BuilderUtils.java | 32 + .../hadoop/yarn/server/webapp/AppBlock.java | 144 +- .../server/webapp/YarnWebServiceParams.java | 37 + .../server/webapp/dao/ContainerLogsInfo.java | 87 + .../hadoop-yarn-server-nodemanager/pom.xml | 5 +- .../src/CMakeLists.txt | 16 + .../server/nodemanager/ContainerExecutor.java | 20 +- .../hadoop/yarn/server/nodemanager/Context.java | 18 +- .../nodemanager/LinuxContainerExecutor.java | 23 + .../yarn/server/nodemanager/NodeManager.java | 46 +- .../nodemanager/NodeResourceMonitorImpl.java | 5 + .../nodemanager/NodeStatusUpdaterImpl.java | 54 - .../WindowsSecureContainerExecutor.java | 4 +- .../AMRMProxyApplicationContextImpl.java | 2 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 7 +- .../containermanager/AuxServices.java | 4 +- .../containermanager/ContainerManager.java | 4 + .../containermanager/ContainerManagerImpl.java | 62 +- .../application/ApplicationImpl.java | 2 +- .../containermanager/container/Container.java | 6 + .../container/ContainerImpl.java | 88 +- .../container/ContainerState.java | 2 +- .../launcher/ContainerLaunch.java | 64 +- .../launcher/RecoveredContainerLaunch.java | 4 +- .../CGroupsBlkioResourceHandlerImpl.java | 2 +- .../CGroupsCpuResourceHandlerImpl.java | 2 +- .../linux/resources/CGroupsHandler.java | 58 +- .../linux/resources/CGroupsHandlerImpl.java | 171 +- .../CGroupsMemoryResourceHandlerImpl.java | 2 +- .../TrafficControlBandwidthHandlerImpl.java | 3 +- .../linux/resources/TrafficController.java | 13 +- .../DelegatingLinuxContainerRuntime.java | 30 +- .../JavaSandboxLinuxContainerRuntime.java | 495 + .../runtime/LinuxContainerRuntimeConstants.java | 2 + .../localizer/ContainerLocalizer.java | 53 +- .../localizer/LocalizedResource.java | 2 +- .../localizer/ResourceLocalizationService.java | 45 +- .../sharedcache/SharedCacheUploader.java | 6 +- .../logaggregation/AppLogAggregatorImpl.java | 6 +- .../monitor/ContainersMonitor.java | 52 +- .../monitor/ContainersMonitorImpl.java | 118 +- .../queuing/QueuingContainerManagerImpl.java | 686 - .../containermanager/queuing/package-info.java | 23 - ...locationBasedResourceUtilizationTracker.java | 137 + .../scheduler/ContainerScheduler.java | 419 + .../scheduler/ContainerSchedulerEvent.java | 51 + .../scheduler/ContainerSchedulerEventType.java | 29 + .../scheduler/ResourceUtilizationTracker.java | 59 + .../scheduler/package-info.java | 22 + .../executor/ContainerPrepareContext.java | 119 + .../nodemanager/metrics/NodeManagerMetrics.java | 38 + .../recovery/NMLeveldbStateStoreService.java | 19 +- .../recovery/NMStateStoreService.java | 17 + .../scheduler/DistributedScheduler.java | 11 +- .../timelineservice/NMTimelinePublisher.java | 24 +- .../util/NodeManagerHardwareUtils.java | 160 +- .../nodemanager/webapp/ContainerLogsUtils.java | 5 +- .../nodemanager/webapp/NMWebAppFilter.java | 8 + .../nodemanager/webapp/NMWebServices.java | 220 +- .../server/nodemanager/webapp/WebServer.java | 35 +- .../webapp/dao/ContainerLogsInfo.java | 112 - .../webapp/dao/NMContainerLogsInfo.java | 79 + .../container-executor/impl/configuration.c | 1 + .../impl/container-executor.c | 23 +- .../impl/container-executor.h | 10 - .../test/test-container-executor.c | 41 - .../src/main/resources/java.policy | 63 + .../TestContainerManagerWithLCE.java | 11 + .../yarn/server/nodemanager/TestEventFlow.java | 3 +- .../nodemanager/TestNodeHealthService.java | 2 +- .../nodemanager/TestNodeManagerResync.java | 8 + .../nodemanager/TestNodeManagerShutdown.java | 5 +- .../nodemanager/TestNodeStatusUpdater.java | 127 +- .../amrmproxy/BaseAMRMProxyTest.java | 8 +- .../amrmproxy/MockResourceManagerFacade.java | 7 +- .../amrmproxy/TestAMRMProxyService.java | 13 + .../BaseContainerManagerTest.java | 34 +- .../containermanager/TestContainerManager.java | 85 +- .../TestContainerManagerRecovery.java | 34 +- .../TestContainerManagerRegression.java | 84 - .../container/TestContainer.java | 26 +- .../launcher/TestContainerLaunch.java | 73 +- .../TestCGroupsBlkioResourceHandlerImpl.java | 2 +- .../TestCGroupsCpuResourceHandlerImpl.java | 6 +- .../linux/resources/TestCGroupsHandlerImpl.java | 153 +- .../TestCGroupsMemoryResourceHandlerImpl.java | 2 +- .../TestTrafficControlBandwidthHandlerImpl.java | 2 +- .../TestJavaSandboxLinuxContainerRuntime.java | 364 + .../localizer/TestContainerLocalizer.java | 317 +- .../TestResourceLocalizationService.java | 130 +- .../TestAppLogAggregatorImpl.java | 4 +- .../TestLogAggregationService.java | 2 +- .../TestContainersMonitorResourceChange.java | 16 +- .../queuing/TestQueuingContainerManager.java | 596 - .../TestContainerSchedulerQueuing.java | 872 + .../TestNMLeveldbStateStoreService.java | 69 + .../TestNMTimelinePublisher.java | 10 +- .../nodemanager/webapp/MockContainer.java | 15 + .../nodemanager/webapp/TestNMWebFilter.java | 176 + .../nodemanager/webapp/TestNMWebServices.java | 215 +- .../conf/capacity-scheduler.xml | 9 + .../hadoop-yarn-server-resourcemanager/pom.xml | 21 +- ...ActiveStandbyElectorBasedElectorService.java | 271 + .../server/resourcemanager/AdminService.java | 92 +- .../ApplicationMasterService.java | 444 +- .../server/resourcemanager/ClientRMService.java | 120 +- .../CuratorBasedElectorService.java | 139 + .../DecommissioningNodesWatcher.java | 4 +- .../server/resourcemanager/EmbeddedElector.java | 41 + .../resourcemanager/EmbeddedElectorService.java | 260 - .../resourcemanager/LeaderElectorService.java | 129 - .../resourcemanager/NMLivelinessMonitor.java | 3 +- ...pportunisticContainerAllocatorAMService.java | 150 +- .../server/resourcemanager/RMAppManager.java | 278 +- .../resourcemanager/RMAppManagerEvent.java | 11 + .../resourcemanager/RMAppManagerEventType.java | 3 +- .../server/resourcemanager/RMAuditLogger.java | 1 + .../yarn/server/resourcemanager/RMContext.java | 8 +- .../server/resourcemanager/RMContextImpl.java | 35 +- ...MCriticalThreadUncaughtExceptionHandler.java | 58 + .../resourcemanager/RMFatalEventType.java | 5 +- .../server/resourcemanager/RMServerUtils.java | 169 +- .../server/resourcemanager/ResourceManager.java | 118 +- .../ahs/RMApplicationHistoryWriter.java | 4 +- .../resourcemanager/amlauncher/AMLauncher.java | 13 - .../metrics/AbstractSystemMetricsPublisher.java | 4 +- .../metrics/TimelineServiceV1Publisher.java | 4 +- .../metrics/TimelineServiceV2Publisher.java | 4 +- .../AbstractPreemptableResourceCalculator.java | 14 +- .../capacity/FifoCandidatesSelector.java | 5 +- .../FifoIntraQueuePreemptionPlugin.java | 6 +- .../capacity/IntraQueueCandidatesSelector.java | 5 +- .../capacity/PreemptionCandidatesSelector.java | 3 +- .../ProportionalCapacityPreemptionPolicy.java | 51 +- ...QueuePriorityContainerCandidateSelector.java | 510 + .../monitor/capacity/TempQueuePerPartition.java | 53 +- .../monitor/capacity/TempSchedulerNode.java | 120 + .../nodelabels/RMNodeLabelsManager.java | 16 + .../recovery/FileSystemRMStateStore.java | 2 +- .../recovery/MemoryRMStateStore.java | 2 +- .../resourcemanager/recovery/RMStateStore.java | 42 +- .../recovery/ZKRMStateStore.java | 30 +- .../reservation/AbstractReservationSystem.java | 8 +- .../reservation/InMemoryPlan.java | 4 +- .../reservation/ReservationInputValidator.java | 2 +- .../planning/AlignedPlannerWithGreedy.java | 16 +- .../planning/GreedyReservationAgent.java | 19 +- .../reservation/planning/PlanningAlgorithm.java | 4 + .../reservation/planning/ReservationAgent.java | 8 + .../planning/TryManyReservationAgents.java | 5 +- .../server/resourcemanager/rmapp/RMApp.java | 14 + .../resourcemanager/rmapp/RMAppEventType.java | 1 - .../server/resourcemanager/rmapp/RMAppImpl.java | 192 +- .../resourcemanager/rmapp/RMAppMoveEvent.java | 44 - .../rmapp/attempt/AMLivelinessMonitor.java | 3 +- .../rmapp/attempt/RMAppAttemptImpl.java | 255 +- .../rmapp/monitor/RMAppLifetimeMonitor.java | 4 +- .../rmcontainer/RMContainer.java | 7 +- .../RMContainerChangeResourceEvent.java | 44 - .../rmcontainer/RMContainerImpl.java | 94 +- .../rmcontainer/RMContainerReservedEvent.java | 2 +- .../resourcemanager/rmnode/RMNodeImpl.java | 34 +- .../scheduler/AbstractUsersManager.java | 54 + .../scheduler/AbstractYarnScheduler.java | 315 +- .../scheduler/ActiveUsersManager.java | 23 +- .../resourcemanager/scheduler/Allocation.java | 26 +- .../scheduler/AppSchedulingInfo.java | 802 +- .../scheduler/ContainerUpdateContext.java | 374 + .../scheduler/ContainerUpdates.java | 68 + .../server/resourcemanager/scheduler/Queue.java | 2 +- .../scheduler/QueueStateManager.java | 100 + .../scheduler/SchedulerApplicationAttempt.java | 337 +- .../scheduler/SchedulerNode.java | 163 +- .../scheduler/SchedulerQueue.java | 69 + .../scheduler/SchedulerQueueManager.java | 76 + .../scheduler/SchedulerRequestKey.java | 122 - .../scheduler/SchedulerUtils.java | 91 +- .../scheduler/YarnScheduler.java | 44 +- .../scheduler/capacity/AbstractCSQueue.java | 190 +- .../AppPriorityACLConfigurationParser.java | 219 + .../scheduler/capacity/AppPriorityACLGroup.java | 108 + .../scheduler/capacity/CSQueue.java | 51 +- .../capacity/CapacityHeadroomProvider.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 682 +- .../CapacitySchedulerConfiguration.java | 302 +- .../capacity/CapacitySchedulerContext.java | 13 +- .../capacity/CapacitySchedulerQueueManager.java | 407 + .../scheduler/capacity/LeafQueue.java | 836 +- .../scheduler/capacity/ParentQueue.java | 183 +- .../capacity/PartitionedQueueComparator.java | 72 - .../scheduler/capacity/UsersManager.java | 982 + .../allocator/AbstractContainerAllocator.java | 4 + .../capacity/allocator/ContainerAllocator.java | 31 +- .../allocator/IncreaseContainerAllocator.java | 337 - .../allocator/RegularContainerAllocator.java | 138 +- .../PriorityUtilizationQueueOrderingPolicy.java | 192 + .../capacity/policy/QueueOrderingPolicy.java | 52 + .../common/ContainerAllocationProposal.java | 9 +- .../scheduler/common/PendingAsk.java | 57 + .../scheduler/common/SchedulerContainer.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 427 +- .../common/fica/FiCaSchedulerNode.java | 20 +- .../distributed/NodeQueueLoadMonitor.java | 24 +- .../scheduler/event/AppAddedSchedulerEvent.java | 5 +- .../scheduler/fair/AllocationConfiguration.java | 49 +- .../fair/AllocationFileLoaderService.java | 92 +- .../scheduler/fair/FSAppAttempt.java | 536 +- .../scheduler/fair/FSContext.java | 65 + .../scheduler/fair/FSLeafQueue.java | 391 +- .../scheduler/fair/FSOpDurations.java | 8 - .../scheduler/fair/FSParentQueue.java | 80 +- .../scheduler/fair/FSPreemptionThread.java | 276 + .../resourcemanager/scheduler/fair/FSQueue.java | 136 +- .../scheduler/fair/FSQueueMetrics.java | 92 +- .../scheduler/fair/FSSchedulerNode.java | 38 +- .../scheduler/fair/FSStarvedApps.java | 85 + .../scheduler/fair/FairScheduler.java | 499 +- .../fair/FairSchedulerConfiguration.java | 23 +- .../scheduler/fair/QueueManager.java | 43 +- .../scheduler/fair/Schedulable.java | 37 +- .../scheduler/fair/SchedulingPolicy.java | 55 +- .../fair/VisitedResourceRequestTracker.java | 146 + .../DominantResourceFairnessPolicy.java | 29 +- .../fair/policies/FairSharePolicy.java | 37 +- .../scheduler/fair/policies/FifoPolicy.java | 14 +- .../scheduler/fifo/FifoAppAttempt.java | 18 +- .../scheduler/fifo/FifoScheduler.java | 155 +- .../LocalitySchedulingPlacementSet.java | 414 + .../placement/SchedulingPlacementSet.java | 94 +- .../security/AppPriorityACLsManager.java | 230 + .../security/DelegationTokenRenewer.java | 74 +- .../security/NMTokenSecretManagerInRM.java | 4 +- .../security/QueueACLsManager.java | 69 +- .../webapp/CapacitySchedulerPage.java | 2 +- .../webapp/FairSchedulerPage.java | 28 + .../webapp/JAXBContextResolver.java | 3 +- .../resourcemanager/webapp/NodesPage.java | 6 +- .../server/resourcemanager/webapp/RMWebApp.java | 3 +- .../resourcemanager/webapp/RMWebServices.java | 192 +- .../resourcemanager/webapp/dao/AppInfo.java | 37 +- .../webapp/dao/AppTimeoutInfo.java | 71 + .../webapp/dao/AppTimeoutsInfo.java | 47 + .../dao/CapacitySchedulerLeafQueueInfo.java | 2 +- .../resourcemanager/webapp/dao/ClusterInfo.java | 2 +- .../resourcemanager/webapp/dao/NodeInfo.java | 10 +- .../server/resourcemanager/ACLsTestBase.java | 5 +- .../server/resourcemanager/Application.java | 6 +- .../yarn/server/resourcemanager/MockAM.java | 9 +- .../yarn/server/resourcemanager/MockNM.java | 6 + .../yarn/server/resourcemanager/MockRM.java | 174 +- .../ParameterizedSchedulerTestBase.java | 1 + .../server/resourcemanager/RMHATestBase.java | 4 +- .../yarn/server/resourcemanager/Task.java | 2 +- .../server/resourcemanager/TestAppManager.java | 42 +- .../resourcemanager/TestApplicationCleanup.java | 15 - .../TestApplicationMasterLauncher.java | 5 - .../TestApplicationMasterService.java | 9 +- .../resourcemanager/TestClientRMService.java | 335 +- .../TestContainerResourceUsage.java | 8 +- .../TestLeaderElectorService.java | 17 +- .../resourcemanager/TestMoveApplication.java | 12 +- .../TestNodeBlacklistingOnAMFailures.java | 14 - ...pportunisticContainerAllocatorAMService.java | 550 +- .../yarn/server/resourcemanager/TestRM.java | 15 +- .../resourcemanager/TestRMAdminService.java | 56 +- .../resourcemanager/TestRMEmbeddedElector.java | 49 +- .../yarn/server/resourcemanager/TestRMHA.java | 48 +- .../resourcemanager/TestRMNodeTransitions.java | 2 +- .../server/resourcemanager/TestRMRestart.java | 125 +- .../resourcemanager/TestRMStoreCommands.java | 103 + .../TestResourceTrackerService.java | 4 + .../TestTokenClientRMService.java | 5 +- .../applicationsmanager/MockAsm.java | 11 + .../TestAMRMRPCNodeUpdates.java | 13 - .../applicationsmanager/TestAMRestart.java | 84 +- .../TestRMAppLogAggregationStatus.java | 8 +- .../metrics/TestSystemMetricsPublisher.java | 2 + .../TestSystemMetricsPublisherForV2.java | 1 + ...alCapacityPreemptionPolicyMockFramework.java | 99 +- .../TestPreemptionForQueueWithPriorities.java | 361 + ...estProportionalCapacityPreemptionPolicy.java | 102 +- ...pacityPreemptionPolicyForNodePartitions.java | 11 +- ...alCapacityPreemptionPolicyMockFramework.java | 13 +- .../recovery/RMStateStoreTestBase.java | 2 +- .../recovery/TestZKRMStateStore.java | 42 +- .../TestReservationInputValidator.java | 16 +- .../planning/TestAlignedPlanner.java | 3 + .../planning/TestGreedyReservationAgent.java | 4 +- .../server/resourcemanager/rmapp/MockRMApp.java | 13 +- .../rmapp/TestApplicationLifetimeMonitor.java | 27 +- .../rmapp/TestNodesListManager.java | 3 +- .../rmapp/TestRMAppTransitions.java | 4 +- .../rmapp/attempt/TestBoundedAppender.java | 116 + .../TestRMAppAttemptImplDiagnostics.java | 111 + .../attempt/TestRMAppAttemptTransitions.java | 38 +- .../rmcontainer/TestRMContainerImpl.java | 7 +- .../scheduler/TestAbstractYarnScheduler.java | 131 +- .../scheduler/TestAppSchedulingInfo.java | 5 +- .../TestSchedulerApplicationAttempt.java | 29 +- .../scheduler/TestSchedulerUtils.java | 30 +- .../CapacitySchedulerPreemptionTestBase.java | 22 +- .../capacity/TestApplicationLimits.java | 47 +- .../TestApplicationLimitsByPartition.java | 35 +- .../capacity/TestApplicationPriority.java | 28 +- ...TestApplicationPriorityACLConfiguration.java | 120 + .../capacity/TestApplicationPriorityACLs.java | 206 + .../capacity/TestCapacityScheduler.java | 469 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 78 +- ...TestCapacitySchedulerSurgicalPreemption.java | 572 +- .../scheduler/capacity/TestChildQueueOrder.java | 24 +- .../capacity/TestContainerAllocation.java | 246 + .../capacity/TestContainerResizing.java | 134 +- .../capacity/TestIncreaseAllocationExpirer.java | 16 +- .../scheduler/capacity/TestLeafQueue.java | 231 +- .../TestNodeLabelContainerAllocation.java | 72 +- .../scheduler/capacity/TestParentQueue.java | 46 +- .../scheduler/capacity/TestQueueParsing.java | 58 + .../scheduler/capacity/TestQueueState.java | 200 + .../capacity/TestQueueStateManager.java | 162 + .../scheduler/capacity/TestReservations.java | 41 +- .../scheduler/capacity/TestUtils.java | 7 +- ...tPriorityUtilizationQueueOrderingPolicy.java | 222 + .../scheduler/fair/FairSchedulerTestBase.java | 65 +- .../fair/FairSchedulerWithMockPreemption.java | 61 + .../scheduler/fair/FakeSchedulable.java | 10 +- .../fair/TestAllocationFileLoaderService.java | 88 + .../fair/TestContinuousScheduling.java | 13 +- .../scheduler/fair/TestFSAppAttempt.java | 2 +- .../scheduler/fair/TestFSAppStarvation.java | 284 + .../scheduler/fair/TestFSLeafQueue.java | 166 +- .../scheduler/fair/TestFSQueueMetrics.java | 69 + .../scheduler/fair/TestFairScheduler.java | 478 +- .../fair/TestFairSchedulerPreemption.java | 1623 +- .../fair/TestQueueManagerRealScheduler.java | 128 + .../scheduler/fair/TestSchedulingPolicy.java | 312 +- .../fair/TestVisitedResourceRequestTracker.java | 112 + .../TestDominantResourceFairnessPolicy.java | 25 +- .../scheduler/fifo/TestFifoScheduler.java | 48 +- .../policy/TestFairOrderingPolicy.java | 11 +- .../security/TestDelegationTokenRenewer.java | 127 +- .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServiceAppsNodelabel.java | 2 +- .../webapp/TestRMWebServicesApps.java | 2 +- .../TestRMWebServicesAppsModification.java | 154 +- .../webapp/TestRMWebServicesNodes.java | 11 +- .../pom.xml | 5 +- .../store/InMemorySCMStore.java | 8 +- .../hadoop-yarn-server-tests/pom.xml | 5 +- .../hadoop/yarn/server/MiniYARNCluster.java | 72 +- .../server/TestContainerManagerSecurity.java | 2 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 80 +- .../TestTimelineServiceClientIntegration.java | 10 +- .../pom.xml | 17 +- .../yarn/server/timeline/EntityCacheItem.java | 9 +- .../timeline/EntityGroupFSTimelineStore.java | 12 +- .../hadoop/yarn/server/timeline/LogInfo.java | 12 +- .../server/timeline/PluginStoreTestUtils.java | 22 +- .../yarn/server/timeline/TestLogInfo.java | 12 +- .../pom.xml | 155 +- ...stTimelineReaderWebServicesHBaseStorage.java | 8 +- .../storage/DataGeneratorForTest.java | 4 +- .../storage/TestHBaseTimelineStorageApps.java | 8 +- .../TestHBaseTimelineStorageEntities.java | 6 +- ...TestPhoenixOfflineAggregationWriterImpl.java | 161 - .../flow/TestHBaseStorageFlowActivity.java | 18 +- .../storage/flow/TestHBaseStorageFlowRun.java | 28 +- .../flow/TestHBaseStorageFlowRunCompaction.java | 46 +- .../pom.xml | 191 + .../reader/filter/TimelineFilterUtils.java | 290 + .../reader/filter/package-info.java | 28 + .../storage/HBaseTimelineReaderImpl.java | 88 + .../storage/HBaseTimelineWriterImpl.java | 566 + .../storage/TimelineSchemaCreator.java | 250 + .../storage/application/ApplicationColumn.java | 156 + .../application/ApplicationColumnFamily.java | 65 + .../application/ApplicationColumnPrefix.java | 288 + .../storage/application/ApplicationRowKey.java | 206 + .../application/ApplicationRowKeyPrefix.java | 69 + .../storage/application/ApplicationTable.java | 161 + .../storage/application/package-info.java | 28 + .../storage/apptoflow/AppToFlowColumn.java | 148 + .../apptoflow/AppToFlowColumnFamily.java | 51 + .../storage/apptoflow/AppToFlowRowKey.java | 143 + .../storage/apptoflow/AppToFlowTable.java | 113 + .../storage/apptoflow/package-info.java | 28 + .../storage/common/AppIdKeyConverter.java | 96 + .../storage/common/BaseTable.java | 140 + .../common/BufferedMutatorDelegator.java | 73 + .../timelineservice/storage/common/Column.java | 80 + .../storage/common/ColumnFamily.java | 34 + .../storage/common/ColumnHelper.java | 388 + .../storage/common/ColumnPrefix.java | 145 + .../storage/common/EventColumnName.java | 63 + .../common/EventColumnNameConverter.java | 99 + .../storage/common/GenericConverter.java | 48 + .../common/HBaseTimelineStorageUtils.java | 243 + .../storage/common/KeyConverter.java | 41 + .../storage/common/LongConverter.java | 94 + .../storage/common/LongKeyConverter.java | 68 + .../storage/common/NumericValueConverter.java | 39 + .../timelineservice/storage/common/Range.java | 62 + .../storage/common/RowKeyPrefix.java | 42 + .../storage/common/Separator.java | 575 + .../storage/common/StringKeyConverter.java | 54 + .../common/TimelineHBaseSchemaConstants.java | 71 + .../storage/common/TimestampGenerator.java | 116 + .../storage/common/TypedBufferedMutator.java | 28 + .../storage/common/ValueConverter.java | 47 + .../storage/common/package-info.java | 28 + .../storage/entity/EntityColumn.java | 160 + .../storage/entity/EntityColumnFamily.java | 65 + .../storage/entity/EntityColumnPrefix.java | 300 + .../storage/entity/EntityRowKey.java | 225 + .../storage/entity/EntityRowKeyPrefix.java | 74 + .../storage/entity/EntityTable.java | 161 + .../storage/entity/package-info.java | 28 + .../flow/AggregationCompactionDimension.java | 63 + .../storage/flow/AggregationOperation.java | 94 + .../timelineservice/storage/flow/Attribute.java | 39 + .../storage/flow/FlowActivityColumnFamily.java | 55 + .../storage/flow/FlowActivityColumnPrefix.java | 277 + .../storage/flow/FlowActivityRowKey.java | 196 + .../storage/flow/FlowActivityRowKeyPrefix.java | 60 + .../storage/flow/FlowActivityTable.java | 108 + .../storage/flow/FlowRunColumn.java | 182 + .../storage/flow/FlowRunColumnFamily.java | 54 + .../storage/flow/FlowRunColumnPrefix.java | 268 + .../storage/flow/FlowRunCoprocessor.java | 304 + .../storage/flow/FlowRunRowKey.java | 190 + .../storage/flow/FlowRunRowKeyPrefix.java | 54 + .../storage/flow/FlowRunTable.java | 141 + .../storage/flow/FlowScanner.java | 728 + .../storage/flow/FlowScannerOperation.java | 46 + .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../storage/reader/ApplicationEntityReader.java | 481 + .../reader/FlowActivityEntityReader.java | 163 + .../storage/reader/FlowRunEntityReader.java | 269 + .../storage/reader/GenericEntityReader.java | 648 + .../storage/reader/TimelineEntityReader.java | 496 + .../reader/TimelineEntityReaderFactory.java | 89 + .../storage/reader/package-info.java | 28 + .../storage/common/TestKeyConverters.java | 130 + .../storage/common/TestRowKeys.java | 246 + .../storage/common/TestSeparator.java | 215 + .../hadoop-yarn-server-timelineservice/pom.xml | 103 +- .../collector/TimelineCollectorManager.java | 26 +- .../reader/TimelineReaderServer.java | 39 +- .../reader/filter/TimelineFilterUtils.java | 290 - .../storage/FileSystemTimelineReaderImpl.java | 6 +- .../storage/HBaseTimelineReaderImpl.java | 88 - .../storage/HBaseTimelineWriterImpl.java | 571 - .../PhoenixOfflineAggregationWriterImpl.java | 358 - .../storage/TimelineSchemaCreator.java | 272 - .../storage/application/ApplicationColumn.java | 156 - .../application/ApplicationColumnFamily.java | 65 - .../application/ApplicationColumnPrefix.java | 288 - .../storage/application/ApplicationRowKey.java | 206 - .../application/ApplicationRowKeyPrefix.java | 69 - .../storage/application/ApplicationTable.java | 161 - .../storage/application/package-info.java | 28 - .../storage/apptoflow/AppToFlowColumn.java | 148 - .../apptoflow/AppToFlowColumnFamily.java | 51 - .../storage/apptoflow/AppToFlowRowKey.java | 143 - .../storage/apptoflow/AppToFlowTable.java | 113 - .../storage/apptoflow/package-info.java | 28 - .../storage/common/AppIdKeyConverter.java | 95 - .../storage/common/BaseTable.java | 140 - .../common/BufferedMutatorDelegator.java | 73 - .../timelineservice/storage/common/Column.java | 80 - .../storage/common/ColumnFamily.java | 34 - .../storage/common/ColumnHelper.java | 388 - .../storage/common/ColumnPrefix.java | 145 - .../storage/common/EventColumnName.java | 63 - .../common/EventColumnNameConverter.java | 99 - .../storage/common/GenericConverter.java | 48 - .../storage/common/KeyConverter.java | 41 - .../storage/common/LongConverter.java | 94 - .../storage/common/LongKeyConverter.java | 68 - .../storage/common/NumericValueConverter.java | 39 - .../timelineservice/storage/common/Range.java | 62 - .../storage/common/RowKeyPrefix.java | 42 - .../storage/common/Separator.java | 575 - .../storage/common/StringKeyConverter.java | 54 - .../common/TimelineHBaseSchemaConstants.java | 71 - .../storage/common/TimelineStorageUtils.java | 207 - .../storage/common/TimestampGenerator.java | 116 - .../storage/common/TypedBufferedMutator.java | 28 - .../storage/common/ValueConverter.java | 47 - .../storage/common/package-info.java | 28 - .../storage/entity/EntityColumn.java | 160 - .../storage/entity/EntityColumnFamily.java | 65 - .../storage/entity/EntityColumnPrefix.java | 300 - .../storage/entity/EntityRowKey.java | 225 - .../storage/entity/EntityRowKeyPrefix.java | 74 - .../storage/entity/EntityTable.java | 161 - .../storage/entity/package-info.java | 28 - .../flow/AggregationCompactionDimension.java | 63 - .../storage/flow/AggregationOperation.java | 94 - .../timelineservice/storage/flow/Attribute.java | 39 - .../storage/flow/FlowActivityColumnFamily.java | 55 - .../storage/flow/FlowActivityColumnPrefix.java | 277 - .../storage/flow/FlowActivityRowKey.java | 196 - .../storage/flow/FlowActivityRowKeyPrefix.java | 60 - .../storage/flow/FlowActivityTable.java | 108 - .../storage/flow/FlowRunColumn.java | 182 - .../storage/flow/FlowRunColumnFamily.java | 54 - .../storage/flow/FlowRunColumnPrefix.java | 268 - .../storage/flow/FlowRunCoprocessor.java | 304 - .../storage/flow/FlowRunRowKey.java | 190 - .../storage/flow/FlowRunRowKeyPrefix.java | 54 - .../storage/flow/FlowRunTable.java | 141 - .../storage/flow/FlowScanner.java | 727 - .../storage/flow/FlowScannerOperation.java | 46 - .../storage/flow/package-info.java | 29 - .../storage/reader/ApplicationEntityReader.java | 481 - .../reader/FlowActivityEntityReader.java | 163 - .../storage/reader/FlowRunEntityReader.java | 269 - .../storage/reader/GenericEntityReader.java | 648 - .../storage/reader/TimelineEntityReader.java | 496 - .../reader/TimelineEntityReaderFactory.java | 89 - .../storage/reader/package-info.java | 28 - .../collector/TestTimelineCollectorManager.java | 74 + .../reader/TestTimelineReaderServer.java | 43 + .../storage/common/TestKeyConverters.java | 130 - .../storage/common/TestRowKeys.java | 246 - .../storage/common/TestSeparator.java | 215 - .../hadoop-yarn-server-web-proxy/pom.xml | 5 +- .../yarn/server/webproxy/ProxyUriUtils.java | 2 +- .../yarn/server/webproxy/WebAppProxy.java | 2 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 6 +- .../hadoop-yarn/hadoop-yarn-site/pom.xml | 5 +- .../src/site/markdown/CapacityScheduler.md | 14 +- .../src/site/markdown/DockerContainers.md | 293 + .../src/site/markdown/FairScheduler.md | 24 +- .../src/site/markdown/NodeLabel.md | 20 +- .../src/site/markdown/NodeManager.md | 8 +- .../src/site/markdown/NodeManagerCgroups.md | 5 +- .../src/site/markdown/NodeManagerRest.md | 116 +- .../site/markdown/OpportunisticContainers.md | 212 + .../src/site/markdown/ReservationSystem.md | 6 +- .../src/site/markdown/ResourceManagerHA.md | 10 +- .../src/site/markdown/ResourceManagerRest.md | 273 +- .../src/site/markdown/ResourceManagerRestart.md | 13 +- .../src/site/markdown/SecureContainer.md | 2 +- .../src/site/markdown/TimelineServer.md | 16 +- .../src/site/markdown/TimelineServiceV2.md | 158 +- .../src/site/markdown/WebApplicationProxy.md | 7 +- .../src/site/markdown/WebServicesIntro.md | 13 +- .../site/markdown/WritingYarnApplications.md | 14 +- .../site/markdown/YarnApplicationSecurity.md | 2 + .../src/site/markdown/YarnCommands.md | 35 +- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 5 +- .../hadoop-yarn-ui/src/main/webapp/.jshintrc | 5 +- .../src/main/webapp/app/adapters/abstract.js | 1 + .../main/webapp/app/adapters/cluster-info.js | 4 +- .../main/webapp/app/adapters/cluster-metric.js | 4 +- .../webapp/app/adapters/yarn-app-attempt.js | 4 +- .../src/main/webapp/app/adapters/yarn-app.js | 10 +- .../webapp/app/adapters/yarn-container-log.js | 4 +- .../main/webapp/app/adapters/yarn-container.js | 12 +- .../src/main/webapp/app/adapters/yarn-node.js | 4 +- .../src/main/webapp/app/adapters/yarn-queue.js | 4 +- .../main/webapp/app/adapters/yarn-rm-node.js | 6 +- .../webapp/app/components/app-attempt-table.js | 9 +- .../webapp/app/components/app-timeout-bar.js | 60 + .../app/components/app-usage-donut-chart.js | 7 +- .../src/main/webapp/app/components/bar-chart.js | 36 +- .../app/components/base-chart-component.js | 8 +- .../app/components/base-usage-donut-chart.js | 11 +- .../main/webapp/app/components/donut-chart.js | 11 +- .../main/webapp/app/components/nodes-heatmap.js | 21 +- ...er-app-memusage-by-nodes-stacked-barchart.js | 4 +- ...app-ncontainers-by-nodes-stacked-barchart.js | 4 +- .../app/components/queue-usage-donut-chart.js | 4 +- .../main/webapp/app/components/queue-view.js | 11 +- .../main/webapp/app/components/simple-table.js | 6 +- .../webapp/app/components/stacked-barchart.js | 23 +- .../main/webapp/app/components/timeline-view.js | 25 +- .../main/webapp/app/components/tree-selector.js | 41 +- .../webapp/app/controllers/app-table-columns.js | 98 + .../main/webapp/app/controllers/application.js | 8 +- .../webapp/app/controllers/yarn-app-attempt.js | 2 +- .../webapp/app/controllers/yarn-app-attempts.js | 2 +- .../src/main/webapp/app/controllers/yarn-app.js | 8 +- .../main/webapp/app/controllers/yarn-apps.js | 2 +- .../webapp/app/controllers/yarn-apps/apps.js | 25 + .../app/controllers/yarn-apps/services.js | 25 + .../app/controllers/yarn-container-log.js | 7 +- .../webapp/app/controllers/yarn-node-app.js | 7 +- .../webapp/app/controllers/yarn-node-apps.js | 2 +- .../app/controllers/yarn-node-container.js | 39 + .../app/controllers/yarn-node-containers.js | 2 +- .../main/webapp/app/controllers/yarn-node.js | 5 +- .../webapp/app/controllers/yarn-nodes/table.js | 109 + .../webapp/app/controllers/yarn-queue-apps.js | 46 - .../main/webapp/app/controllers/yarn-queue.js | 17 +- .../webapp/app/controllers/yarn-queue/apps.js | 31 + .../main/webapp/app/controllers/yarn-queues.js | 1 + .../webapp/app/controllers/yarn-services.js | 2 +- .../src/main/webapp/app/helpers/divide.js | 2 +- .../main/webapp/app/helpers/log-files-comma.js | 5 +- .../src/main/webapp/app/helpers/node-link.js | 2 +- .../src/main/webapp/app/helpers/node-menu.js | 10 +- .../src/main/webapp/app/helpers/node-name.js | 4 +- .../src/main/webapp/app/initializers/loader.js | 8 +- .../main/webapp/app/models/yarn-app-attempt.js | 17 +- .../src/main/webapp/app/models/yarn-app.js | 12 +- .../src/main/webapp/app/models/yarn-node-app.js | 8 +- .../webapp/app/models/yarn-node-container.js | 12 +- .../src/main/webapp/app/models/yarn-queue.js | 14 +- .../src/main/webapp/app/models/yarn-rm-node.js | 18 +- .../src/main/webapp/app/models/yarn-user.js | 2 +- .../src/main/webapp/app/router.js | 6 +- .../src/main/webapp/app/routes/application.js | 3 +- .../main/webapp/app/routes/cluster-overview.js | 4 +- .../main/webapp/app/routes/yarn-app-attempt.js | 12 +- .../main/webapp/app/routes/yarn-app-attempts.js | 2 - .../src/main/webapp/app/routes/yarn-app.js | 2 +- .../src/main/webapp/app/routes/yarn-apps.js | 4 +- .../main/webapp/app/routes/yarn-apps/apps.js | 9 + .../webapp/app/routes/yarn-apps/services.js | 11 + .../webapp/app/routes/yarn-container-log.js | 2 +- .../src/main/webapp/app/routes/yarn-node-app.js | 2 +- .../webapp/app/routes/yarn-node-container.js | 2 +- .../src/main/webapp/app/routes/yarn-node.js | 4 +- .../src/main/webapp/app/routes/yarn-nodes.js | 4 +- .../main/webapp/app/routes/yarn-queue-apps.js | 42 - .../src/main/webapp/app/routes/yarn-queue.js | 5 +- .../main/webapp/app/routes/yarn-queue/apps.js | 22 + .../main/webapp/app/routes/yarn-queue/info.js | 22 + .../src/main/webapp/app/routes/yarn-queues.js | 36 +- .../main/webapp/app/routes/yarn-queues/index.js | 2 + .../webapp/app/serializers/yarn-app-attempt.js | 23 +- .../src/main/webapp/app/serializers/yarn-app.js | 43 +- .../app/serializers/yarn-container-log.js | 3 +- .../webapp/app/serializers/yarn-container.js | 22 +- .../webapp/app/serializers/yarn-node-app.js | 24 +- .../app/serializers/yarn-node-container.js | 7 +- .../main/webapp/app/serializers/yarn-node.js | 8 +- .../main/webapp/app/serializers/yarn-queue.js | 10 +- .../main/webapp/app/serializers/yarn-rm-node.js | 7 +- .../src/main/webapp/app/styles/app.css | 28 +- .../main/webapp/app/templates/application.hbs | 8 +- .../webapp/app/templates/cluster-overview.hbs | 10 +- .../templates/components/app-attempt-table.hbs | 6 +- .../templates/components/app-timeout-bar.hbs | 34 + .../templates/components/container-table.hbs | 6 +- .../src/main/webapp/app/templates/loading.hbs | 23 + .../src/main/webapp/app/templates/yarn-app.hbs | 18 +- .../webapp/app/templates/yarn-apps/apps.hbs | 13 +- .../webapp/app/templates/yarn-apps/loading.hbs | 23 + .../webapp/app/templates/yarn-apps/services.hbs | 4 +- .../webapp/app/templates/yarn-nodes/table.hbs | 42 +- .../webapp/app/templates/yarn-queue-apps.hbs | 64 - .../main/webapp/app/templates/yarn-queue.hbs | 69 +- .../webapp/app/templates/yarn-queue/apps.hbs | 27 + .../webapp/app/templates/yarn-queue/info.hbs | 84 + .../main/webapp/app/templates/yarn-queues.hbs | 8 +- .../src/main/webapp/app/utils/color-utils.js | 6 +- .../src/main/webapp/app/utils/converter.js | 5 +- .../main/webapp/app/utils/href-address-utils.js | 2 - .../src/main/webapp/app/utils/mock.js | 4 +- .../src/main/webapp/app/utils/sorter.js | 8 +- .../src/main/webapp/ember-cli-build.js | 2 +- .../hadoop-yarn-ui/src/main/webapp/package.json | 9 +- .../webapp/public/assets/images/spinner.gif | Bin 0 -> 33076 bytes .../components/app-timeout-bar-test.js | 34 + .../unit/controllers/app-table-columns-test.js | 30 + .../unit/controllers/yarn-apps/apps-test.js | 30 + .../unit/controllers/yarn-apps/services-test.js | 30 + .../controllers/yarn-node-container-test.js | 30 + .../unit/controllers/yarn-nodes/table-test.js | 30 + .../unit/controllers/yarn-queue-apps-test.js | 30 - .../unit/controllers/yarn-queue/apps-test.js | 30 + .../unit/models/yarn-container-log-test.js | 1 + .../tests/unit/models/yarn-node-app-test.js | 1 + .../unit/models/yarn-node-container-test.js | 1 + .../webapp/tests/unit/models/yarn-node-test.js | 1 + .../tests/unit/models/yarn-rm-node-test.js | 1 + .../unit/routes/yarn-container-log-test.js | 24 +- .../tests/unit/routes/yarn-node-app-test.js | 28 +- .../tests/unit/routes/yarn-node-apps-test.js | 30 +- .../unit/routes/yarn-node-container-test.js | 28 +- .../unit/routes/yarn-node-containers-test.js | 30 +- .../webapp/tests/unit/routes/yarn-node-test.js | 4 +- .../webapp/tests/unit/routes/yarn-nodes-test.js | 2 +- .../tests/unit/routes/yarn-queue-apps-test.js | 29 - .../tests/unit/routes/yarn-queue/apps-test.js | 29 + .../tests/unit/routes/yarn-queue/info-test.js | 29 + hadoop-yarn-project/hadoop-yarn/pom.xml | 5 +- hadoop-yarn-project/pom.xml | 9 +- pom.xml | 38 +- 1980 files changed, 126997 insertions(+), 75039 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org