Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E0B07200D3A for ; Wed, 15 Nov 2017 20:55:13 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id DF5A0160C0A; Wed, 15 Nov 2017 19:55:13 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3CC5D160BF6 for ; Wed, 15 Nov 2017 20:55:12 +0100 (CET) Received: (qmail 69177 invoked by uid 500); 15 Nov 2017 19:54:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 68334 invoked by uid 99); 15 Nov 2017 19:54:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 15 Nov 2017 19:54:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7919FF5E97; Wed, 15 Nov 2017 19:54:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Wed, 15 Nov 2017 19:55:45 -0000 Message-Id: <46f17e1908d6477fa02bc96fbf4d1397@git.apache.org> In-Reply-To: <7a67b0e02187425a9e66417a8dd9b483@git.apache.org> References: <7a67b0e02187425a9e66417a8dd9b483@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240 archived-at: Wed, 15 Nov 2017 19:55:14 -0000 Merge branch 'trunk' into HDFS-7240 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f21b3a5b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f21b3a5b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f21b3a5b Branch: refs/heads/HDFS-7240 Commit: f21b3a5bd40d7cb2213f9f0ee93ad793bff84f1b Parents: a61aafd fac72ee Author: Anu Engineer Authored: Wed Nov 15 11:45:56 2017 -0800 Committer: Anu Engineer Committed: Wed Nov 15 11:45:56 2017 -0800 ---------------------------------------------------------------------- .gitignore | 1 + LICENSE.txt | 26 + NOTICE.txt | 10 + dev-support/bin/create-release | 4 +- .../resources/assemblies/hadoop-yarn-dist.xml | 25 + .../hadoop-cloud-storage/pom.xml | 5 + .../org/apache/hadoop/conf/Configuration.java | 169 +- .../apache/hadoop/crypto/CryptoInputStream.java | 32 +- .../crypto/key/kms/KMSClientProvider.java | 46 +- .../org/apache/hadoop/fs/FSDataInputStream.java | 15 +- .../java/org/apache/hadoop/fs/FileSystem.java | 3 +- .../org/apache/hadoop/fs/LocatedFileStatus.java | 11 +- .../main/java/org/apache/hadoop/fs/Options.java | 3 +- .../apache/hadoop/fs/StreamCapabilities.java | 48 +- .../hadoop/fs/StreamCapabilitiesPolicy.java | 51 + .../apache/hadoop/io/compress/GzipCodec.java | 37 +- .../metrics2/impl/MetricsSinkAdapter.java | 12 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 7 +- .../hadoop/metrics2/source/JvmMetrics.java | 15 + .../hadoop/metrics2/source/JvmMetricsInfo.java | 3 +- .../hadoop/security/UserGroupInformation.java | 47 +- .../org/apache/hadoop/util/GcTimeMonitor.java | 242 +++ .../src/site/markdown/Compatibility.md | 132 +- .../src/site/markdown/DownstreamDev.md | 432 +++++ .../src/site/markdown/filesystem/filesystem.md | 21 +- .../conf/TestConfigurationFieldsBase.java | 422 ++--- .../hadoop/crypto/CryptoStreamsTestBase.java | 72 +- .../apache/hadoop/crypto/TestCryptoStreams.java | 28 +- .../crypto/TestCryptoStreamsForLocalFS.java | 5 + .../hadoop/crypto/TestCryptoStreamsNormal.java | 5 + .../kms/TestLoadBalancingKMSClientProvider.java | 15 + .../hadoop/io/compress/TestGzipCodec.java | 169 ++ .../metrics2/impl/TestMetricsSystemImpl.java | 49 + .../hadoop/metrics2/source/TestJvmMetrics.java | 92 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 16 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 12 +- .../hadoop/hdfs/DistributedFileSystem.java | 7 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 4 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hdfs/protocol/ErasureCodingPolicy.java | 62 +- .../hdfs/protocol/ErasureCodingPolicyInfo.java | 106 ++ .../hdfs/protocol/ErasureCodingPolicyState.java | 4 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 80 +- .../hdfs/protocol/HdfsLocatedFileStatus.java | 110 -- .../ClientNamenodeProtocolTranslatorPB.java | 13 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 122 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 1 + .../protocol/TestErasureCodingPolicyInfo.java | 72 + .../hadoop/hdfs/nfs/nfs3/TestExportsTable.java | 88 +- .../dev-support/findbugsExcludeFile.xml | 7 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 8 +- .../hdfs/qjournal/server/JournalNodeSyncer.java | 55 +- .../server/blockmanagement/BlockManager.java | 9 +- .../BlockPlacementPolicyDefault.java | 4 +- .../BlockPlacementPolicyRackFaultTolerant.java | 70 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 51 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 40 +- .../datanode/web/webhdfs/ParameterParser.java | 66 +- .../federation/router/RouterRpcServer.java | 4 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 6 +- .../namenode/ErasureCodingPolicyManager.java | 126 +- .../server/namenode/FSDirErasureCodingOp.java | 7 +- .../server/namenode/FSDirStatAndListingOp.java | 61 +- .../server/namenode/FSImageFormatProtobuf.java | 10 +- .../hdfs/server/namenode/FSNamesystem.java | 7 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 4 +- .../hdfs/server/namenode/NamenodeFsck.java | 11 +- .../server/namenode/ha/StandbyCheckpointer.java | 1 + .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 +- .../OfflineImageReconstructor.java | 79 +- .../offlineImageViewer/PBImageXmlWriter.java | 80 + .../src/main/webapps/hdfs/explorer.js | 11 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 16 + .../hadoop/hdfs/TestBlockStoragePolicy.java | 10 +- .../apache/hadoop/hdfs/TestDFSOutputStream.java | 2 +- .../hdfs/TestErasureCodingMultipleRacks.java | 131 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 22 +- .../hdfs/qjournal/server/TestJournalNode.java | 103 +- .../blockmanagement/TestBlockManager.java | 5 +- .../web/webhdfs/TestParameterParser.java | 81 +- .../hdfs/server/mover/TestStorageMover.java | 8 +- .../server/namenode/TestFSEditLogLoader.java | 12 +- .../hdfs/server/namenode/TestFSImage.java | 17 +- .../TestOfflineImageViewer.java | 101 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 59 + .../org/apache/hadoop/mapred/BackupStore.java | 7 +- .../org/apache/hadoop/mapred/CleanupQueue.java | 8 +- .../java/org/apache/hadoop/mapred/Counters.java | 4 +- .../DeprecatedQueueConfigurationParser.java | 8 +- .../apache/hadoop/mapred/FileInputFormat.java | 8 +- .../hadoop/mapred/FileOutputCommitter.java | 6 +- .../java/org/apache/hadoop/mapred/IFile.java | 6 +- .../apache/hadoop/mapred/IFileInputStream.java | 7 +- .../org/apache/hadoop/mapred/IndexCache.java | 6 +- .../apache/hadoop/mapred/JobACLsManager.java | 6 +- .../java/org/apache/hadoop/mapred/JobConf.java | 6 +- .../apache/hadoop/mapred/JobEndNotifier.java | 8 +- .../org/apache/hadoop/mapred/JvmContext.java | 8 +- .../apache/hadoop/mapred/LineRecordReader.java | 8 +- .../java/org/apache/hadoop/mapred/MapTask.java | 7 +- .../java/org/apache/hadoop/mapred/Merger.java | 6 +- .../java/org/apache/hadoop/mapred/Queue.java | 10 +- .../hadoop/mapred/QueueConfigurationParser.java | 8 +- .../org/apache/hadoop/mapred/QueueManager.java | 6 +- .../org/apache/hadoop/mapred/ReduceTask.java | 9 +- .../org/apache/hadoop/mapred/SortedRanges.java | 8 +- .../java/org/apache/hadoop/mapred/Task.java | 30 +- .../java/org/apache/hadoop/mapred/TaskLog.java | 11 +- .../org/apache/hadoop/mapred/TaskStatus.java | 8 +- .../apache/hadoop/mapred/jobcontrol/Job.java | 6 +- .../mapred/lib/FieldSelectionMapReduce.java | 7 +- .../apache/hadoop/mapred/lib/InputSampler.java | 6 +- .../mapred/lib/MultithreadedMapRunner.java | 8 +- .../apache/hadoop/mapred/pipes/Application.java | 7 +- .../hadoop/mapred/pipes/BinaryProtocol.java | 8 +- .../hadoop/mapred/pipes/PipesReducer.java | 7 +- .../apache/hadoop/mapred/pipes/Submitter.java | 6 +- .../org/apache/hadoop/mapreduce/Cluster.java | 7 +- .../apache/hadoop/mapreduce/CryptoUtils.java | 6 +- .../java/org/apache/hadoop/mapreduce/Job.java | 6 +- .../hadoop/mapreduce/JobResourceUploader.java | 7 +- .../hadoop/mapreduce/JobSubmissionFiles.java | 7 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 11 +- .../mapreduce/counters/AbstractCounters.java | 7 +- .../counters/FileSystemCounterGroup.java | 7 +- .../counters/FrameworkCounterGroup.java | 7 +- .../mapreduce/jobhistory/EventWriter.java | 8 +- .../mapreduce/jobhistory/JobHistoryParser.java | 7 +- .../mapreduce/lib/db/BigDecimalSplitter.java | 7 +- .../hadoop/mapreduce/lib/db/DBInputFormat.java | 7 +- .../hadoop/mapreduce/lib/db/DBOutputFormat.java | 7 +- .../hadoop/mapreduce/lib/db/DBRecordReader.java | 7 +- .../lib/db/DataDrivenDBInputFormat.java | 7 +- .../lib/db/DataDrivenDBRecordReader.java | 7 +- .../hadoop/mapreduce/lib/db/DateSplitter.java | 6 +- .../hadoop/mapreduce/lib/db/FloatSplitter.java | 7 +- .../mapreduce/lib/db/OracleDBRecordReader.java | 7 +- .../lib/db/OracleDataDrivenDBInputFormat.java | 3 - .../hadoop/mapreduce/lib/db/TextSplitter.java | 6 +- .../lib/fieldsel/FieldSelectionMapper.java | 7 +- .../lib/fieldsel/FieldSelectionReducer.java | 7 +- .../lib/input/CombineFileInputFormat.java | 7 +- .../mapreduce/lib/input/FileInputFormat.java | 7 +- .../lib/input/FixedLengthRecordReader.java | 8 +- .../mapreduce/lib/input/LineRecordReader.java | 7 +- .../lib/input/SequenceFileInputFilter.java | 9 +- .../mapreduce/lib/jobcontrol/ControlledJob.java | 7 +- .../mapreduce/lib/jobcontrol/JobControl.java | 6 +- .../mapreduce/lib/map/MultithreadedMapper.java | 7 +- .../lib/output/FileOutputCommitter.java | 7 +- .../lib/output/PartialFileOutputCommitter.java | 8 +- .../mapreduce/lib/partition/InputSampler.java | 6 +- .../lib/partition/KeyFieldBasedPartitioner.java | 6 +- .../lib/partition/TotalOrderPartitioner.java | 9 +- .../mapreduce/security/SecureShuffleUtils.java | 7 +- .../hadoop/mapreduce/security/TokenCache.java | 6 +- .../hadoop/mapreduce/split/JobSplitWriter.java | 7 +- .../mapreduce/task/reduce/EventFetcher.java | 6 +- .../hadoop/mapreduce/task/reduce/Fetcher.java | 10 +- .../task/reduce/InMemoryMapOutput.java | 9 +- .../mapreduce/task/reduce/LocalFetcher.java | 8 +- .../mapreduce/task/reduce/MergeManagerImpl.java | 7 +- .../mapreduce/task/reduce/MergeThread.java | 6 +- .../mapreduce/task/reduce/OnDiskMapOutput.java | 9 +- .../task/reduce/ShuffleSchedulerImpl.java | 9 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 8 +- .../hadoop/mapreduce/util/ProcessTree.java | 7 +- .../org/apache/hadoop/mapred/TestCounters.java | 6 +- .../hadoop/mapred/TestFileInputFormat.java | 7 +- .../TestClientDistributedCacheManager.java | 8 +- .../jobhistory/TestHistoryViewerPrinter.java | 8 +- .../lib/input/TestFileInputFormat.java | 7 +- .../lib/output/TestFileOutputCommitter.java | 8 +- .../mapreduce/task/reduce/TestFetcher.java | 6 +- .../mapreduce/v2/hs/CachedHistoryStorage.java | 7 +- .../hadoop/mapreduce/v2/hs/CompletedJob.java | 6 +- .../hadoop/mapreduce/v2/hs/HSAuditLogger.java | 7 +- .../hadoop/mapreduce/v2/hs/HSProxies.java | 6 +- .../mapreduce/v2/hs/HistoryClientService.java | 7 +- .../mapreduce/v2/hs/HistoryFileManager.java | 12 +- ...istoryServerFileSystemStateStoreService.java | 20 +- .../HistoryServerLeveldbStateStoreService.java | 22 +- .../v2/hs/JHSDelegationTokenSecretManager.java | 6 +- .../hadoop/mapreduce/v2/hs/JobHistory.java | 6 +- .../mapreduce/v2/hs/JobHistoryServer.java | 9 +- .../hadoop/mapreduce/v2/hs/PartialJob.java | 6 +- .../mapreduce/v2/hs/server/HSAdminServer.java | 7 +- .../mapreduce/v2/hs/TestJobHistoryEvents.java | 7 +- .../mapreduce/v2/hs/TestJobHistoryParsing.java | 7 +- .../mapreduce/v2/hs/webapp/TestHSWebApp.java | 6 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 16 +- .../io/TestSequenceFileMergeProgress.java | 4 +- .../TestSequenceFileAsBinaryInputFormat.java | 4 +- .../TestSequenceFileAsTextInputFormat.java | 4 +- .../mapred/TestSequenceFileInputFilter.java | 4 +- .../mapred/TestSequenceFileInputFormat.java | 4 +- .../hadoop/mapred/FadvisedChunkedFile.java | 7 +- .../hadoop/mapred/FadvisedFileRegion.java | 7 +- .../apache/hadoop/mapred/ShuffleHandler.java | 13 +- .../hadoop/mapred/TestFadvisedFileRegion.java | 18 +- .../hadoop/mapred/TestShuffleHandler.java | 7 +- hadoop-project/pom.xml | 44 +- hadoop-project/src/site/markdown/index.md.vm | 20 +- hadoop-project/src/site/site.xml | 15 +- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 75 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 106 ++ .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 12 + .../fs/aliyun/oss/FileStatusAcceptor.java | 125 ++ .../site/markdown/tools/hadoop-aliyun/index.md | 6 +- .../hadoop/fs/azure/BlockBlobAppendStream.java | 17 +- .../tools/mapred/lib/DynamicInputChunk.java | 2 +- .../service/ResourceEstimatorService.java | 90 +- .../translator/impl/BaseLogParser.java | 5 +- .../translator/impl/LogParserUtil.java | 11 +- .../service/TestResourceEstimatorService.java | 27 +- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 86 +- .../hadoop/yarn/sls/conf/SLSConfiguration.java | 6 +- .../yarn/sls/nodemanager/NMSimulator.java | 15 +- .../src/site/markdown/SchedulerLoadSimulator.md | 48 + .../yarn/sls/nodemanager/TestNMSimulator.java | 3 +- .../hadoop-yarn/bin/start-yarn.sh | 2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 26 +- .../hadoop-yarn/conf/yarn-env.sh | 18 + .../dev-support/findbugs-exclude.xml | 28 +- .../hadoop/yarn/api/records/Resource.java | 50 +- .../yarn/api/records/ResourceInformation.java | 5 + .../api/records/impl/LightWeightResource.java | 94 +- .../org/apache/hadoop/yarn/conf/HAUtil.java | 12 +- .../hadoop/yarn/conf/YarnConfiguration.java | 2 + .../yarn/util/resource/ResourceUtils.java | 193 +- .../src/main/proto/yarn_protos.proto | 2 +- .../dev-support/findbugs-exclude.xml | 20 + .../hadoop-yarn-services-api/pom.xml | 130 ++ .../hadoop/yarn/service/webapp/ApiServer.java | 298 +++ .../yarn/service/webapp/ApiServerWebApp.java | 161 ++ .../definition/YARN-Services-Examples.md | 245 +++ ...RN-Simplified-V1-API-Layer-For-Services.yaml | 471 +++++ .../src/main/resources/log4j-server.properties | 76 + .../src/main/resources/webapps/api-server/app | 16 + .../src/main/webapp/WEB-INF/web.xml | 36 + .../hadoop/yarn/service/ServiceClientTest.java | 107 ++ .../hadoop/yarn/service/TestApiServer.java | 366 ++++ .../dev-support/findbugs-exclude.xml | 48 + .../conf/yarnservice-log4j.properties | 62 + .../examples/httpd-no-dns/httpd-no-dns.json | 62 + .../httpd-no-dns/httpd-proxy-no-dns.conf | 24 + .../examples/httpd/httpd-proxy.conf | 24 + .../examples/httpd/httpd.json | 55 + .../examples/sleeper/sleeper.json | 15 + .../hadoop-yarn-services-core/pom.xml | 255 +++ .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java | 132 ++ .../yarn/service/ContainerFailureTracker.java | 89 + .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 169 ++ .../hadoop/yarn/service/ServiceMetrics.java | 94 + .../hadoop/yarn/service/ServiceScheduler.java | 691 +++++++ .../yarn/service/api/ServiceApiConstants.java | 74 + .../yarn/service/api/records/Artifact.java | 168 ++ .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 430 +++++ .../service/api/records/ComponentState.java | 30 + .../yarn/service/api/records/ConfigFile.java | 233 +++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 +++ .../yarn/service/api/records/Container.java | 298 +++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 ++ .../service/api/records/PlacementPolicy.java | 102 + .../service/api/records/ReadinessCheck.java | 183 ++ .../yarn/service/api/records/Resource.java | 161 ++ .../yarn/service/api/records/Service.java | 390 ++++ .../yarn/service/api/records/ServiceState.java | 33 + .../yarn/service/api/records/ServiceStatus.java | 148 ++ .../yarn/service/client/ClientAMProxy.java | 57 + .../yarn/service/client/ServiceClient.java | 960 ++++++++++ .../yarn/service/component/Component.java | 584 ++++++ .../yarn/service/component/ComponentEvent.java | 83 + .../service/component/ComponentEventType.java | 27 + .../yarn/service/component/ComponentState.java | 25 + .../component/instance/ComponentInstance.java | 549 ++++++ .../instance/ComponentInstanceEvent.java | 58 + .../instance/ComponentInstanceEventType.java | 26 + .../component/instance/ComponentInstanceId.java | 91 + .../instance/ComponentInstanceState.java | 26 + .../yarn/service/conf/RestApiConstants.java | 39 + .../yarn/service/conf/SliderExitCodes.java | 88 + .../yarn/service/conf/YarnServiceConf.java | 113 ++ .../yarn/service/conf/YarnServiceConstants.java | 92 + .../containerlaunch/AbstractLauncher.java | 271 +++ .../containerlaunch/ClasspathConstructor.java | 172 ++ .../containerlaunch/CommandLineBuilder.java | 86 + .../containerlaunch/ContainerLaunchService.java | 101 + .../containerlaunch/CredentialUtils.java | 319 ++++ .../containerlaunch/JavaCommandLineBuilder.java | 181 ++ .../exceptions/BadClusterStateException.java | 36 + .../BadCommandArgumentsException.java | 30 + .../service/exceptions/BadConfigException.java | 39 + .../yarn/service/exceptions/ErrorStrings.java | 42 + .../service/exceptions/ExitCodeProvider.java | 32 + .../service/exceptions/LauncherExitCodes.java | 196 ++ .../exceptions/RestApiErrorMessages.java | 92 + .../exceptions/ServiceLaunchException.java | 73 + .../service/exceptions/SliderException.java | 66 + .../yarn/service/exceptions/UsageException.java | 34 + .../pb/client/ClientAMProtocolPBClientImpl.java | 91 + .../impl/pb/service/ClientAMProtocolPB.java | 29 + .../service/ClientAMProtocolPBServiceImpl.java | 70 + .../yarn/service/monitor/ServiceMonitor.java | 147 ++ .../yarn/service/monitor/probe/HttpProbe.java | 110 ++ .../service/monitor/probe/LogEntryBuilder.java | 76 + .../yarn/service/monitor/probe/MonitorKeys.java | 66 + .../service/monitor/probe/MonitorUtils.java | 84 + .../yarn/service/monitor/probe/PortProbe.java | 98 + .../yarn/service/monitor/probe/Probe.java | 100 + .../yarn/service/monitor/probe/ProbeStatus.java | 160 ++ .../provider/AbstractClientProvider.java | 129 ++ .../provider/AbstractProviderService.java | 113 ++ .../yarn/service/provider/ProviderFactory.java | 76 + .../yarn/service/provider/ProviderService.java | 39 + .../yarn/service/provider/ProviderUtils.java | 408 ++++ .../defaultImpl/DefaultClientProvider.java | 46 + .../defaultImpl/DefaultProviderFactory.java | 51 + .../defaultImpl/DefaultProviderService.java | 36 + .../provider/docker/DockerClientProvider.java | 53 + .../service/provider/docker/DockerKeys.java | 30 + .../provider/docker/DockerProviderFactory.java | 52 + .../provider/docker/DockerProviderService.java | 57 + .../provider/tarball/TarballClientProvider.java | 65 + .../tarball/TarballProviderFactory.java | 52 + .../tarball/TarballProviderService.java | 48 + .../registry/CustomRegistryConstants.java | 57 + .../registry/YarnRegistryViewForProviders.java | 249 +++ .../timelineservice/ServiceMetricsSink.java | 102 + .../ServiceTimelineEntityType.java | 39 + .../timelineservice/ServiceTimelineEvent.java | 36 + .../ServiceTimelineMetricsConstants.java | 92 + .../ServiceTimelinePublisher.java | 381 ++++ .../service/timelineservice/package-info.java | 27 + .../utils/ApplicationReportSerDeser.java | 56 + .../service/utils/ClientRegistryBinder.java | 201 ++ .../hadoop/yarn/service/utils/Comparators.java | 62 + .../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++ .../hadoop/yarn/service/utils/ConfigUtils.java | 97 + .../yarn/service/utils/CoreFileSystem.java | 521 ++++++ .../hadoop/yarn/service/utils/Duration.java | 109 ++ .../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++ .../yarn/service/utils/PatternValidator.java | 58 + .../hadoop/yarn/service/utils/PortScanner.java | 113 ++ .../service/utils/PublishedConfiguration.java | 196 ++ .../utils/PublishedConfigurationOutputter.java | 212 +++ .../utils/SerializedApplicationReport.java | 98 + .../yarn/service/utils/ServiceApiUtil.java | 401 ++++ .../service/utils/ServiceRegistryUtils.java | 71 + .../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++ .../yarn/service/utils/SliderFileSystem.java | 51 + .../yarn/service/utils/ZookeeperUtils.java | 146 ++ .../src/main/proto/ClientAMProtocol.proto | 56 + .../hadoop/yarn/service/MockServiceAM.java | 221 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 262 +++ .../hadoop/yarn/service/TestServiceApiUtil.java | 480 +++++ .../yarn/service/TestYarnNativeServices.java | 403 ++++ .../client/TestBuildExternalComponents.java | 119 ++ .../yarn/service/client/TestServiceCLI.java | 127 ++ .../yarn/service/conf/ExampleAppJson.java | 65 + .../yarn/service/conf/TestAppJsonResolve.java | 200 ++ .../service/conf/TestLoadExampleAppJson.java | 71 + .../service/conf/TestValidateServiceNames.java | 125 ++ .../service/monitor/TestServiceMonitor.java | 104 ++ .../providers/TestAbstractClientProvider.java | 118 ++ .../service/providers/TestProviderFactory.java | 76 + .../TestServiceTimelinePublisher.java | 291 +++ .../src/test/resources/example-app.json | 15 + .../service/conf/examples/app-override.json | 76 + .../hadoop/yarn/service/conf/examples/app.json | 50 + .../yarn/service/conf/examples/default.json | 16 + .../yarn/service/conf/examples/external0.json | 15 + .../yarn/service/conf/examples/external1.json | 30 + .../yarn/service/conf/examples/external2.json | 22 + .../src/test/resources/yarn-site.xml | 19 + .../hadoop-yarn-services/pom.xml | 38 + .../hadoop-yarn-applications/pom.xml | 2 + .../hadoop/yarn/client/api/AppAdminClient.java | 222 +++ .../hadoop/yarn/client/cli/ApplicationCLI.java | 318 +++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 158 +- .../impl/pb/ContainerLaunchContextPBImpl.java | 16 +- .../api/records/impl/pb/ContainerPBImpl.java | 2 +- .../api/records/impl/pb/ResourcePBImpl.java | 88 +- .../yarn/security/ContainerTokenIdentifier.java | 20 +- .../java/org/apache/hadoop/yarn/util/Apps.java | 36 + .../hadoop/yarn/util/resource/Resources.java | 59 + .../src/main/proto/yarn_security_token.proto | 1 + .../src/main/resources/yarn-default.xml | 11 +- .../yarn/util/resource/TestResourceUtils.java | 37 + .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 78 +- .../registry/client/api/DNSOperations.java | 60 + .../client/api/DNSOperationsFactory.java | 78 + .../registry/client/api/RegistryConstants.java | 136 +- .../client/api/RegistryOperationsFactory.java | 8 + .../client/binding/RegistryTypeUtils.java | 3 +- .../registry/client/impl/zk/CuratorService.java | 271 ++- .../registry/client/impl/zk/ListenerHandle.java | 25 + .../registry/client/impl/zk/PathListener.java | 30 + .../impl/zk/RegistryOperationsService.java | 4 +- .../types/yarn/YarnRegistryAttributes.java | 16 +- .../dns/ApplicationServiceRecordProcessor.java | 366 ++++ .../server/dns/BaseServiceRecordProcessor.java | 468 +++++ .../dns/ContainerServiceRecordProcessor.java | 283 +++ .../dns/PrivilegedRegistryDNSStarter.java | 80 + .../server/dns/RecordCreatorFactory.java | 275 +++ .../hadoop/registry/server/dns/RegistryDNS.java | 1756 ++++++++++++++++++ .../registry/server/dns/RegistryDNSServer.java | 267 +++ .../registry/server/dns/ReverseZoneUtils.java | 171 ++ .../registry/server/dns/SecureableZone.java | 151 ++ .../server/dns/ServiceRecordProcessor.java | 53 + .../registry/server/dns/ZoneSelector.java | 33 + .../registry/server/dns/package-info.java | 26 + .../registry/server/dns/TestRegistryDNS.java | 627 +++++++ .../server/dns/TestReverseZoneUtils.java | 89 + .../server/dns/TestSecureRegistryDNS.java | 44 + .../test/resources/0.17.172.in-addr.arpa.zone | 36 + .../src/test/resources/test.private | 32 + .../api/protocolrecords/NMContainerStatus.java | 18 +- .../impl/pb/NMContainerStatusPBImpl.java | 15 +- .../OpportunisticContainerAllocator.java | 2 +- .../yarn/server/webapp/dao/ContainerInfo.java | 39 +- .../yarn_server_common_service_protos.proto | 1 + .../yarn/server/nodemanager/NodeManager.java | 3 +- .../containermanager/ContainerManagerImpl.java | 5 +- .../container/ContainerImpl.java | 8 +- .../runtime/DockerLinuxContainerRuntime.java | 38 +- .../localizer/ResourceLocalizationService.java | 32 +- .../test/utils/test_docker_util.cc | 62 +- .../runtime/TestDockerContainerRuntime.java | 43 + .../TestResourceLocalizationService.java | 144 ++ .../TestContainerSchedulerRecovery.java | 314 ++++ .../server/resourcemanager/ResourceManager.java | 12 +- .../blacklist/SimpleBlacklistManager.java | 8 +- .../monitor/SchedulingMonitor.java | 15 +- .../ProportionalCapacityPreemptionPolicy.java | 79 +- .../recovery/ZKRMStateStore.java | 5 +- .../CapacitySchedulerPlanFollower.java | 10 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 7 +- .../rmapp/attempt/RMAppAttemptImpl.java | 38 +- .../rmapp/attempt/RMAppAttemptMetrics.java | 15 +- .../scheduler/AbstractYarnScheduler.java | 13 +- .../scheduler/AppSchedulingInfo.java | 93 +- .../scheduler/ClusterNodeTracker.java | 18 +- .../scheduler/ContainerUpdateContext.java | 15 +- .../scheduler/SchedulerApplicationAttempt.java | 30 +- .../scheduler/SchedulerUtils.java | 13 + .../scheduler/activities/ActivitiesLogger.java | 2 +- .../scheduler/capacity/AbstractCSQueue.java | 4 +- .../capacity/AbstractManagedParentQueue.java | 232 +++ .../capacity/AutoCreatedLeafQueue.java | 129 ++ .../scheduler/capacity/CSQueue.java | 12 +- .../scheduler/capacity/CapacityScheduler.java | 132 +- .../CapacitySchedulerConfiguration.java | 11 +- .../capacity/CapacitySchedulerQueueManager.java | 4 +- .../scheduler/capacity/LeafQueue.java | 43 +- .../scheduler/capacity/ParentQueue.java | 50 +- .../scheduler/capacity/PlanQueue.java | 191 +- .../scheduler/capacity/ReservationQueue.java | 122 -- .../allocator/AbstractContainerAllocator.java | 10 +- .../capacity/allocator/ContainerAllocator.java | 11 +- .../allocator/RegularContainerAllocator.java | 57 +- .../capacity/conf/ZKConfigurationStore.java | 3 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 23 +- .../scheduler/fair/FSAppAttempt.java | 17 +- .../scheduler/fair/FairScheduler.java | 27 +- .../scheduler/fifo/FifoScheduler.java | 10 + .../placement/AppPlacementAllocator.java | 163 ++ .../scheduler/placement/CandidateNodeSet.java | 61 + .../placement/CandidateNodeSetUtils.java | 44 + .../LocalityAppPlacementAllocator.java | 422 +++++ .../LocalitySchedulingPlacementSet.java | 416 ----- .../scheduler/placement/PlacementSet.java | 65 - .../scheduler/placement/PlacementSetUtils.java | 36 - .../placement/SchedulingPlacementSet.java | 158 -- .../placement/SimpleCandidateNodeSet.java | 68 + .../scheduler/placement/SimplePlacementSet.java | 70 - .../scheduler/placement/package-info.java | 28 + .../security/RMContainerTokenSecretManager.java | 36 +- .../server/resourcemanager/webapp/RMWebApp.java | 19 + .../yarn/server/resourcemanager/TestRM.java | 11 +- .../server/resourcemanager/TestRMRestart.java | 2 +- .../TestResourceTrackerService.java | 6 +- .../applicationsmanager/TestAMRestart.java | 36 +- ...estProportionalCapacityPreemptionPolicy.java | 42 +- .../rmapp/TestApplicationLifetimeMonitor.java | 24 +- .../capacity/TestAutoCreatedLeafQueue.java | 113 ++ .../capacity/TestCapacityScheduler.java | 5 +- .../TestCapacitySchedulerDynamicBehavior.java | 32 +- .../TestCapacitySchedulerLazyPreemption.java | 2 +- .../scheduler/capacity/TestChildQueueOrder.java | 18 +- .../capacity/TestContainerAllocation.java | 5 +- .../capacity/TestContainerResizing.java | 7 +- .../TestNodeLabelContainerAllocation.java | 2 +- .../scheduler/capacity/TestParentQueue.java | 119 +- .../capacity/TestReservationQueue.java | 110 -- .../webapp/FederationInterceptorREST.java | 26 +- .../server/TestContainerManagerSecurity.java | 38 +- .../src/site/markdown/DockerContainers.md | 26 +- .../src/site/markdown/ResourceModel.md | 275 +++ .../src/site/markdown/ResourceProfiles.md | 116 -- .../src/site/markdown/YarnCommands.md | 33 +- .../src/site/markdown/registry/index.md | 1 + .../markdown/registry/registry-configuration.md | 4 +- .../src/site/markdown/yarn-service/Concepts.md | 46 + .../markdown/yarn-service/Configurations.md | 165 ++ .../src/site/markdown/yarn-service/Examples.md | 161 ++ .../src/site/markdown/yarn-service/Overview.md | 61 + .../site/markdown/yarn-service/QuickStart.md | 217 +++ .../site/markdown/yarn-service/RegistryDNS.md | 200 ++ .../markdown/yarn-service/ServiceDiscovery.md | 136 ++ .../markdown/yarn-service/YarnServiceAPI.md | 598 ++++++ .../src/site/resources/images/dns_overview.png | Bin 0 -> 41908 bytes .../resources/images/dns_record_creation.jpeg | Bin 0 -> 51911 bytes .../resources/images/dns_record_removal.jpeg | Bin 0 -> 58041 bytes .../main/webapp/app/adapters/restabstract.js | 50 + .../app/adapters/yarn-component-instance.js | 32 + .../webapp/app/adapters/yarn-container-log.js | 4 +- .../app/adapters/yarn-service-component.js | 32 + .../webapp/app/adapters/yarn-service-info.js | 32 + .../main/webapp/app/adapters/yarn-servicedef.js | 44 + .../webapp/app/components/breadcrumb-bar.js | 1 + .../webapp/app/components/confirm-dialog.js | 31 + .../webapp/app/components/deploy-service.js | 167 ++ .../components/em-table-simple-status-cell.js | 31 + .../app/components/fileconfig-viewer-dialog.js | 36 + .../main/webapp/app/components/info-tooltip.js | 44 + .../main/webapp/app/components/metrics-table.js | 24 + .../app/components/service-component-table.js | 56 + .../app/components/service-config-table.js | 89 + .../app/components/service-fileconfig-table.js | 112 ++ .../main/webapp/app/components/upload-config.js | 54 + .../webapp/app/controllers/app-table-columns.js | 30 +- .../app/controllers/yarn-app/components.js | 59 + .../webapp/app/controllers/yarn-app/configs.js | 24 + .../webapp/app/controllers/yarn-app/info.js | 60 + .../webapp/app/controllers/yarn-apps/apps.js | 5 +- .../app/controllers/yarn-component-instance.js | 59 + .../controllers/yarn-component-instance/info.js | 25 + .../app/controllers/yarn-component-instances.js | 59 + .../yarn-component-instances/configs.js | 25 + .../yarn-component-instances/info.js | 62 + .../app/controllers/yarn-deploy-service.js | 69 + .../webapp/app/controllers/yarn-nodes-status.js | 36 + .../webapp/app/controllers/yarn-nodes/table.js | 17 +- .../webapp/app/controllers/yarn-services.js | 4 +- .../webapp/app/helpers/check-availability.js | 28 + .../main/webapp/app/helpers/date-formatter.js | 29 + .../src/main/webapp/app/helpers/lower.js | 27 + .../src/main/webapp/app/initializers/jquery.js | 35 + .../src/main/webapp/app/models/cluster-info.js | 7 +- .../main/webapp/app/models/cluster-metric.js | 2 +- .../src/main/webapp/app/models/yarn-app.js | 1 + .../app/models/yarn-component-instance.js | 51 + .../src/main/webapp/app/models/yarn-rm-node.js | 2 +- .../webapp/app/models/yarn-service-component.js | 46 + .../main/webapp/app/models/yarn-service-info.js | 57 + .../main/webapp/app/models/yarn-servicedef.js | 278 +++ .../src/main/webapp/app/router.js | 11 + .../src/main/webapp/app/routes/application.js | 13 +- .../main/webapp/app/routes/cluster-overview.js | 5 +- .../webapp/app/routes/yarn-app/components.js | 49 + .../main/webapp/app/routes/yarn-app/configs.js | 52 + .../src/main/webapp/app/routes/yarn-app/info.js | 12 +- .../app/routes/yarn-component-instance.js | 29 + .../app/routes/yarn-component-instance/info.js | 45 + .../app/routes/yarn-component-instances.js | 29 + .../routes/yarn-component-instances/configs.js | 44 + .../app/routes/yarn-component-instances/info.js | 53 + .../webapp/app/routes/yarn-deploy-service.js | 27 + .../src/main/webapp/app/routes/yarn-services.js | 2 +- .../src/main/webapp/app/serializers/yarn-app.js | 1 + .../app/serializers/yarn-component-instance.js | 72 + .../app/serializers/yarn-service-component.js | 77 + .../webapp/app/serializers/yarn-service-info.js | 87 + .../src/main/webapp/app/services/hosts.js | 4 + .../src/main/webapp/app/styles/app.css | 299 ++- .../main/webapp/app/templates/application.hbs | 15 +- .../app/templates/components/breadcrumb-bar.hbs | 4 +- .../app/templates/components/confirm-dialog.hbs | 37 + .../app/templates/components/deploy-service.hbs | 157 ++ .../components/em-table-simple-status-cell.hbs | 27 + .../components/fileconfig-viewer-dialog.hbs | 53 + .../app/templates/components/info-tooltip.hbs | 20 + .../app/templates/components/metrics-table.hbs | 55 + .../app/templates/components/nodes-heatmap.hbs | 16 +- .../components/service-component-table.hbs | 113 ++ .../components/service-config-table.hbs | 130 ++ .../components/service-fileconfig-table.hbs | 152 ++ .../app/templates/components/upload-config.hbs | 44 + .../src/main/webapp/app/templates/yarn-app.hbs | 8 + .../app/templates/yarn-app/components.hbs | 23 + .../webapp/app/templates/yarn-app/configs.hbs | 57 + .../main/webapp/app/templates/yarn-app/info.hbs | 153 +- .../src/main/webapp/app/templates/yarn-apps.hbs | 64 +- .../app/templates/yarn-component-instance.hbs | 43 + .../templates/yarn-component-instance/info.hbs | 77 + .../app/templates/yarn-component-instances.hbs | 46 + .../yarn-component-instances/configs.hbs | 53 + .../templates/yarn-component-instances/info.hbs | 28 + .../yarn-component-instances/loading.hbs | 23 + .../app/templates/yarn-deploy-service.hbs | 33 + .../main/webapp/app/templates/yarn-nodes.hbs | 59 +- .../webapp/app/templates/yarn-nodes/heatmap.hbs | 10 +- .../webapp/app/templates/yarn-nodes/status.hbs | 35 + .../webapp/app/templates/yarn-nodes/table.hbs | 4 +- .../main/webapp/app/templates/yarn-services.hbs | 70 +- .../main/webapp/app/utils/href-address-utils.js | 2 +- .../src/main/webapp/app/utils/info-seeder.js | 26 + .../hadoop-yarn-ui/src/main/webapp/bower.json | 3 +- .../src/main/webapp/config/default-config.js | 6 +- .../src/main/webapp/config/environment.js | 1 - .../src/main/webapp/ember-cli-build.js | 1 + .../src/main/webapp/jsconfig.json | 10 +- .../hadoop-yarn-ui/src/main/webapp/package.json | 2 +- .../components/confirm-dialog-test.js | 43 + .../components/deploy-service-test.js | 43 + .../em-table-simple-status-cell-test.js | 43 + .../components/fileconfig-viewer-dialog-test.js | 43 + .../integration/components/info-tooltip-test.js | 43 + .../components/metrics-table-test.js | 43 + .../components/service-component-table-test.js | 43 + .../components/service-config-table-test.js | 43 + .../components/service-fileconfig-table-test.js | 43 + .../components/upload-config-test.js | 43 + .../adapters/yarn-component-instance-test.js | 30 + .../adapters/yarn-service-component-test.js | 30 + .../unit/adapters/yarn-service-info-test.js | 30 + .../tests/unit/adapters/yarn-servicedef-test.js | 30 + .../controllers/yarn-app/components-test.js | 30 + .../unit/controllers/yarn-app/configs-test.js | 30 + .../controllers/yarn-component-instance-test.js | 30 + .../yarn-component-instance/info-test.js | 30 + .../yarn-component-instances-test.js | 30 + .../yarn-component-instances/configs-test.js | 30 + .../yarn-component-instances/info-test.js | 30 + .../controllers/yarn-deploy-service-test.js | 30 + .../unit/helpers/check-availability-test.js | 28 + .../tests/unit/helpers/date-formatter-test.js | 28 + .../webapp/tests/unit/helpers/lower-test.js | 28 + .../unit/models/yarn-component-instance-test.js | 30 + .../unit/models/yarn-service-component-test.js | 30 + .../tests/unit/models/yarn-service-info-test.js | 30 + .../tests/unit/models/yarn-servicedef-test.js | 30 + .../unit/routes/yarn-app/components-test.js | 29 + .../tests/unit/routes/yarn-app/configs-test.js | 29 + .../unit/routes/yarn-component-instance-test.js | 29 + .../routes/yarn-component-instance/info-test.js | 29 + .../routes/yarn-component-instances-test.js | 29 + .../yarn-component-instances/configs-test.js | 29 + .../yarn-component-instances/info-test.js | 29 + .../unit/routes/yarn-deploy-service-test.js | 29 + .../serializers/yarn-component-instance-test.js | 33 + .../serializers/yarn-service-component-test.js | 33 + .../unit/serializers/yarn-service-info-test.js | 33 + .../hadoop-yarn-ui/src/main/webapp/yarn.lock | 18 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +- hadoop-yarn-project/pom.xml | 4 + 664 files changed, 41641 insertions(+), 4026 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f21b3a5b/LICENSE.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f21b3a5b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f21b3a5b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f21b3a5b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f21b3a5b/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f21b3a5b/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org