From common-commits-return-80916-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Apr 3 05:41:38 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id BA67B180677 for ; Tue, 3 Apr 2018 05:41:37 +0200 (CEST) Received: (qmail 74708 invoked by uid 500); 3 Apr 2018 03:41:20 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 67582 invoked by uid 99); 3 Apr 2018 03:41:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Apr 2018 03:41:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B709BF6AFA; Tue, 3 Apr 2018 03:41:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Tue, 03 Apr 2018 03:42:03 -0000 Message-Id: <1a888d04b5284a96a4cc72807b6dedf1@git.apache.org> In-Reply-To: <4763e3c8ae6e45b6b012694980544311@git.apache.org> References: <4763e3c8ae6e45b6b012694980544311@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [53/53] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240 Merge branch 'trunk' into HDFS-7240 Conflicts: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/DummyMXBean.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7de7daa5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7de7daa5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7de7daa5 Branch: refs/heads/HDFS-7240 Commit: 7de7daa5bb5a595fc280b05bdcb701322eae9168 Parents: 6c4221b c78cb18 Author: Anu Engineer Authored: Mon Apr 2 20:27:26 2018 -0700 Committer: Anu Engineer Committed: Mon Apr 2 20:27:26 2018 -0700 ---------------------------------------------------------------------- BUILDING.txt | 3 +- dev-support/bin/create-release | 2 +- .../org/apache/hadoop/conf/Configuration.java | 43 +- .../apache/hadoop/io/DataOutputByteBuffer.java | 137 ---- .../apache/hadoop/io/MultipleIOException.java | 10 + .../main/java/org/apache/hadoop/ipc/Server.java | 3 + .../org/apache/hadoop/metrics2/util/MBeans.java | 14 +- .../hadoop/security/LdapGroupsMapping.java | 90 ++- .../src/main/resources/core-default.xml | 40 ++ .../src/site/markdown/GroupsMapping.md | 3 + .../markdown/release/3.0.1/CHANGES.3.0.1.md | 241 +++++++ .../release/3.0.1/RELEASENOTES.3.0.1.md | 54 ++ .../apache/hadoop/conf/TestConfiguration.java | 34 +- .../apache/hadoop/io/TestDataByteBuffers.java | 196 ++---- .../hadoop/metrics2/util/DummyMXBean.java | 2 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 38 +- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 1 + .../federation/metrics/FederationMetrics.java | 2 +- .../federation/metrics/NamenodeBeanMetrics.java | 68 +- .../MultipleDestinationMountTableResolver.java | 3 + .../resolver/order/AvailableSpaceResolver.java | 179 +++++ .../resolver/order/DestinationOrder.java | 3 +- .../resolver/order/LocalResolver.java | 147 +---- .../resolver/order/RouterResolver.java | 160 +++++ .../hdfs/server/federation/router/Router.java | 13 + .../federation/router/RouterHttpServer.java | 4 + .../federation/router/RouterMetricsService.java | 9 + .../federation/router/RouterRpcServer.java | 41 +- .../federation/router/RouterWebHdfsMethods.java | 655 +++++++++++++++++++ .../federation/store/records/MountTable.java | 3 +- .../store/records/impl/pb/MountTablePBImpl.java | 4 + .../src/main/proto/FederationProtocol.proto | 1 + .../main/webapps/router/federationhealth.html | 42 +- .../src/main/webapps/router/federationhealth.js | 35 +- .../src/main/webapps/static/rbf.css | 138 ++++ .../src/main/xsl/configuration.xsl | 37 ++ .../router/web/RouterWebHDFSContract.java | 129 ++++ .../web/TestRouterWebHDFSContractAppend.java | 45 ++ .../web/TestRouterWebHDFSContractConcat.java | 52 ++ .../web/TestRouterWebHDFSContractCreate.java | 49 ++ .../web/TestRouterWebHDFSContractDelete.java | 49 ++ .../web/TestRouterWebHDFSContractMkdir.java | 48 ++ .../web/TestRouterWebHDFSContractOpen.java | 63 ++ .../web/TestRouterWebHDFSContractRename.java | 49 ++ .../TestRouterWebHDFSContractRootDirectory.java | 64 ++ .../web/TestRouterWebHDFSContractSeek.java | 63 ++ .../fs/contract/router/web/package-info.java | 22 + .../server/federation/MiniRouterDFSCluster.java | 5 + .../order/TestAvailableSpaceResolver.java | 232 +++++++ .../router/TestConnectionManager.java | 22 +- .../router/TestRouterAllResolver.java | 10 +- .../server/federation/router/TestRouterRpc.java | 41 +- .../src/test/resources/contract/webhdfs.xml | 26 + .../jdiff/Apache_Hadoop_HDFS_3.0.1.xml | 324 +++++++++ .../hadoop/hdfs/server/namenode/AclStorage.java | 3 +- .../server/namenode/EncryptionZoneManager.java | 74 ++- .../server/namenode/FSDirEncryptionZoneOp.java | 15 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 8 +- .../hadoop/hdfs/server/namenode/FSImage.java | 29 +- .../server/namenode/FSImageFormatProtobuf.java | 31 +- .../server/namenode/NameNodeHttpServer.java | 27 +- .../hdfs/server/namenode/XAttrStorage.java | 9 +- .../snapshot/FSImageFormatPBSnapshot.java | 55 +- .../web/resources/NamenodeWebHdfsMethods.java | 41 +- .../org/apache/hadoop/TestRefreshCallQueue.java | 17 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 15 +- .../mapred/LocalDistributedCacheManager.java | 13 +- .../apache/hadoop/mapred/LocalJobRunner.java | 2 +- .../TestLocalDistributedCacheManager.java | 95 ++- .../org/apache/hadoop/mapreduce/TestTaskID.java | 21 +- hadoop-project-dist/pom.xml | 2 +- hadoop-project/pom.xml | 2 +- .../aliyun/oss/AliyunCredentialsProvider.java | 3 +- .../aliyun/oss/AliyunOSSBlockOutputStream.java | 206 ++++++ .../fs/aliyun/oss/AliyunOSSFileSystem.java | 34 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 173 ++--- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 111 ---- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 115 +++- .../apache/hadoop/fs/aliyun/oss/Constants.java | 22 +- .../oss/TestAliyunOSSBlockOutputStream.java | 115 ++++ .../fs/aliyun/oss/TestAliyunOSSInputStream.java | 10 +- .../aliyun/oss/TestAliyunOSSOutputStream.java | 91 --- .../contract/TestAliyunOSSContractDistCp.java | 2 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 40 -- .../hadoop/fs/azure/NativeAzureFileSystem.java | 46 -- ...TestNativeAzureFileSystemBlockLocations.java | 141 ---- .../hadoop/tools/rumen/state/StatePool.java | 8 +- .../hadoop/tools/rumen/TestHistograms.java | 3 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 + .../yarn/api/resource/PlacementConstraint.java | 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 27 +- .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../distributedshell/ApplicationMaster.java | 1 - .../applications/distributedshell/Client.java | 15 + .../DistributedShellTimelinePlugin.java | 2 +- .../distributedshell/TestDistributedShell.java | 9 + .../yarn/service/client/ApiServiceClient.java | 18 + .../hadoop/yarn/service/webapp/ApiServer.java | 24 + ...RN-Simplified-V1-API-Layer-For-Services.yaml | 91 ++- .../hadoop/yarn/service/ClientAMProtocol.java | 12 + .../hadoop/yarn/service/ClientAMService.java | 24 + .../hadoop/yarn/service/ServiceEvent.java | 49 ++ .../hadoop/yarn/service/ServiceEventType.java | 28 + .../hadoop/yarn/service/ServiceManager.java | 225 +++++++ .../hadoop/yarn/service/ServiceScheduler.java | 47 +- .../yarn/service/UpgradeComponentsFinder.java | 157 +++++ .../yarn/service/api/records/Component.java | 7 +- .../service/api/records/ComponentState.java | 2 +- .../yarn/service/api/records/ConfigFile.java | 2 +- .../api/records/PlacementConstraint.java | 283 ++++++++ .../service/api/records/PlacementPolicy.java | 52 +- .../service/api/records/PlacementScope.java | 56 ++ .../yarn/service/api/records/PlacementType.java | 38 ++ .../yarn/service/api/records/Resource.java | 3 +- .../yarn/service/api/records/Service.java | 27 - .../yarn/service/api/records/ServiceState.java | 2 +- .../yarn/service/api/records/ServiceStatus.java | 4 +- .../yarn/service/client/ServiceClient.java | 153 ++++- .../yarn/service/component/Component.java | 143 +++- .../yarn/service/component/ComponentEvent.java | 12 + .../service/component/ComponentEventType.java | 4 +- .../yarn/service/component/ComponentState.java | 3 +- .../yarn/service/conf/YarnServiceConf.java | 2 + .../yarn/service/conf/YarnServiceConstants.java | 2 + .../containerlaunch/AbstractLauncher.java | 7 +- .../exceptions/RestApiErrorMessages.java | 12 + .../pb/client/ClientAMProtocolPBClientImpl.java | 26 + .../service/ClientAMProtocolPBServiceImpl.java | 24 + .../provider/AbstractProviderService.java | 5 +- .../ServiceTimelinePublisher.java | 4 - .../yarn/service/utils/CoreFileSystem.java | 31 +- .../yarn/service/utils/ServiceApiUtil.java | 56 +- .../src/main/proto/ClientAMProtocol.proto | 19 +- .../hadoop/yarn/service/ServiceTestUtils.java | 90 ++- .../TestDefaultUpgradeComponentsFinder.java | 63 ++ .../hadoop/yarn/service/TestServiceApiUtil.java | 35 + .../hadoop/yarn/service/TestServiceManager.java | 156 +++++ .../yarn/service/TestYarnNativeServices.java | 199 +++++- .../yarn/service/client/TestServiceClient.java | 125 ++++ .../TestServiceTimelinePublisher.java | 10 +- .../yarn/service/utils/TestCoreFileSystem.java | 46 ++ .../hadoop/yarn/client/api/AppAdminClient.java | 16 + .../TestFederationRMFailoverProxyProvider.java | 81 ++- .../impl/pb/SchedulingRequestPBImpl.java | 1 + .../api/impl/FileSystemTimelineWriter.java | 4 +- .../src/main/resources/yarn-default.xml | 22 + .../registry/server/dns/TestRegistryDNS.java | 66 +- .../test/resources/0.17.172.in-addr.arpa.zone | 24 +- .../FederationRMFailoverProxyProvider.java | 11 +- .../nodemanager/LinuxContainerExecutor.java | 5 +- .../localizer/LocalizedResource.java | 4 +- .../monitor/ContainersMonitorImpl.java | 12 +- .../webapp/TestContainerLogsPage.java | 1 + .../resourcemanager/RMActiveServiceContext.java | 11 + .../server/resourcemanager/RMContextImpl.java | 6 +- .../server/resourcemanager/ResourceManager.java | 33 +- .../placement/AppNameMappingPlacementRule.java | 197 ++++++ .../placement/PlacementRule.java | 9 +- .../placement/QueueMappingEntity.java | 72 ++ .../resourcemanager/placement/QueuePath.java | 53 ++ .../placement/QueuePlacementRuleUtils.java | 135 ++++ .../UserGroupMappingPlacementRule.java | 18 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 5 + .../scheduler/capacity/CapacityScheduler.java | 85 ++- .../CapacitySchedulerConfiguration.java | 49 ++ .../conf/FSSchedulerConfigurationStore.java | 303 +++++++++ .../conf/MutableCSConfigurationProvider.java | 3 + .../capacity/conf/YarnConfigurationStore.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 16 +- .../MemoryPlacementConstraintManager.java | 12 +- .../webapp/RMWebServiceProtocol.java | 3 +- .../resourcemanager/webapp/RMWebServices.java | 40 +- .../webapp/dao/CapacitySchedulerHealthInfo.java | 35 +- .../webapp/dao/RMQueueAclInfo.java | 65 ++ .../yarn/server/resourcemanager/TestRMHA.java | 39 ++ .../TestAppNameMappingPlacementRule.java | 93 +++ .../placement/TestPlacementManager.java | 112 ++++ .../capacity/TestCapacityScheduler.java | 53 ++ ...estCapacitySchedulerQueueMappingFactory.java | 128 ++++ .../conf/TestFSSchedulerConfigurationStore.java | 173 +++++ .../TestMutableCSConfigurationProvider.java | 50 ++ .../webapp/TestRMWebServices.java | 52 +- .../webapp/TestRMWebServicesCapacitySched.java | 4 + .../webapp/DefaultRequestInterceptorREST.java | 5 +- .../webapp/FederationInterceptorREST.java | 3 +- .../server/router/webapp/RouterWebServices.java | 3 +- .../webapp/MockRESTRequestInterceptor.java | 5 +- .../PassThroughRESTRequestInterceptor.java | 3 +- .../hadoop/yarn/server/timeline/LogInfo.java | 3 +- .../server/timeline/PluginStoreTestUtils.java | 3 +- .../yarn/server/timeline/TestLogInfo.java | 6 +- .../src/site/markdown/CapacityScheduler.md | 33 +- .../src/site/markdown/ResourceManagerRest.md | 132 ++++ .../src/site/markdown/TimelineServiceV2.md | 2 + .../markdown/yarn-service/Configurations.md | 5 +- .../markdown/yarn-service/YarnServiceAPI.md | 270 +++++++- .../serializers/yarn-queue/capacity-queue.js | 4 +- 198 files changed, 8711 insertions(+), 1639 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7de7daa5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7de7daa5/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7de7daa5/hadoop-project/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org