From common-commits-return-85429-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Thu Jul 12 01:10:15 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 6F6C518062A for ; Thu, 12 Jul 2018 01:10:13 +0200 (CEST) Received: (qmail 1686 invoked by uid 500); 11 Jul 2018 23:10:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 99743 invoked by uid 99); 11 Jul 2018 23:10:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Jul 2018 23:10:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC1D9DFA59; Wed, 11 Jul 2018 23:10:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shv@apache.org To: common-commits@hadoop.apache.org Date: Wed, 11 Jul 2018 23:10:31 -0000 Message-Id: <770fdc8ee9714463ab6c099c391b1935@git.apache.org> In-Reply-To: <320516e702f945de9de2c3795be1ebee@git.apache.org> References: <320516e702f945de9de2c3795be1ebee@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [31/56] [abbrv] hadoop git commit: Merge trunk into HDDS-48 Merge trunk into HDDS-48 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c275a9a6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c275a9a6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c275a9a6 Branch: refs/heads/HDFS-12943 Commit: c275a9a6a07b2bd889bdba4d05b420027f430b34 Parents: 44e19fc 83cd84b Author: Bharat Viswanadham Authored: Mon Jul 9 12:13:03 2018 -0700 Committer: Bharat Viswanadham Committed: Mon Jul 9 12:13:03 2018 -0700 ---------------------------------------------------------------------- .gitignore | 4 + dev-support/bin/ozone-dist-layout-stitching | 2 +- ...ExcludePrivateAnnotationsStandardDoclet.java | 6 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 6 +- .../org/apache/hadoop/conf/Configuration.java | 458 +++--- .../java/org/apache/hadoop/fs/FileContext.java | 9 +- .../org/apache/hadoop/fs/LocalDirAllocator.java | 7 +- .../hadoop-common/src/site/markdown/Metrics.md | 39 +- .../org/apache/hadoop/fs/TestFileContext.java | 44 +- .../apache/hadoop/fs/TestLocalDirAllocator.java | 59 + .../src/main/compose/ozone/docker-compose.yaml | 6 +- .../src/main/compose/ozone/docker-config | 2 +- .../src/main/compose/ozoneperf/README.md | 4 +- .../main/compose/ozoneperf/docker-compose.yaml | 6 +- .../src/main/compose/ozoneperf/docker-config | 2 +- .../scm/client/ContainerOperationClient.java | 117 +- hadoop-hdds/common/pom.xml | 18 + .../hadoop/hdds/protocol/DatanodeDetails.java | 13 +- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +- .../hadoop/hdds/scm/client/ScmClient.java | 43 +- .../container/common/helpers/ContainerInfo.java | 167 ++- .../common/helpers/ContainerWithPipeline.java | 131 ++ .../StorageContainerLocationProtocol.java | 18 +- ...rLocationProtocolClientSideTranslatorPB.java | 34 +- .../org/apache/hadoop/ozone/OzoneConsts.java | 22 +- .../apache/hadoop/ozone/audit/AuditAction.java | 30 + .../hadoop/ozone/audit/AuditEventStatus.java | 36 + .../apache/hadoop/ozone/audit/AuditLogger.java | 128 ++ .../hadoop/ozone/audit/AuditLoggerType.java | 37 + .../apache/hadoop/ozone/audit/AuditMarker.java | 38 + .../apache/hadoop/ozone/audit/Auditable.java | 32 + .../apache/hadoop/ozone/audit/package-info.java | 123 ++ .../org/apache/hadoop/ozone/common/Storage.java | 6 +- ...rLocationProtocolServerSideTranslatorPB.java | 33 +- .../main/proto/ScmBlockLocationProtocol.proto | 10 +- .../StorageContainerLocationProtocol.proto | 34 +- hadoop-hdds/common/src/main/proto/hdds.proto | 28 +- .../common/src/main/resources/ozone-default.xml | 131 +- .../apache/hadoop/ozone/audit/DummyAction.java | 51 + .../apache/hadoop/ozone/audit/DummyEntity.java | 57 + .../ozone/audit/TestOzoneAuditLogger.java | 147 ++ .../apache/hadoop/ozone/audit/package-info.java | 23 + .../common/src/test/resources/log4j2.properties | 76 + .../apache/hadoop/hdds/scm/HddsServerUtil.java | 11 - .../DeleteBlocksCommandHandler.java | 30 +- .../protocol/StorageContainerNodeProtocol.java | 4 +- .../src/main/resources/webapps/static/ozone.js | 4 +- .../webapps/static/templates/config.html | 4 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 80 +- .../block/DatanodeDeletedBlockTransactions.java | 11 +- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 2 +- .../container/CloseContainerEventHandler.java | 35 +- .../hdds/scm/container/ContainerMapping.java | 128 +- .../scm/container/ContainerStateManager.java | 30 +- .../hadoop/hdds/scm/container/Mapping.java | 26 +- .../scm/container/closer/ContainerCloser.java | 15 +- .../scm/container/states/ContainerStateMap.java | 13 +- .../hadoop/hdds/scm/events/SCMEvents.java | 80 ++ .../hadoop/hdds/scm/events/package-info.java | 23 + .../hadoop/hdds/scm/node/CommandQueue.java | 2 +- .../hadoop/hdds/scm/node/DatanodeInfo.java | 109 ++ .../hdds/scm/node/HeartbeatQueueItem.java | 98 -- .../hadoop/hdds/scm/node/NodeManager.java | 16 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 575 ++++++++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 511 +------ .../node/states/NodeAlreadyExistsException.java | 45 + .../hdds/scm/node/states/NodeException.java | 44 + .../scm/node/states/NodeNotFoundException.java | 49 + .../hdds/scm/node/states/NodeStateMap.java | 281 ++++ .../hdds/scm/pipelines/PipelineManager.java | 27 +- .../hdds/scm/pipelines/PipelineSelector.java | 16 + .../scm/pipelines/ratis/RatisManagerImpl.java | 1 + .../standalone/StandaloneManagerImpl.java | 1 + .../hdds/scm/server/SCMBlockProtocolServer.java | 2 +- .../scm/server/SCMClientProtocolServer.java | 74 +- .../server/SCMDatanodeHeartbeatDispatcher.java | 13 +- .../scm/server/SCMDatanodeProtocolServer.java | 2 +- .../scm/server/StorageContainerManager.java | 7 +- .../hdds/scm/block/TestDeletedBlockLog.java | 15 +- .../hdds/scm/container/MockNodeManager.java | 58 +- .../TestCloseContainerEventHandler.java | 54 +- .../scm/container/TestContainerMapping.java | 27 +- .../container/closer/TestContainerCloser.java | 18 +- .../hdds/scm/node/TestContainerPlacement.java | 16 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 186 +-- .../TestSCMDatanodeHeartbeatDispatcher.java | 20 +- .../testutils/ReplicationNodeManagerMock.java | 37 +- .../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 2 +- .../cli/container/CloseContainerHandler.java | 10 +- .../cli/container/DeleteContainerHandler.java | 9 +- .../scm/cli/container/InfoContainerHandler.java | 11 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 19 - .../org/apache/hadoop/hdfs/DFSInputStream.java | 46 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 15 +- .../hdfs/client/HdfsClientConfigKeys.java | 3 + .../hdfs/client/impl/BlockReaderFactory.java | 21 +- .../hdfs/client/impl/BlockReaderLocal.java | 93 +- .../client/impl/BlockReaderLocalLegacy.java | 44 +- .../hdfs/client/impl/BlockReaderRemote.java | 33 +- .../datanode/ReplicaNotFoundException.java | 2 +- .../ha/ConfiguredFailoverProxyProvider.java | 9 +- .../InMemoryAliasMapFailoverProxyProvider.java | 38 + .../hdfs/server/federation/router/Quota.java | 10 +- .../router/RouterQuotaUpdateService.java | 43 +- .../federation/router/RouterRpcServer.java | 1 - .../router/TestDisableRouterQuota.java | 94 ++ .../federation/router/TestRouterQuota.java | 212 ++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 37 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 15 +- ...yAliasMapProtocolClientSideTranslatorPB.java | 95 +- .../aliasmap/InMemoryAliasMapProtocol.java | 5 + .../aliasmap/InMemoryLevelDBAliasMapServer.java | 19 +- .../impl/InMemoryLevelDBAliasMapClient.java | 80 +- .../impl/TextFileRegionAliasMap.java | 5 +- .../hadoop/hdfs/server/datanode/DataNode.java | 21 +- .../hdfs/server/datanode/DiskBalancer.java | 29 +- .../erasurecode/StripedBlockReader.java | 2 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 8 + .../hdfs/server/namenode/NamenodeFsck.java | 1 - .../src/main/resources/hdfs-default.xml | 35 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 13 +- .../apache/hadoop/hdfs/MiniDFSNNTopology.java | 2 +- .../hdfs/client/impl/BlockReaderTestUtil.java | 2 - .../hdfs/client/impl/TestBlockReaderLocal.java | 2 - .../blockmanagement/TestBlockTokenWithDFS.java | 2 - .../TestNameNodePrunesMissingStorages.java | 5 +- .../impl/TestInMemoryLevelDBAliasMapClient.java | 7 + .../datanode/TestDataNodeVolumeFailure.java | 2 - .../server/diskbalancer/TestDiskBalancer.java | 80 +- .../shortcircuit/TestShortCircuitCache.java | 89 ++ .../src/test/acceptance/basic/basic.robot | 6 +- .../test/acceptance/basic/docker-compose.yaml | 8 +- .../src/test/acceptance/basic/docker-config | 4 +- .../src/test/acceptance/basic/ozone-shell.robot | 18 +- .../src/test/acceptance/commonlib.robot | 4 +- .../test/acceptance/ozonefs/docker-compose.yaml | 8 +- .../src/test/acceptance/ozonefs/docker-config | 4 +- .../src/test/acceptance/ozonefs/ozonefs.robot | 6 +- .../apache/hadoop/ozone/client/BucketArgs.java | 4 +- .../hadoop/ozone/client/OzoneClientFactory.java | 89 +- .../apache/hadoop/ozone/client/OzoneKey.java | 2 +- .../apache/hadoop/ozone/client/VolumeArgs.java | 4 +- .../ozone/client/io/ChunkGroupInputStream.java | 33 +- .../ozone/client/io/ChunkGroupOutputStream.java | 63 +- .../client/rest/DefaultRestServerSelector.java | 2 +- .../hadoop/ozone/client/rest/RestClient.java | 15 +- .../ozone/client/rest/RestServerSelector.java | 2 +- .../hadoop/ozone/client/rpc/RpcClient.java | 142 +- .../ozone/client/TestHddsClientUtils.java | 24 +- hadoop-ozone/common/pom.xml | 2 +- hadoop-ozone/common/src/main/bin/ozone | 9 +- hadoop-ozone/common/src/main/bin/start-ozone.sh | 16 +- hadoop-ozone/common/src/main/bin/stop-ozone.sh | 16 +- .../java/org/apache/hadoop/ozone/KsmUtils.java | 87 -- .../java/org/apache/hadoop/ozone/OmUtils.java | 94 ++ .../org/apache/hadoop/ozone/audit/OMAction.java | 51 + .../apache/hadoop/ozone/audit/package-info.java | 22 + .../apache/hadoop/ozone/freon/OzoneGetConf.java | 16 +- .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 -- .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 --- .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 --- .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 -- .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 277 ---- .../ozone/ksm/helpers/KsmKeyLocationInfo.java | 129 -- .../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 -- .../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 -- .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 --- .../ozone/ksm/helpers/OpenKeySession.java | 50 - .../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 --- .../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 -- .../hadoop/ozone/ksm/helpers/package-info.java | 18 - .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../ksm/protocol/KeySpaceManagerProtocol.java | 252 ---- .../hadoop/ozone/ksm/protocol/package-info.java | 19 - ...ceManagerProtocolClientSideTranslatorPB.java | 769 ---------- .../protocolPB/KeySpaceManagerProtocolPB.java | 34 - .../ozone/ksm/protocolPB/package-info.java | 19 - .../apache/hadoop/ozone/om/OMConfigKeys.java | 81 ++ .../hadoop/ozone/om/helpers/OmBucketArgs.java | 233 +++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 235 +++ .../hadoop/ozone/om/helpers/OmKeyArgs.java | 119 ++ .../hadoop/ozone/om/helpers/OmKeyInfo.java | 277 ++++ .../ozone/om/helpers/OmKeyLocationInfo.java | 129 ++ .../om/helpers/OmKeyLocationInfoGroup.java | 118 ++ .../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 110 ++ .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 223 +++ .../hadoop/ozone/om/helpers/OpenKeySession.java | 50 + .../hadoop/ozone/om/helpers/ServiceInfo.java | 237 +++ .../hadoop/ozone/om/helpers/VolumeArgs.java | 140 ++ .../hadoop/ozone/om/helpers/package-info.java | 18 + .../apache/hadoop/ozone/om/package-info.java | 21 + .../ozone/om/protocol/OzoneManagerProtocol.java | 252 ++++ .../hadoop/ozone/om/protocol/package-info.java | 19 + ...neManagerProtocolClientSideTranslatorPB.java | 769 ++++++++++ .../om/protocolPB/OzoneManagerProtocolPB.java | 34 + .../ozone/om/protocolPB/package-info.java | 19 + .../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 -- .../hadoop/ozone/protocolPB/OMPBHelper.java | 113 ++ .../hadoop/ozone/protocolPB/OzonePBHelper.java | 30 + .../main/proto/KeySpaceManagerProtocol.proto | 474 ------ .../src/main/proto/OzoneManagerProtocol.proto | 480 +++++++ hadoop-ozone/docs/content/GettingStarted.md | 18 +- hadoop-ozone/docs/content/Metrics.md | 10 +- hadoop-ozone/docs/content/_index.md | 12 +- hadoop-ozone/docs/static/OzoneOverview.svg | 2 +- .../container/TestContainerStateManager.java | 161 ++- .../apache/hadoop/ozone/MiniOzoneCluster.java | 24 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 66 +- .../hadoop/ozone/TestContainerOperations.java | 11 +- .../ozone/TestOzoneConfigurationFields.java | 4 +- .../ozone/TestStorageContainerManager.java | 28 +- .../TestStorageContainerManagerHelper.java | 22 +- .../ozone/client/rest/TestOzoneRestClient.java | 6 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 22 +- .../commandhandler/TestBlockDeletion.java | 212 +++ .../TestCloseContainerByPipeline.java | 97 +- .../TestCloseContainerHandler.java | 14 +- .../ozone/ksm/TestContainerReportWithKeys.java | 143 -- .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 306 ---- .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 284 ---- .../hadoop/ozone/ksm/TestKeySpaceManager.java | 1350 ------------------ .../ksm/TestKeySpaceManagerRestInterface.java | 135 -- .../ozone/ksm/TestKsmBlockVersioning.java | 253 ---- .../ksm/TestMultipleContainerReadWrite.java | 215 --- .../ozone/om/TestContainerReportWithKeys.java | 143 ++ .../om/TestMultipleContainerReadWrite.java | 215 +++ .../hadoop/ozone/om/TestOmBlockVersioning.java | 253 ++++ .../apache/hadoop/ozone/om/TestOmMetrics.java | 313 ++++ .../apache/hadoop/ozone/om/TestOmSQLCli.java | 284 ++++ .../hadoop/ozone/om/TestOzoneManager.java | 1349 +++++++++++++++++ .../ozone/om/TestOzoneManagerRestInterface.java | 135 ++ .../hadoop/ozone/ozShell/TestOzoneShell.java | 14 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 6 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 3 +- .../ozone/scm/TestContainerSmallFile.java | 36 +- .../org/apache/hadoop/ozone/scm/TestSCMCli.java | 127 +- .../ozone/scm/TestXceiverClientManager.java | 62 +- .../ozone/scm/TestXceiverClientMetrics.java | 14 +- .../hadoop/ozone/scm/node/TestQueryNode.java | 19 +- .../ozone/web/TestDistributedOzoneVolumes.java | 12 +- .../hadoop/ozone/web/client/TestKeys.java | 58 +- .../src/test/resources/webapps/ksm/.gitkeep | 15 - .../resources/webapps/ozoneManager/.gitkeep | 15 + .../server/datanode/ObjectStoreHandler.java | 33 +- .../ozone/web/handlers/KeyProcessTemplate.java | 4 +- .../web/handlers/VolumeProcessTemplate.java | 4 +- .../web/storage/DistributedStorageHandler.java | 153 +- .../apache/hadoop/ozone/ksm/BucketManager.java | 79 - .../hadoop/ozone/ksm/BucketManagerImpl.java | 315 ---- .../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 - .../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ---- .../ozone/ksm/KSMMetadataManagerImpl.java | 526 ------- .../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 459 ------ .../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 -- .../hadoop/ozone/ksm/KeyDeletingService.java | 142 -- .../org/apache/hadoop/ozone/ksm/KeyManager.java | 175 --- .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 566 -------- .../hadoop/ozone/ksm/KeySpaceManager.java | 914 ------------ .../ozone/ksm/KeySpaceManagerHttpServer.java | 78 - .../hadoop/ozone/ksm/OpenKeyCleanupService.java | 117 -- .../ozone/ksm/ServiceListJSONServlet.java | 103 -- .../apache/hadoop/ozone/ksm/VolumeManager.java | 100 -- .../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ----- .../ozone/ksm/exceptions/KSMException.java | 118 -- .../ozone/ksm/exceptions/package-info.java | 19 - .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../apache/hadoop/ozone/om/BucketManager.java | 79 + .../hadoop/ozone/om/BucketManagerImpl.java | 315 ++++ .../hadoop/ozone/om/KeyDeletingService.java | 142 ++ .../org/apache/hadoop/ozone/om/KeyManager.java | 175 +++ .../apache/hadoop/ozone/om/KeyManagerImpl.java | 566 ++++++++ .../org/apache/hadoop/ozone/om/OMMXBean.java | 31 + .../hadoop/ozone/om/OMMetadataManager.java | 253 ++++ .../org/apache/hadoop/ozone/om/OMMetrics.java | 459 ++++++ .../org/apache/hadoop/ozone/om/OMStorage.java | 90 ++ .../hadoop/ozone/om/OmMetadataManagerImpl.java | 526 +++++++ .../hadoop/ozone/om/OpenKeyCleanupService.java | 117 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 911 ++++++++++++ .../hadoop/ozone/om/OzoneManagerHttpServer.java | 78 + .../hadoop/ozone/om/ServiceListJSONServlet.java | 103 ++ .../apache/hadoop/ozone/om/VolumeManager.java | 100 ++ .../hadoop/ozone/om/VolumeManagerImpl.java | 390 +++++ .../hadoop/ozone/om/exceptions/OMException.java | 118 ++ .../ozone/om/exceptions/package-info.java | 19 + .../apache/hadoop/ozone/om/package-info.java | 21 + ...ceManagerProtocolServerSideTranslatorPB.java | 559 -------- ...neManagerProtocolServerSideTranslatorPB.java | 571 ++++++++ .../hadoop/ozone/protocolPB/package-info.java | 2 +- .../src/main/webapps/ksm/index.html | 70 - .../src/main/webapps/ksm/ksm-metrics.html | 44 - .../ozone-manager/src/main/webapps/ksm/ksm.js | 110 -- .../ozone-manager/src/main/webapps/ksm/main.css | 23 - .../src/main/webapps/ksm/main.html | 18 - .../src/main/webapps/ozoneManager/index.html | 70 + .../src/main/webapps/ozoneManager/main.css | 23 + .../src/main/webapps/ozoneManager/main.html | 18 + .../main/webapps/ozoneManager/om-metrics.html | 44 + .../main/webapps/ozoneManager/ozoneManager.js | 110 ++ .../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ----- .../hadoop/ozone/ksm/TestChunkStreams.java | 234 --- .../ksm/TestKeySpaceManagerHttpServer.java | 141 -- .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../hadoop/ozone/om/TestBucketManagerImpl.java | 394 +++++ .../hadoop/ozone/om/TestChunkStreams.java | 234 +++ .../ozone/om/TestOzoneManagerHttpServer.java | 141 ++ .../apache/hadoop/ozone/om/package-info.java | 21 + .../hadoop/fs/ozone/contract/OzoneContract.java | 4 +- .../genesis/BenchMarkContainerStateMap.java | 16 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 111 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 10 + .../s3guard/AbstractS3GuardToolTestBase.java | 18 + .../namenode/ITestProvidedImplementation.java | 373 ++++- .../dev-support/findbugs-exclude.xml | 17 +- .../hadoop/yarn/api/records/Resource.java | 13 + .../api/records/impl/LightWeightResource.java | 23 +- .../hadoop/yarn/conf/YarnConfiguration.java | 7 + .../impl/pb/GetApplicationsRequestPBImpl.java | 44 +- .../logaggregation/AggregatedLogFormat.java | 6 +- .../timeline/RollingLevelDBTimelineStore.java | 6 + .../server/timeline/TimelineDataManager.java | 7 +- .../timeline/webapp/TimelineWebServices.java | 4 + .../webapp/TestTimelineWebServices.java | 2 +- .../amrmproxy/BroadcastAMRMProxyPolicy.java | 11 - .../amrmproxy/RejectAMRMProxyPolicy.java | 4 - .../TestBroadcastAMRMProxyFederationPolicy.java | 11 +- .../yarn/server/nodemanager/NodeManager.java | 66 +- .../runtime/DockerLinuxContainerRuntime.java | 4 +- .../runtime/ContainerExecutionException.java | 6 + .../impl/container-executor.c | 30 +- .../container-executor/impl/utils/docker-util.c | 2 +- .../test/test-container-executor.c | 20 + .../nodemanager/TestNodeManagerResync.java | 56 + .../runtime/TestDockerContainerRuntime.java | 10 +- .../conf/capacity-scheduler.xml | 10 + .../scheduler/capacity/CapacityScheduler.java | 45 +- .../CapacitySchedulerConfiguration.java | 10 + .../scheduler/capacity/ParentQueue.java | 36 +- .../allocator/AbstractContainerAllocator.java | 13 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 5 + .../scheduler/fair/ConfigurableResource.java | 69 +- .../fair/FairSchedulerConfiguration.java | 174 ++- .../allocation/AllocationFileQueueParser.java | 2 +- .../resourcemanager/webapp/dao/AppInfo.java | 2 +- .../webapp/dao/SchedulerInfo.java | 8 +- .../TestWorkPreservingRMRestart.java | 2 + .../fair/TestFairSchedulerConfiguration.java | 160 ++- .../webapp/TestRMWebServices.java | 31 +- .../webapp/TestRMWebServicesApps.java | 14 +- ...estRMWebServicesAppsCustomResourceTypes.java | 242 ++++ .../webapp/TestRMWebServicesCapacitySched.java | 30 +- .../TestRMWebServicesConfigurationMutation.java | 5 + .../webapp/TestRMWebServicesFairScheduler.java | 95 +- .../TestRMWebServicesSchedulerActivities.java | 2 +- ...ustomResourceTypesConfigurationProvider.java | 138 ++ .../FairSchedulerJsonVerifications.java | 139 ++ .../FairSchedulerXmlVerifications.java | 153 ++ ...ervicesFairSchedulerCustomResourceTypes.java | 271 ++++ .../webapp/helper/AppInfoJsonVerifications.java | 123 ++ .../webapp/helper/AppInfoXmlVerifications.java | 132 ++ .../webapp/helper/BufferedClientResponse.java | 57 + .../helper/JsonCustomResourceTypeTestcase.java | 77 + .../ResourceRequestsJsonVerifications.java | 252 ++++ .../ResourceRequestsXmlVerifications.java | 215 +++ .../helper/XmlCustomResourceTypeTestCase.java | 112 ++ .../router/clientrm/RouterClientRMService.java | 53 +- .../router/rmadmin/RouterRMAdminService.java | 51 +- .../server/router/webapp/RouterWebServices.java | 48 +- .../clientrm/TestRouterClientRMService.java | 60 + .../rmadmin/TestRouterRMAdminService.java | 60 + .../router/webapp/TestRouterWebServices.java | 65 + .../pom.xml | 10 + .../storage/TestTimelineReaderHBaseDown.java | 220 +++ .../storage/HBaseTimelineReaderImpl.java | 93 ++ .../reader/TimelineFromIdConverter.java | 93 ++ .../reader/TimelineReaderWebServices.java | 198 ++- .../TestTimelineReaderWebServicesBasicAcl.java | 154 ++ .../src/site/markdown/FairScheduler.md | 6 +- .../src/main/webapp/app/initializers/loader.js | 10 +- 379 files changed, 22363 insertions(+), 15606 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 82d67b7,4fad5d8..0db5993 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@@ -98,11 -93,8 +98,11 @@@ public final class OzoneConsts public static final String BLOCK_DB = "block.db"; public static final String OPEN_CONTAINERS_DB = "openContainers.db"; public static final String DELETED_BLOCK_DB = "deletedBlock.db"; - public static final String KSM_DB_NAME = "ksm.db"; + public static final String OM_DB_NAME = "om.db"; + public static final String STORAGE_DIR_CHUNKS = "chunks"; + public static final String CONTAINER_FILE_CHECKSUM_EXTENSION = ".chksm"; + /** * Supports Bucket Versioning. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java index 4fc1cd9,d215da9..c3d1596 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java @@@ -31,13 -29,11 +31,12 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; import org.apache.hadoop.ozone.container.common.helpers .DeletedContainerBlocksSummary; -import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; -import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager; +import org.apache.hadoop.ozone.container.common.interfaces.Container; - import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.statemachine .EndpointStateMachine; import org.apache.hadoop.ozone.container.common.statemachine @@@ -167,21 -145,28 +166,28 @@@ public class DeleteBlocksCommandHandle * Move a bunch of blocks from a container to deleting state. * This is a meta update, the actual deletes happen in async mode. * + * @param containerData - KeyValueContainerData * @param delTX a block deletion transaction. - * @param config configuration. * @throws IOException if I/O error occurs. */ - private void deleteContainerBlocks(DeletedBlocksTransaction delTX, - Configuration config) throws IOException { + private void deleteKeyValueContainerBlocks( + KeyValueContainerData containerData, DeletedBlocksTransaction delTX) + throws IOException { long containerId = delTX.getContainerID(); - ContainerData containerInfo = containerManager.readContainer(containerId); if (LOG.isDebugEnabled()) { LOG.debug("Processing Container : {}, DB path : {}", containerId, - containerInfo.getDBPath()); + containerData.getMetadataPath()); } - if (delTX.getTxID() < containerInfo.getDeleteTransactionId()) { ++ if (delTX.getTxID() < containerData.getDeleteTransactionId()) { + LOG.debug(String.format("Ignoring delete blocks for containerId: %d." + + " Outdated delete transactionId %d < %d", containerId, - delTX.getTxID(), containerInfo.getDeleteTransactionId())); ++ delTX.getTxID(), containerData.getDeleteTransactionId())); + return; + } + int newDeletionBlocks = 0; - MetadataStore containerDB = KeyUtils.getDB(containerInfo, config); + MetadataStore containerDB = KeyUtils.getDB(containerData, conf); for (Long blk : delTX.getLocalIDList()) { BatchOperation batch = new BatchOperation(); byte[] blkBytes = Longs.toByteArray(blk); @@@ -208,13 -203,15 +224,15 @@@ LOG.debug("Block {} not found or already under deletion in" + " container {}, skip deleting it.", blk, containerId); } - containerDB.put(DFSUtil.string2Bytes( - OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + containerId), - Longs.toByteArray(delTX.getTxID())); } + containerDB.put(DFSUtil.string2Bytes( + OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()), + Longs.toByteArray(delTX.getTxID())); - containerManager - .updateDeleteTransactionId(delTX.getContainerID(), delTX.getTxID()); ++ containerData ++ .updateDeleteTransactionId(delTX.getTxID()); // update pending deletion blocks count in in-memory container status - containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId); + containerData.incrPendingDeletionBlocks(newDeletionBlocks); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java index ad1e706,a30c6f4..fff8611 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java @@@ -27,13 -27,11 +27,13 @@@ import org.apache.hadoop.hdds.scm.conta import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; -import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; + import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; + import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.web.handlers.BucketArgs; import org.apache.hadoop.ozone.web.handlers.KeyArgs; import org.apache.hadoop.ozone.web.handlers.UserArgs; @@@ -160,14 -158,15 +160,16 @@@ public class TestStorageContainerManage private MetadataStore getContainerMetadata(Long containerID) throws IOException { - ContainerInfo container = cluster.getStorageContainerManager() - .getClientProtocolServer().getContainer(containerID); - DatanodeDetails leadDN = container.getPipeline().getLeader(); + ContainerWithPipeline containerWithPipeline = cluster + .getStorageContainerManager().getClientProtocolServer() + .getContainerWithPipeline(containerID); + + DatanodeDetails leadDN = containerWithPipeline.getPipeline().getLeader(); OzoneContainer containerServer = getContainerServerByDatanodeUuid(leadDN.getUuidString()); - ContainerData containerData = containerServer.getContainerManager() - .readContainer(containerID); + KeyValueContainerData containerData = (KeyValueContainerData) containerServer + .getContainerSet() + .getContainer(containerID).getContainerData(); return KeyUtils.getDB(containerData, conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java index b832dd2,58b831b..30b18c2 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java @@@ -32,10 -32,10 +32,10 @@@ import org.apache.hadoop.ozone.client.O import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo; + import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; + import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; @@@ -204,14 -257,8 +257,8 @@@ public class TestCloseContainerByPipeli if (datanode.equals(datanodeService.getDatanodeDetails())) { containerData = datanodeService.getDatanodeStateMachine().getContainer() - .getContainerManager().readContainer(containerID); + .getContainerSet().getContainer(containerID).getContainerData(); - if (!containerData.isOpen()) { - // make sure the closeContainerHandler on the Datanode is invoked - Assert.assertTrue( - datanodeService.getDatanodeStateMachine().getCommandDispatcher() - .getCloseContainerHandler().getInvocationCount() > 0); - return true; - } + return !containerData.isOpen(); } } catch (StorageContainerException e) { throw new AssertionError(e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java index 114bd04,58a5154..682bd63 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java @@@ -27,9 -27,9 +27,9 @@@ import org.apache.hadoop.hdds.client.Re import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.rest.OzoneException; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo; + import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; + import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java index 0000000,5481506..c25b00e mode 000000,100644..100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java @@@ -1,0 -1,143 +1,143 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package org.apache.hadoop.ozone.om; + + import org.apache.commons.lang3.RandomStringUtils; + + import org.apache.hadoop.hdds.client.ReplicationFactor; + import org.apache.hadoop.hdds.client.ReplicationType; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; + import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; + import org.apache.hadoop.ozone.MiniOzoneCluster; + import org.apache.hadoop.ozone.OzoneConfigKeys; + import org.apache.hadoop.ozone.OzoneConsts; + import org.apache.hadoop.ozone.client.*; + import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; -import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager; ++import org.apache.hadoop.ozone.container.common.impl.ContainerData; ++import org.apache.hadoop.ozone.container.common.impl.ContainerSet; + import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; + import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; + import org.apache.hadoop.hdds.scm.server.StorageContainerManager; + import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; + import org.junit.AfterClass; + import org.junit.BeforeClass; + import org.junit.Rule; + import org.junit.Test; + import org.junit.rules.ExpectedException; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import java.io.IOException; + + /** + * This class tests container report with DN container state info. + */ + public class TestContainerReportWithKeys { + private static final Logger LOG = LoggerFactory.getLogger( + TestContainerReportWithKeys.class); + private static MiniOzoneCluster cluster = null; + private static OzoneConfiguration conf; + private static StorageContainerManager scm; + + @Rule + public ExpectedException exception = ExpectedException.none(); + + /** + * Create a MiniDFSCluster for testing. + *

+ * Ozone is made active by setting OZONE_ENABLED = true and + * OZONE_HANDLER_TYPE_KEY = "distributed" + * + * @throws IOException + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, + OzoneConsts.OZONE_HANDLER_DISTRIBUTED); + cluster = MiniOzoneCluster.newBuilder(conf).build(); + cluster.waitForClusterToBeReady(); + scm = cluster.getStorageContainerManager(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testContainerReportKeyWrite() throws Exception { + final String volumeName = "volume" + RandomStringUtils.randomNumeric(5); + final String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); + final String keyName = "key" + RandomStringUtils.randomNumeric(5); + final int keySize = 100; + + OzoneClient client = OzoneClientFactory.getClient(conf); + ObjectStore objectStore = client.getObjectStore(); + objectStore.createVolume(volumeName); + objectStore.getVolume(volumeName).createBucket(bucketName); + OzoneOutputStream key = + objectStore.getVolume(volumeName).getBucket(bucketName) + .createKey(keyName, keySize, ReplicationType.STAND_ALONE, + ReplicationFactor.ONE); + String dataString = RandomStringUtils.randomAlphabetic(keySize); + key.write(dataString.getBytes()); + key.close(); + + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setType(HddsProtos.ReplicationType.STAND_ALONE) + .setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(keySize) + .build(); + + + OmKeyLocationInfo keyInfo = + cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions() + .get(0).getBlocksLatestVersionOnly().get(0); + + ContainerData cd = getContainerData(keyInfo.getContainerID()); + - LOG.info("DN Container Data: keyCount: {} used: {} ", - cd.getKeyCount(), cd.getBytesUsed()); ++/* LOG.info("DN Container Data: keyCount: {} used: {} ", ++ cd.getKeyCount(), cd.getBytesUsed());*/ + + ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID()); + + LOG.info("SCM Container Info keyCount: {} usedBytes: {}", + cinfo.getNumberOfKeys(), cinfo.getUsedBytes()); + } + + + private static ContainerData getContainerData(long containerID) { + ContainerData containerData; + try { - ContainerManager containerManager = cluster.getHddsDatanodes().get(0) - .getDatanodeStateMachine().getContainer().getContainerManager(); - containerData = containerManager.readContainer(containerID); ++ ContainerSet containerManager = cluster.getHddsDatanodes().get(0) ++ .getDatanodeStateMachine().getContainer().getContainerSet(); ++ containerData = containerManager.getContainer(containerID).getContainerData(); + } catch (StorageContainerException e) { + throw new AssertionError(e); + } + return containerData; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java index 5c62803,42bb936..a2d95e8 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java @@@ -141,9 -144,8 +144,8 @@@ public class TestContainerSmallFile ContainerProtocolCalls.writeSmallFile(client, blockID, "data123".getBytes(), traceID); - thrown.expect(StorageContainerException.class); - thrown.expectMessage("Unable to find the container"); + thrown.expectMessage("ContainerID 8888 does not exist"); // Try to read a invalid key ContainerProtos.GetSmallFileResponseProto response = http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java index 12d444a,a6bb586..cc11feb --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java @@@ -162,21 -158,22 +163,22 @@@ public class TestSCMCli // 1. Test to delete a non-empty container. // **************************************** // Create an non-empty container - ContainerInfo container = containerOperationClient + ContainerWithPipeline container = containerOperationClient .createContainer(xceiverClientManager.getType(), HddsProtos.ReplicationFactor.ONE, containerOwner); -- - ContainerData cdata = ContainerData - .getFromProtBuf(containerOperationClient.readContainer( - container.getContainerInfo().getContainerID()), conf); - KeyUtils.getDB(cdata, conf) + KeyValueContainerData kvData = KeyValueContainerData + .getFromProtoBuf(containerOperationClient.readContainer( - container.getContainerID(), container.getPipeline())); ++ container.getContainerInfo().getContainerID(), container ++ .getPipeline())); + KeyUtils.getDB(kvData, conf) - .put(Longs.toByteArray(container.getContainerID()), + .put(Longs.toByteArray(container.getContainerInfo().getContainerID()), "someKey".getBytes()); - Assert.assertTrue(containerExist(container.getContainerID())); - Assert.assertTrue( - containerExist(container.getContainerInfo().getContainerID())); ++ Assert.assertTrue(containerExist(container.getContainerInfo() ++ .getContainerID())); // Gracefully delete a container should fail because it is open. - delCmd = new String[] {"-container", "-delete", "-c", - Long.toString(container.getContainerID())}; + delCmd = new String[]{"-container", "-delete", "-c", + Long.toString(container.getContainerInfo().getContainerID())}; testErr = new ByteArrayOutputStream(); ByteArrayOutputStream out = new ByteArrayOutputStream(); exitCode = runCommandAndGetOutput(delCmd, out, testErr); @@@ -275,26 -267,24 +272,27 @@@ EXECUTION_ERROR, exitCode); // Create an empty container. - ContainerInfo container = containerOperationClient + ContainerWithPipeline container = containerOperationClient .createContainer(xceiverClientManager.getType(), HddsProtos.ReplicationFactor.ONE, containerOwner); - ContainerData data = ContainerData.getFromProtBuf(containerOperationClient - .readContainer(container.getContainerInfo().getContainerID()), conf); - + KeyValueContainerData data = KeyValueContainerData + .getFromProtoBuf(containerOperationClient. - readContainer(container.getContainerID(), ++ readContainer(container.getContainerInfo().getContainerID(), + container.getPipeline())); - - info = new String[] { "-container", "-info", "-c", - Long.toString(container.getContainerID()) }; + info = new String[]{"-container", "-info", "-c", + Long.toString(container.getContainerInfo().getContainerID())}; ByteArrayOutputStream out = new ByteArrayOutputStream(); exitCode = runCommandAndGetOutput(info, out, null); assertEquals("Expected Success, did not find it.", ResultCode.SUCCESS, - exitCode); + exitCode); String openStatus = data.isOpen() ? "OPEN" : "CLOSED"; - String expected = String.format(formatStr, container.getContainerInfo() - .getContainerID(), openStatus, data.getDBPath(), - data.getContainerPath(), "", datanodeDetails.getHostName(), - datanodeDetails.getHostName()); + String expected = - String.format(formatStr, container.getContainerID(), openStatus, - data.getDbFile().getPath(), data.getContainerPath(), "", - datanodeDetails.getHostName(), datanodeDetails.getHostName()); ++ String.format(formatStr, container.getContainerInfo().getContainerID ++ (), openStatus, data.getDbFile().getPath(), data ++ .getContainerPath(), "", datanodeDetails.getHostName(), ++ datanodeDetails.getHostName()); ++ assertEquals(expected, out.toString()); out.reset(); @@@ -303,9 -293,9 +301,10 @@@ container = containerOperationClient .createContainer(xceiverClientManager.getType(), HddsProtos.ReplicationFactor.ONE, containerOwner); - data = ContainerData - .getFromProtBuf(containerOperationClient.readContainer( - container.getContainerInfo().getContainerID()), conf); + data = KeyValueContainerData + .getFromProtoBuf(containerOperationClient.readContainer( - container.getContainerID(), container.getPipeline())); ++ container.getContainerInfo().getContainerID(), container ++ .getPipeline())); KeyUtils.getDB(data, conf) .put(containerID.getBytes(), "someKey".getBytes()); @@@ -315,25 -305,24 +314,27 @@@ assertEquals(ResultCode.SUCCESS, exitCode); openStatus = data.isOpen() ? "OPEN" : "CLOSED"; - expected = String.format(formatStr, container.getContainerID(), openStatus, - data.getDbFile().getPath(), data.getContainerPath(), "", - datanodeDetails.getHostName(), datanodeDetails.getHostName()); - expected = String.format(formatStr, container.getContainerInfo(). - getContainerID(), openStatus, data.getDBPath(), - data.getContainerPath(), "", datanodeDetails.getHostName(), ++ ++ expected = String.format(formatStr, container.getContainerInfo() ++ .getContainerID(), openStatus, data.getDbFile().getPath(), data ++ .getContainerPath(), "", datanodeDetails.getHostName(), + datanodeDetails.getHostName()); assertEquals(expected, out.toString()); out.reset(); - // Close last container and test info again. - containerOperationClient.closeContainer( - container.getContainerID(), container.getPipeline()); + containerOperationClient + .closeContainer(container.getContainerInfo().getContainerID()); - info = new String[] { "-container", "-info", "-c", - Long.toString(container.getContainerID()) }; + info = new String[]{"-container", "-info", "-c", + Long.toString(container.getContainerInfo().getContainerID())}; exitCode = runCommandAndGetOutput(info, out, null); assertEquals(ResultCode.SUCCESS, exitCode); - data = ContainerData.getFromProtBuf(containerOperationClient - .readContainer(container.getContainerInfo().getContainerID()), conf); + data = KeyValueContainerData + .getFromProtoBuf(containerOperationClient.readContainer( - container.getContainerID(), container.getPipeline())); ++ container.getContainerInfo().getContainerID(), container ++ .getPipeline())); openStatus = data.isOpen() ? "OPEN" : "CLOSED"; expected = String http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java ---------------------------------------------------------------------- diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index 2f592c1,a95bd0e..c144db2 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@@ -44,18 -44,17 +44,18 @@@ import org.apache.hadoop.ozone.client.i import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.client.rpc.RpcClient; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; -import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; - import org.apache.hadoop.ozone.ksm.KeySpaceManager; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; - import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs; - import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo; - import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo; - import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos + import org.apache.hadoop.ozone.om.OzoneManager; + import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; + import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; + import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; + import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; + import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; + import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .Status; import org.apache.hadoop.ozone.client.rest.OzoneException; import org.apache.hadoop.ozone.web.utils.OzoneUtils; @@@ -663,12 -661,11 +663,12 @@@ public class TestKeys } @Test + @Ignore("Needs to be fixed for new SCM and Storage design") public void testDeleteKey() throws Exception { - KeySpaceManager ksm = ozoneCluster.getKeySpaceManager(); + OzoneManager ozoneManager = ozoneCluster.getOzoneManager(); // To avoid interference from other test cases, // we collect number of existing keys at the beginning - int numOfExistedKeys = countKsmKeys(ksm); + int numOfExistedKeys = countOmKeys(ozoneManager); // Keep tracking bucket keys info while creating them PutHelper helper = new PutHelper(client, path); @@@ -697,20 -694,17 +697,20 @@@ // Memorize chunks that has been created, // so we can verify actual deletions at DN side later. - for (KsmKeyInfo keyInfo : createdKeys) { - List locations = + for (OmKeyInfo keyInfo : createdKeys) { + List locations = keyInfo.getLatestVersionLocations().getLocationList(); - for (KsmKeyLocationInfo location : locations) { + for (OmKeyLocationInfo location : locations) { - KeyData keyData = new KeyData(location.getBlockID()); - KeyData blockInfo = cm.getContainerManager() - .getKeyManager().getKey(keyData); - ContainerData containerData = cm.getContainerManager() - .readContainer(keyData.getContainerID()); - File dataDir = ContainerUtils - .getDataDirectory(containerData).toFile(); + KeyValueHandler keyValueHandler = (KeyValueHandler) cm + .getDispatcher().getHandler(ContainerProtos.ContainerType + .KeyValueContainer); + KeyValueContainer container = (KeyValueContainer) cm.getContainerSet() + .getContainer(location.getBlockID().getContainerID()); + KeyData blockInfo = keyValueHandler + .getKeyManager().getKey(container, location.getBlockID()); + KeyValueContainerData containerData = (KeyValueContainerData) container + .getContainerData(); + File dataDir = new File(containerData.getChunksPath()); for (ContainerProtos.ChunkInfo chunkInfo : blockInfo.getChunks()) { File chunkFile = dataDir.toPath() .resolve(chunkInfo.getChunkName()).toFile(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org