hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [31/56] [abbrv] hadoop git commit: Merge trunk into HDDS-48
Date Wed, 11 Jul 2018 23:10:31 GMT
Merge trunk into HDDS-48


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c275a9a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c275a9a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c275a9a6

Branch: refs/heads/HDFS-12943
Commit: c275a9a6a07b2bd889bdba4d05b420027f430b34
Parents: 44e19fc 83cd84b
Author: Bharat Viswanadham <bharat@apache.org>
Authored: Mon Jul 9 12:13:03 2018 -0700
Committer: Bharat Viswanadham <bharat@apache.org>
Committed: Mon Jul 9 12:13:03 2018 -0700

----------------------------------------------------------------------
 .gitignore                                      |    4 +
 dev-support/bin/ozone-dist-layout-stitching     |    2 +-
 ...ExcludePrivateAnnotationsStandardDoclet.java |    6 +-
 .../hadoop-common/src/main/conf/hadoop-env.sh   |    6 +-
 .../org/apache/hadoop/conf/Configuration.java   |  458 +++---
 .../java/org/apache/hadoop/fs/FileContext.java  |    9 +-
 .../org/apache/hadoop/fs/LocalDirAllocator.java |    7 +-
 .../hadoop-common/src/site/markdown/Metrics.md  |   39 +-
 .../org/apache/hadoop/fs/TestFileContext.java   |   44 +-
 .../apache/hadoop/fs/TestLocalDirAllocator.java |   59 +
 .../src/main/compose/ozone/docker-compose.yaml  |    6 +-
 .../src/main/compose/ozone/docker-config        |    2 +-
 .../src/main/compose/ozoneperf/README.md        |    4 +-
 .../main/compose/ozoneperf/docker-compose.yaml  |    6 +-
 .../src/main/compose/ozoneperf/docker-config    |    2 +-
 .../scm/client/ContainerOperationClient.java    |  117 +-
 hadoop-hdds/common/pom.xml                      |   18 +
 .../hadoop/hdds/protocol/DatanodeDetails.java   |   13 +-
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |    6 +-
 .../hadoop/hdds/scm/client/ScmClient.java       |   43 +-
 .../container/common/helpers/ContainerInfo.java |  167 ++-
 .../common/helpers/ContainerWithPipeline.java   |  131 ++
 .../StorageContainerLocationProtocol.java       |   18 +-
 ...rLocationProtocolClientSideTranslatorPB.java |   34 +-
 .../org/apache/hadoop/ozone/OzoneConsts.java    |   22 +-
 .../apache/hadoop/ozone/audit/AuditAction.java  |   30 +
 .../hadoop/ozone/audit/AuditEventStatus.java    |   36 +
 .../apache/hadoop/ozone/audit/AuditLogger.java  |  128 ++
 .../hadoop/ozone/audit/AuditLoggerType.java     |   37 +
 .../apache/hadoop/ozone/audit/AuditMarker.java  |   38 +
 .../apache/hadoop/ozone/audit/Auditable.java    |   32 +
 .../apache/hadoop/ozone/audit/package-info.java |  123 ++
 .../org/apache/hadoop/ozone/common/Storage.java |    6 +-
 ...rLocationProtocolServerSideTranslatorPB.java |   33 +-
 .../main/proto/ScmBlockLocationProtocol.proto   |   10 +-
 .../StorageContainerLocationProtocol.proto      |   34 +-
 hadoop-hdds/common/src/main/proto/hdds.proto    |   28 +-
 .../common/src/main/resources/ozone-default.xml |  131 +-
 .../apache/hadoop/ozone/audit/DummyAction.java  |   51 +
 .../apache/hadoop/ozone/audit/DummyEntity.java  |   57 +
 .../ozone/audit/TestOzoneAuditLogger.java       |  147 ++
 .../apache/hadoop/ozone/audit/package-info.java |   23 +
 .../common/src/test/resources/log4j2.properties |   76 +
 .../apache/hadoop/hdds/scm/HddsServerUtil.java  |   11 -
 .../DeleteBlocksCommandHandler.java             |   30 +-
 .../protocol/StorageContainerNodeProtocol.java  |    4 +-
 .../src/main/resources/webapps/static/ozone.js  |    4 +-
 .../webapps/static/templates/config.html        |    4 +-
 .../hadoop/hdds/scm/block/BlockManagerImpl.java |   80 +-
 .../block/DatanodeDeletedBlockTransactions.java |   11 +-
 .../hadoop/hdds/scm/block/DeletedBlockLog.java  |    2 +-
 .../container/CloseContainerEventHandler.java   |   35 +-
 .../hdds/scm/container/ContainerMapping.java    |  128 +-
 .../scm/container/ContainerStateManager.java    |   30 +-
 .../hadoop/hdds/scm/container/Mapping.java      |   26 +-
 .../scm/container/closer/ContainerCloser.java   |   15 +-
 .../scm/container/states/ContainerStateMap.java |   13 +-
 .../hadoop/hdds/scm/events/SCMEvents.java       |   80 ++
 .../hadoop/hdds/scm/events/package-info.java    |   23 +
 .../hadoop/hdds/scm/node/CommandQueue.java      |    2 +-
 .../hadoop/hdds/scm/node/DatanodeInfo.java      |  109 ++
 .../hdds/scm/node/HeartbeatQueueItem.java       |   98 --
 .../hadoop/hdds/scm/node/NodeManager.java       |   16 +-
 .../hadoop/hdds/scm/node/NodeStateManager.java  |  575 ++++++++
 .../hadoop/hdds/scm/node/SCMNodeManager.java    |  511 +------
 .../node/states/NodeAlreadyExistsException.java |   45 +
 .../hdds/scm/node/states/NodeException.java     |   44 +
 .../scm/node/states/NodeNotFoundException.java  |   49 +
 .../hdds/scm/node/states/NodeStateMap.java      |  281 ++++
 .../hdds/scm/pipelines/PipelineManager.java     |   27 +-
 .../hdds/scm/pipelines/PipelineSelector.java    |   16 +
 .../scm/pipelines/ratis/RatisManagerImpl.java   |    1 +
 .../standalone/StandaloneManagerImpl.java       |    1 +
 .../hdds/scm/server/SCMBlockProtocolServer.java |    2 +-
 .../scm/server/SCMClientProtocolServer.java     |   74 +-
 .../server/SCMDatanodeHeartbeatDispatcher.java  |   13 +-
 .../scm/server/SCMDatanodeProtocolServer.java   |    2 +-
 .../scm/server/StorageContainerManager.java     |    7 +-
 .../hdds/scm/block/TestDeletedBlockLog.java     |   15 +-
 .../hdds/scm/container/MockNodeManager.java     |   58 +-
 .../TestCloseContainerEventHandler.java         |   54 +-
 .../scm/container/TestContainerMapping.java     |   27 +-
 .../container/closer/TestContainerCloser.java   |   18 +-
 .../hdds/scm/node/TestContainerPlacement.java   |   16 +-
 .../hadoop/hdds/scm/node/TestNodeManager.java   |  186 +--
 .../TestSCMDatanodeHeartbeatDispatcher.java     |   20 +-
 .../testutils/ReplicationNodeManagerMock.java   |   37 +-
 .../hadoop/hdds/scm/cli/OzoneBaseCLI.java       |    2 +-
 .../cli/container/CloseContainerHandler.java    |   10 +-
 .../cli/container/DeleteContainerHandler.java   |    9 +-
 .../scm/cli/container/InfoContainerHandler.java |   11 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |   19 -
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |   46 +-
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |   15 +-
 .../hdfs/client/HdfsClientConfigKeys.java       |    3 +
 .../hdfs/client/impl/BlockReaderFactory.java    |   21 +-
 .../hdfs/client/impl/BlockReaderLocal.java      |   93 +-
 .../client/impl/BlockReaderLocalLegacy.java     |   44 +-
 .../hdfs/client/impl/BlockReaderRemote.java     |   33 +-
 .../datanode/ReplicaNotFoundException.java      |    2 +-
 .../ha/ConfiguredFailoverProxyProvider.java     |    9 +-
 .../InMemoryAliasMapFailoverProxyProvider.java  |   38 +
 .../hdfs/server/federation/router/Quota.java    |   10 +-
 .../router/RouterQuotaUpdateService.java        |   43 +-
 .../federation/router/RouterRpcServer.java      |    1 -
 .../router/TestDisableRouterQuota.java          |   94 ++
 .../federation/router/TestRouterQuota.java      |  212 ++-
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |    5 +-
 .../java/org/apache/hadoop/hdfs/DFSUtil.java    |   37 +-
 .../org/apache/hadoop/hdfs/NameNodeProxies.java |   15 +-
 ...yAliasMapProtocolClientSideTranslatorPB.java |   95 +-
 .../aliasmap/InMemoryAliasMapProtocol.java      |    5 +
 .../aliasmap/InMemoryLevelDBAliasMapServer.java |   19 +-
 .../impl/InMemoryLevelDBAliasMapClient.java     |   80 +-
 .../impl/TextFileRegionAliasMap.java            |    5 +-
 .../hadoop/hdfs/server/datanode/DataNode.java   |   21 +-
 .../hdfs/server/datanode/DiskBalancer.java      |   29 +-
 .../erasurecode/StripedBlockReader.java         |    2 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |    8 +
 .../hdfs/server/namenode/NamenodeFsck.java      |    1 -
 .../src/main/resources/hdfs-default.xml         |   35 +-
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |   13 +-
 .../apache/hadoop/hdfs/MiniDFSNNTopology.java   |    2 +-
 .../hdfs/client/impl/BlockReaderTestUtil.java   |    2 -
 .../hdfs/client/impl/TestBlockReaderLocal.java  |    2 -
 .../blockmanagement/TestBlockTokenWithDFS.java  |    2 -
 .../TestNameNodePrunesMissingStorages.java      |    5 +-
 .../impl/TestInMemoryLevelDBAliasMapClient.java |    7 +
 .../datanode/TestDataNodeVolumeFailure.java     |    2 -
 .../server/diskbalancer/TestDiskBalancer.java   |   80 +-
 .../shortcircuit/TestShortCircuitCache.java     |   89 ++
 .../src/test/acceptance/basic/basic.robot       |    6 +-
 .../test/acceptance/basic/docker-compose.yaml   |    8 +-
 .../src/test/acceptance/basic/docker-config     |    4 +-
 .../src/test/acceptance/basic/ozone-shell.robot |   18 +-
 .../src/test/acceptance/commonlib.robot         |    4 +-
 .../test/acceptance/ozonefs/docker-compose.yaml |    8 +-
 .../src/test/acceptance/ozonefs/docker-config   |    4 +-
 .../src/test/acceptance/ozonefs/ozonefs.robot   |    6 +-
 .../apache/hadoop/ozone/client/BucketArgs.java  |    4 +-
 .../hadoop/ozone/client/OzoneClientFactory.java |   89 +-
 .../apache/hadoop/ozone/client/OzoneKey.java    |    2 +-
 .../apache/hadoop/ozone/client/VolumeArgs.java  |    4 +-
 .../ozone/client/io/ChunkGroupInputStream.java  |   33 +-
 .../ozone/client/io/ChunkGroupOutputStream.java |   63 +-
 .../client/rest/DefaultRestServerSelector.java  |    2 +-
 .../hadoop/ozone/client/rest/RestClient.java    |   15 +-
 .../ozone/client/rest/RestServerSelector.java   |    2 +-
 .../hadoop/ozone/client/rpc/RpcClient.java      |  142 +-
 .../ozone/client/TestHddsClientUtils.java       |   24 +-
 hadoop-ozone/common/pom.xml                     |    2 +-
 hadoop-ozone/common/src/main/bin/ozone          |    9 +-
 hadoop-ozone/common/src/main/bin/start-ozone.sh |   16 +-
 hadoop-ozone/common/src/main/bin/stop-ozone.sh  |   16 +-
 .../java/org/apache/hadoop/ozone/KsmUtils.java  |   87 --
 .../java/org/apache/hadoop/ozone/OmUtils.java   |   94 ++
 .../org/apache/hadoop/ozone/audit/OMAction.java |   51 +
 .../apache/hadoop/ozone/audit/package-info.java |   22 +
 .../apache/hadoop/ozone/freon/OzoneGetConf.java |   16 +-
 .../apache/hadoop/ozone/ksm/KSMConfigKeys.java  |   81 --
 .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java |  233 ---
 .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java |  235 ---
 .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java    |  119 --
 .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java    |  277 ----
 .../ozone/ksm/helpers/KsmKeyLocationInfo.java   |  129 --
 .../ksm/helpers/KsmKeyLocationInfoGroup.java    |  118 --
 .../ozone/ksm/helpers/KsmOzoneAclMap.java       |  110 --
 .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java |  223 ---
 .../ozone/ksm/helpers/OpenKeySession.java       |   50 -
 .../hadoop/ozone/ksm/helpers/ServiceInfo.java   |  237 ---
 .../hadoop/ozone/ksm/helpers/VolumeArgs.java    |  140 --
 .../hadoop/ozone/ksm/helpers/package-info.java  |   18 -
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 -
 .../ksm/protocol/KeySpaceManagerProtocol.java   |  252 ----
 .../hadoop/ozone/ksm/protocol/package-info.java |   19 -
 ...ceManagerProtocolClientSideTranslatorPB.java |  769 ----------
 .../protocolPB/KeySpaceManagerProtocolPB.java   |   34 -
 .../ozone/ksm/protocolPB/package-info.java      |   19 -
 .../apache/hadoop/ozone/om/OMConfigKeys.java    |   81 ++
 .../hadoop/ozone/om/helpers/OmBucketArgs.java   |  233 +++
 .../hadoop/ozone/om/helpers/OmBucketInfo.java   |  235 +++
 .../hadoop/ozone/om/helpers/OmKeyArgs.java      |  119 ++
 .../hadoop/ozone/om/helpers/OmKeyInfo.java      |  277 ++++
 .../ozone/om/helpers/OmKeyLocationInfo.java     |  129 ++
 .../om/helpers/OmKeyLocationInfoGroup.java      |  118 ++
 .../hadoop/ozone/om/helpers/OmOzoneAclMap.java  |  110 ++
 .../hadoop/ozone/om/helpers/OmVolumeArgs.java   |  223 +++
 .../hadoop/ozone/om/helpers/OpenKeySession.java |   50 +
 .../hadoop/ozone/om/helpers/ServiceInfo.java    |  237 +++
 .../hadoop/ozone/om/helpers/VolumeArgs.java     |  140 ++
 .../hadoop/ozone/om/helpers/package-info.java   |   18 +
 .../apache/hadoop/ozone/om/package-info.java    |   21 +
 .../ozone/om/protocol/OzoneManagerProtocol.java |  252 ++++
 .../hadoop/ozone/om/protocol/package-info.java  |   19 +
 ...neManagerProtocolClientSideTranslatorPB.java |  769 ++++++++++
 .../om/protocolPB/OzoneManagerProtocolPB.java   |   34 +
 .../ozone/om/protocolPB/package-info.java       |   19 +
 .../hadoop/ozone/protocolPB/KSMPBHelper.java    |  113 --
 .../hadoop/ozone/protocolPB/OMPBHelper.java     |  113 ++
 .../hadoop/ozone/protocolPB/OzonePBHelper.java  |   30 +
 .../main/proto/KeySpaceManagerProtocol.proto    |  474 ------
 .../src/main/proto/OzoneManagerProtocol.proto   |  480 +++++++
 hadoop-ozone/docs/content/GettingStarted.md     |   18 +-
 hadoop-ozone/docs/content/Metrics.md            |   10 +-
 hadoop-ozone/docs/content/_index.md             |   12 +-
 hadoop-ozone/docs/static/OzoneOverview.svg      |    2 +-
 .../container/TestContainerStateManager.java    |  161 ++-
 .../apache/hadoop/ozone/MiniOzoneCluster.java   |   24 +-
 .../hadoop/ozone/MiniOzoneClusterImpl.java      |   66 +-
 .../hadoop/ozone/TestContainerOperations.java   |   11 +-
 .../ozone/TestOzoneConfigurationFields.java     |    4 +-
 .../ozone/TestStorageContainerManager.java      |   28 +-
 .../TestStorageContainerManagerHelper.java      |   22 +-
 .../ozone/client/rest/TestOzoneRestClient.java  |    6 +-
 .../ozone/client/rpc/TestOzoneRpcClient.java    |   22 +-
 .../commandhandler/TestBlockDeletion.java       |  212 +++
 .../TestCloseContainerByPipeline.java           |   97 +-
 .../TestCloseContainerHandler.java              |   14 +-
 .../ozone/ksm/TestContainerReportWithKeys.java  |  143 --
 .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java |  306 ----
 .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java  |  284 ----
 .../hadoop/ozone/ksm/TestKeySpaceManager.java   | 1350 ------------------
 .../ksm/TestKeySpaceManagerRestInterface.java   |  135 --
 .../ozone/ksm/TestKsmBlockVersioning.java       |  253 ----
 .../ksm/TestMultipleContainerReadWrite.java     |  215 ---
 .../ozone/om/TestContainerReportWithKeys.java   |  143 ++
 .../om/TestMultipleContainerReadWrite.java      |  215 +++
 .../hadoop/ozone/om/TestOmBlockVersioning.java  |  253 ++++
 .../apache/hadoop/ozone/om/TestOmMetrics.java   |  313 ++++
 .../apache/hadoop/ozone/om/TestOmSQLCli.java    |  284 ++++
 .../hadoop/ozone/om/TestOzoneManager.java       | 1349 +++++++++++++++++
 .../ozone/om/TestOzoneManagerRestInterface.java |  135 ++
 .../hadoop/ozone/ozShell/TestOzoneShell.java    |   14 +-
 .../hadoop/ozone/scm/TestAllocateContainer.java |    6 +-
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |    3 +-
 .../ozone/scm/TestContainerSmallFile.java       |   36 +-
 .../org/apache/hadoop/ozone/scm/TestSCMCli.java |  127 +-
 .../ozone/scm/TestXceiverClientManager.java     |   62 +-
 .../ozone/scm/TestXceiverClientMetrics.java     |   14 +-
 .../hadoop/ozone/scm/node/TestQueryNode.java    |   19 +-
 .../ozone/web/TestDistributedOzoneVolumes.java  |   12 +-
 .../hadoop/ozone/web/client/TestKeys.java       |   58 +-
 .../src/test/resources/webapps/ksm/.gitkeep     |   15 -
 .../resources/webapps/ozoneManager/.gitkeep     |   15 +
 .../server/datanode/ObjectStoreHandler.java     |   33 +-
 .../ozone/web/handlers/KeyProcessTemplate.java  |    4 +-
 .../web/handlers/VolumeProcessTemplate.java     |    4 +-
 .../web/storage/DistributedStorageHandler.java  |  153 +-
 .../apache/hadoop/ozone/ksm/BucketManager.java  |   79 -
 .../hadoop/ozone/ksm/BucketManagerImpl.java     |  315 ----
 .../org/apache/hadoop/ozone/ksm/KSMMXBean.java  |   31 -
 .../hadoop/ozone/ksm/KSMMetadataManager.java    |  253 ----
 .../ozone/ksm/KSMMetadataManagerImpl.java       |  526 -------
 .../org/apache/hadoop/ozone/ksm/KSMMetrics.java |  459 ------
 .../org/apache/hadoop/ozone/ksm/KSMStorage.java |   90 --
 .../hadoop/ozone/ksm/KeyDeletingService.java    |  142 --
 .../org/apache/hadoop/ozone/ksm/KeyManager.java |  175 ---
 .../apache/hadoop/ozone/ksm/KeyManagerImpl.java |  566 --------
 .../hadoop/ozone/ksm/KeySpaceManager.java       |  914 ------------
 .../ozone/ksm/KeySpaceManagerHttpServer.java    |   78 -
 .../hadoop/ozone/ksm/OpenKeyCleanupService.java |  117 --
 .../ozone/ksm/ServiceListJSONServlet.java       |  103 --
 .../apache/hadoop/ozone/ksm/VolumeManager.java  |  100 --
 .../hadoop/ozone/ksm/VolumeManagerImpl.java     |  391 -----
 .../ozone/ksm/exceptions/KSMException.java      |  118 --
 .../ozone/ksm/exceptions/package-info.java      |   19 -
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 -
 .../apache/hadoop/ozone/om/BucketManager.java   |   79 +
 .../hadoop/ozone/om/BucketManagerImpl.java      |  315 ++++
 .../hadoop/ozone/om/KeyDeletingService.java     |  142 ++
 .../org/apache/hadoop/ozone/om/KeyManager.java  |  175 +++
 .../apache/hadoop/ozone/om/KeyManagerImpl.java  |  566 ++++++++
 .../org/apache/hadoop/ozone/om/OMMXBean.java    |   31 +
 .../hadoop/ozone/om/OMMetadataManager.java      |  253 ++++
 .../org/apache/hadoop/ozone/om/OMMetrics.java   |  459 ++++++
 .../org/apache/hadoop/ozone/om/OMStorage.java   |   90 ++
 .../hadoop/ozone/om/OmMetadataManagerImpl.java  |  526 +++++++
 .../hadoop/ozone/om/OpenKeyCleanupService.java  |  117 ++
 .../apache/hadoop/ozone/om/OzoneManager.java    |  911 ++++++++++++
 .../hadoop/ozone/om/OzoneManagerHttpServer.java |   78 +
 .../hadoop/ozone/om/ServiceListJSONServlet.java |  103 ++
 .../apache/hadoop/ozone/om/VolumeManager.java   |  100 ++
 .../hadoop/ozone/om/VolumeManagerImpl.java      |  390 +++++
 .../hadoop/ozone/om/exceptions/OMException.java |  118 ++
 .../ozone/om/exceptions/package-info.java       |   19 +
 .../apache/hadoop/ozone/om/package-info.java    |   21 +
 ...ceManagerProtocolServerSideTranslatorPB.java |  559 --------
 ...neManagerProtocolServerSideTranslatorPB.java |  571 ++++++++
 .../hadoop/ozone/protocolPB/package-info.java   |    2 +-
 .../src/main/webapps/ksm/index.html             |   70 -
 .../src/main/webapps/ksm/ksm-metrics.html       |   44 -
 .../ozone-manager/src/main/webapps/ksm/ksm.js   |  110 --
 .../ozone-manager/src/main/webapps/ksm/main.css |   23 -
 .../src/main/webapps/ksm/main.html              |   18 -
 .../src/main/webapps/ozoneManager/index.html    |   70 +
 .../src/main/webapps/ozoneManager/main.css      |   23 +
 .../src/main/webapps/ozoneManager/main.html     |   18 +
 .../main/webapps/ozoneManager/om-metrics.html   |   44 +
 .../main/webapps/ozoneManager/ozoneManager.js   |  110 ++
 .../hadoop/ozone/ksm/TestBucketManagerImpl.java |  395 -----
 .../hadoop/ozone/ksm/TestChunkStreams.java      |  234 ---
 .../ksm/TestKeySpaceManagerHttpServer.java      |  141 --
 .../apache/hadoop/ozone/ksm/package-info.java   |   21 -
 .../hadoop/ozone/om/TestBucketManagerImpl.java  |  394 +++++
 .../hadoop/ozone/om/TestChunkStreams.java       |  234 +++
 .../ozone/om/TestOzoneManagerHttpServer.java    |  141 ++
 .../apache/hadoop/ozone/om/package-info.java    |   21 +
 .../hadoop/fs/ozone/contract/OzoneContract.java |    4 +-
 .../genesis/BenchMarkContainerStateMap.java     |   16 +-
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  111 +-
 .../hadoop/fs/s3a/s3guard/S3GuardTool.java      |   10 +
 .../s3guard/AbstractS3GuardToolTestBase.java    |   18 +
 .../namenode/ITestProvidedImplementation.java   |  373 ++++-
 .../dev-support/findbugs-exclude.xml            |   17 +-
 .../hadoop/yarn/api/records/Resource.java       |   13 +
 .../api/records/impl/LightWeightResource.java   |   23 +-
 .../hadoop/yarn/conf/YarnConfiguration.java     |    7 +
 .../impl/pb/GetApplicationsRequestPBImpl.java   |   44 +-
 .../logaggregation/AggregatedLogFormat.java     |    6 +-
 .../timeline/RollingLevelDBTimelineStore.java   |    6 +
 .../server/timeline/TimelineDataManager.java    |    7 +-
 .../timeline/webapp/TimelineWebServices.java    |    4 +
 .../webapp/TestTimelineWebServices.java         |    2 +-
 .../amrmproxy/BroadcastAMRMProxyPolicy.java     |   11 -
 .../amrmproxy/RejectAMRMProxyPolicy.java        |    4 -
 .../TestBroadcastAMRMProxyFederationPolicy.java |   11 +-
 .../yarn/server/nodemanager/NodeManager.java    |   66 +-
 .../runtime/DockerLinuxContainerRuntime.java    |    4 +-
 .../runtime/ContainerExecutionException.java    |    6 +
 .../impl/container-executor.c                   |   30 +-
 .../container-executor/impl/utils/docker-util.c |    2 +-
 .../test/test-container-executor.c              |   20 +
 .../nodemanager/TestNodeManagerResync.java      |   56 +
 .../runtime/TestDockerContainerRuntime.java     |   10 +-
 .../conf/capacity-scheduler.xml                 |   10 +
 .../scheduler/capacity/CapacityScheduler.java   |   45 +-
 .../CapacitySchedulerConfiguration.java         |   10 +
 .../scheduler/capacity/ParentQueue.java         |   36 +-
 .../allocator/AbstractContainerAllocator.java   |   13 +-
 .../scheduler/common/fica/FiCaSchedulerApp.java |    5 +
 .../scheduler/fair/ConfigurableResource.java    |   69 +-
 .../fair/FairSchedulerConfiguration.java        |  174 ++-
 .../allocation/AllocationFileQueueParser.java   |    2 +-
 .../resourcemanager/webapp/dao/AppInfo.java     |    2 +-
 .../webapp/dao/SchedulerInfo.java               |    8 +-
 .../TestWorkPreservingRMRestart.java            |    2 +
 .../fair/TestFairSchedulerConfiguration.java    |  160 ++-
 .../webapp/TestRMWebServices.java               |   31 +-
 .../webapp/TestRMWebServicesApps.java           |   14 +-
 ...estRMWebServicesAppsCustomResourceTypes.java |  242 ++++
 .../webapp/TestRMWebServicesCapacitySched.java  |   30 +-
 .../TestRMWebServicesConfigurationMutation.java |    5 +
 .../webapp/TestRMWebServicesFairScheduler.java  |   95 +-
 .../TestRMWebServicesSchedulerActivities.java   |    2 +-
 ...ustomResourceTypesConfigurationProvider.java |  138 ++
 .../FairSchedulerJsonVerifications.java         |  139 ++
 .../FairSchedulerXmlVerifications.java          |  153 ++
 ...ervicesFairSchedulerCustomResourceTypes.java |  271 ++++
 .../webapp/helper/AppInfoJsonVerifications.java |  123 ++
 .../webapp/helper/AppInfoXmlVerifications.java  |  132 ++
 .../webapp/helper/BufferedClientResponse.java   |   57 +
 .../helper/JsonCustomResourceTypeTestcase.java  |   77 +
 .../ResourceRequestsJsonVerifications.java      |  252 ++++
 .../ResourceRequestsXmlVerifications.java       |  215 +++
 .../helper/XmlCustomResourceTypeTestCase.java   |  112 ++
 .../router/clientrm/RouterClientRMService.java  |   53 +-
 .../router/rmadmin/RouterRMAdminService.java    |   51 +-
 .../server/router/webapp/RouterWebServices.java |   48 +-
 .../clientrm/TestRouterClientRMService.java     |   60 +
 .../rmadmin/TestRouterRMAdminService.java       |   60 +
 .../router/webapp/TestRouterWebServices.java    |   65 +
 .../pom.xml                                     |   10 +
 .../storage/TestTimelineReaderHBaseDown.java    |  220 +++
 .../storage/HBaseTimelineReaderImpl.java        |   93 ++
 .../reader/TimelineFromIdConverter.java         |   93 ++
 .../reader/TimelineReaderWebServices.java       |  198 ++-
 .../TestTimelineReaderWebServicesBasicAcl.java  |  154 ++
 .../src/site/markdown/FairScheduler.md          |    6 +-
 .../src/main/webapp/app/initializers/loader.js  |   10 +-
 379 files changed, 22363 insertions(+), 15606 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 82d67b7,4fad5d8..0db5993
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -98,11 -93,8 +98,11 @@@ public final class OzoneConsts 
    public static final String BLOCK_DB = "block.db";
    public static final String OPEN_CONTAINERS_DB = "openContainers.db";
    public static final String DELETED_BLOCK_DB = "deletedBlock.db";
-   public static final String KSM_DB_NAME = "ksm.db";
+   public static final String OM_DB_NAME = "om.db";
  
 +  public static final String STORAGE_DIR_CHUNKS = "chunks";
 +  public static final String CONTAINER_FILE_CHECKSUM_EXTENSION = ".chksm";
 +
    /**
     * Supports Bucket Versioning.
     */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 4fc1cd9,d215da9..c3d1596
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@@ -31,13 -29,11 +31,12 @@@ import org.apache.hadoop.hdds.protocol.
  import org.apache.hadoop.hdds.protocol.proto
      .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
  import org.apache.hadoop.ozone.OzoneConsts;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
  import org.apache.hadoop.ozone.container.common.helpers
      .DeletedContainerBlocksSummary;
 -import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
 -import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
 +import org.apache.hadoop.ozone.container.common.interfaces.Container;
- import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
 +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 +import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
 +import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
  import org.apache.hadoop.ozone.container.common.statemachine
      .EndpointStateMachine;
  import org.apache.hadoop.ozone.container.common.statemachine
@@@ -167,21 -145,28 +166,28 @@@ public class DeleteBlocksCommandHandle
     * Move a bunch of blocks from a container to deleting state.
     * This is a meta update, the actual deletes happen in async mode.
     *
 +   * @param containerData - KeyValueContainerData
     * @param delTX a block deletion transaction.
 -   * @param config configuration.
     * @throws IOException if I/O error occurs.
     */
 -  private void deleteContainerBlocks(DeletedBlocksTransaction delTX,
 -      Configuration config) throws IOException {
 +  private void deleteKeyValueContainerBlocks(
 +      KeyValueContainerData containerData, DeletedBlocksTransaction delTX)
 +      throws IOException {
      long containerId = delTX.getContainerID();
 -    ContainerData containerInfo = containerManager.readContainer(containerId);
      if (LOG.isDebugEnabled()) {
        LOG.debug("Processing Container : {}, DB path : {}", containerId,
 -          containerInfo.getDBPath());
 +          containerData.getMetadataPath());
      }
  
 -    if (delTX.getTxID() < containerInfo.getDeleteTransactionId()) {
++    if (delTX.getTxID() < containerData.getDeleteTransactionId()) {
+       LOG.debug(String.format("Ignoring delete blocks for containerId: %d."
+               + " Outdated delete transactionId %d < %d", containerId,
 -          delTX.getTxID(), containerInfo.getDeleteTransactionId()));
++          delTX.getTxID(), containerData.getDeleteTransactionId()));
+       return;
+     }
+ 
      int newDeletionBlocks = 0;
 -    MetadataStore containerDB = KeyUtils.getDB(containerInfo, config);
 +    MetadataStore containerDB = KeyUtils.getDB(containerData, conf);
      for (Long blk : delTX.getLocalIDList()) {
        BatchOperation batch = new BatchOperation();
        byte[] blkBytes = Longs.toByteArray(blk);
@@@ -208,13 -203,15 +224,15 @@@
          LOG.debug("Block {} not found or already under deletion in"
                  + " container {}, skip deleting it.", blk, containerId);
        }
-       containerDB.put(DFSUtil.string2Bytes(
-           OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + containerId),
-           Longs.toByteArray(delTX.getTxID()));
      }
  
+     containerDB.put(DFSUtil.string2Bytes(
+         OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
+         Longs.toByteArray(delTX.getTxID()));
 -    containerManager
 -        .updateDeleteTransactionId(delTX.getContainerID(), delTX.getTxID());
++    containerData
++        .updateDeleteTransactionId(delTX.getTxID());
      // update pending deletion blocks count in in-memory container status
 -    containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId);
 +    containerData.incrPendingDeletionBlocks(newDeletionBlocks);
    }
  
    @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index ad1e706,a30c6f4..fff8611
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@@ -27,13 -27,11 +27,13 @@@ import org.apache.hadoop.hdds.scm.conta
  import org.apache.hadoop.hdfs.DFSUtil;
  import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
  import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 -import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
 +import org.apache.hadoop.ozone.container.common.impl.ContainerData;
 +import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
 +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
 +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
  import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
  import org.apache.hadoop.ozone.web.handlers.BucketArgs;
  import org.apache.hadoop.ozone.web.handlers.KeyArgs;
  import org.apache.hadoop.ozone.web.handlers.UserArgs;
@@@ -160,14 -158,15 +160,16 @@@ public class TestStorageContainerManage
  
    private MetadataStore getContainerMetadata(Long containerID)
        throws IOException {
-     ContainerInfo container = cluster.getStorageContainerManager()
-         .getClientProtocolServer().getContainer(containerID);
-     DatanodeDetails leadDN = container.getPipeline().getLeader();
+     ContainerWithPipeline containerWithPipeline = cluster
+         .getStorageContainerManager().getClientProtocolServer()
+         .getContainerWithPipeline(containerID);
+ 
+     DatanodeDetails leadDN = containerWithPipeline.getPipeline().getLeader();
      OzoneContainer containerServer =
          getContainerServerByDatanodeUuid(leadDN.getUuidString());
 -    ContainerData containerData = containerServer.getContainerManager()
 -        .readContainer(containerID);
 +    KeyValueContainerData containerData = (KeyValueContainerData) containerServer
 +        .getContainerSet()
 +        .getContainer(containerID).getContainerData();
      return KeyUtils.getDB(containerData, conf);
    }
  

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index b832dd2,58b831b..30b18c2
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@@ -32,10 -32,10 +32,10 @@@ import org.apache.hadoop.ozone.client.O
  import org.apache.hadoop.ozone.client.OzoneClient;
  import org.apache.hadoop.ozone.client.OzoneClientFactory;
  import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 +import org.apache.hadoop.ozone.container.common.impl.ContainerData;
  import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
  import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
  import org.apache.hadoop.test.GenericTestUtils;
  import org.junit.AfterClass;
@@@ -204,14 -257,8 +257,8 @@@ public class TestCloseContainerByPipeli
          if (datanode.equals(datanodeService.getDatanodeDetails())) {
            containerData =
                datanodeService.getDatanodeStateMachine().getContainer()
 -                  .getContainerManager().readContainer(containerID);
 +                  .getContainerSet().getContainer(containerID).getContainerData();
-           if (!containerData.isOpen()) {
-             // make sure the closeContainerHandler on the Datanode is invoked
-             Assert.assertTrue(
-                 datanodeService.getDatanodeStateMachine().getCommandDispatcher()
-                     .getCloseContainerHandler().getInvocationCount() > 0);
-             return true;
-           }
+           return !containerData.isOpen();
          }
      } catch (StorageContainerException e) {
        throw new AssertionError(e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
index 114bd04,58a5154..682bd63
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
@@@ -27,9 -27,9 +27,9 @@@ import org.apache.hadoop.hdds.client.Re
  import org.apache.hadoop.hdds.client.ReplicationType;
  import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
  import org.apache.hadoop.ozone.client.rest.OzoneException;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 +import org.apache.hadoop.ozone.container.common.impl.ContainerData;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
  import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
  import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
  import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
index 0000000,5481506..c25b00e
mode 000000,100644..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
@@@ -1,0 -1,143 +1,143 @@@
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with this
+  * work for additional information regarding copyright ownership.  The ASF
+  * licenses this file to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance with the License.
+  * You may obtain a copy of the License at
+  * <p>
+  * http://www.apache.org/licenses/LICENSE-2.0
+  * <p>
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  * License for the specific language governing permissions and limitations under
+  * the License.
+  */
+ package org.apache.hadoop.ozone.om;
+ 
+ import org.apache.commons.lang3.RandomStringUtils;
+ 
+ import org.apache.hadoop.hdds.client.ReplicationFactor;
+ import org.apache.hadoop.hdds.client.ReplicationType;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+ import org.apache.hadoop.ozone.MiniOzoneCluster;
+ import org.apache.hadoop.ozone.OzoneConfigKeys;
+ import org.apache.hadoop.ozone.OzoneConsts;
+ import org.apache.hadoop.ozone.client.*;
+ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 -import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
++import org.apache.hadoop.ozone.container.common.impl.ContainerData;
++import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+ import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+ import org.junit.AfterClass;
+ import org.junit.BeforeClass;
+ import org.junit.Rule;
+ import org.junit.Test;
+ import org.junit.rules.ExpectedException;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ 
+ import java.io.IOException;
+ 
+ /**
+  * This class tests container report with DN container state info.
+  */
+ public class TestContainerReportWithKeys {
+   private static final Logger LOG = LoggerFactory.getLogger(
+       TestContainerReportWithKeys.class);
+   private static MiniOzoneCluster cluster = null;
+   private static OzoneConfiguration conf;
+   private static StorageContainerManager scm;
+ 
+   @Rule
+   public ExpectedException exception = ExpectedException.none();
+ 
+   /**
+    * Create a MiniDFSCluster for testing.
+    * <p>
+    * Ozone is made active by setting OZONE_ENABLED = true and
+    * OZONE_HANDLER_TYPE_KEY = "distributed"
+    *
+    * @throws IOException
+    */
+   @BeforeClass
+   public static void init() throws Exception {
+     conf = new OzoneConfiguration();
+     conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
+         OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
+     cluster = MiniOzoneCluster.newBuilder(conf).build();
+     cluster.waitForClusterToBeReady();
+     scm = cluster.getStorageContainerManager();
+   }
+ 
+   /**
+    * Shutdown MiniDFSCluster.
+    */
+   @AfterClass
+   public static void shutdown() {
+     if (cluster != null) {
+       cluster.shutdown();
+     }
+   }
+ 
+   @Test
+   public void testContainerReportKeyWrite() throws Exception {
+     final String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+     final String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+     final String keyName = "key" + RandomStringUtils.randomNumeric(5);
+     final int keySize = 100;
+ 
+     OzoneClient client = OzoneClientFactory.getClient(conf);
+     ObjectStore objectStore = client.getObjectStore();
+     objectStore.createVolume(volumeName);
+     objectStore.getVolume(volumeName).createBucket(bucketName);
+     OzoneOutputStream key =
+         objectStore.getVolume(volumeName).getBucket(bucketName)
+             .createKey(keyName, keySize, ReplicationType.STAND_ALONE,
+                 ReplicationFactor.ONE);
+     String dataString = RandomStringUtils.randomAlphabetic(keySize);
+     key.write(dataString.getBytes());
+     key.close();
+ 
+     OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+         .setVolumeName(volumeName)
+         .setBucketName(bucketName)
+         .setKeyName(keyName)
+         .setType(HddsProtos.ReplicationType.STAND_ALONE)
+         .setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(keySize)
+         .build();
+ 
+ 
+     OmKeyLocationInfo keyInfo =
+         cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
+             .get(0).getBlocksLatestVersionOnly().get(0);
+ 
+     ContainerData cd = getContainerData(keyInfo.getContainerID());
+ 
 -    LOG.info("DN Container Data:  keyCount: {} used: {} ",
 -        cd.getKeyCount(), cd.getBytesUsed());
++/*    LOG.info("DN Container Data:  keyCount: {} used: {} ",
++        cd.getKeyCount(), cd.getBytesUsed());*/
+ 
+     ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID());
+ 
+     LOG.info("SCM Container Info keyCount: {} usedBytes: {}",
+         cinfo.getNumberOfKeys(), cinfo.getUsedBytes());
+   }
+ 
+ 
+   private static ContainerData getContainerData(long containerID) {
+     ContainerData containerData;
+     try {
 -      ContainerManager containerManager = cluster.getHddsDatanodes().get(0)
 -          .getDatanodeStateMachine().getContainer().getContainerManager();
 -      containerData = containerManager.readContainer(containerID);
++      ContainerSet containerManager = cluster.getHddsDatanodes().get(0)
++          .getDatanodeStateMachine().getContainer().getContainerSet();
++      containerData = containerManager.getContainer(containerID).getContainerData();
+     } catch (StorageContainerException e) {
+       throw new AssertionError(e);
+     }
+     return containerData;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index 5c62803,42bb936..a2d95e8
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@@ -141,9 -144,8 +144,8 @@@ public class TestContainerSmallFile 
      ContainerProtocolCalls.writeSmallFile(client, blockID,
          "data123".getBytes(), traceID);
  
- 
      thrown.expect(StorageContainerException.class);
 -    thrown.expectMessage("Unable to find the container");
 +    thrown.expectMessage("ContainerID 8888 does not exist");
  
      // Try to read a invalid key
      ContainerProtos.GetSmallFileResponseProto response =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
index 12d444a,a6bb586..cc11feb
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
@@@ -162,21 -158,22 +163,22 @@@ public class TestSCMCli 
      // 1. Test to delete a non-empty container.
      // ****************************************
      // Create an non-empty container
-     ContainerInfo container = containerOperationClient
+     ContainerWithPipeline container = containerOperationClient
          .createContainer(xceiverClientManager.getType(),
              HddsProtos.ReplicationFactor.ONE, containerOwner);
--
 -    ContainerData cdata = ContainerData
 -        .getFromProtBuf(containerOperationClient.readContainer(
 -            container.getContainerInfo().getContainerID()), conf);
 -    KeyUtils.getDB(cdata, conf)
 +    KeyValueContainerData kvData = KeyValueContainerData
 +        .getFromProtoBuf(containerOperationClient.readContainer(
-             container.getContainerID(), container.getPipeline()));
++            container.getContainerInfo().getContainerID(), container
++                .getPipeline()));
 +    KeyUtils.getDB(kvData, conf)
-         .put(Longs.toByteArray(container.getContainerID()),
+         .put(Longs.toByteArray(container.getContainerInfo().getContainerID()),
              "someKey".getBytes());
-     Assert.assertTrue(containerExist(container.getContainerID()));
 -    Assert.assertTrue(
 -        containerExist(container.getContainerInfo().getContainerID()));
++    Assert.assertTrue(containerExist(container.getContainerInfo()
++        .getContainerID()));
  
      // Gracefully delete a container should fail because it is open.
-     delCmd = new String[] {"-container", "-delete", "-c",
-         Long.toString(container.getContainerID())};
+     delCmd = new String[]{"-container", "-delete", "-c",
+         Long.toString(container.getContainerInfo().getContainerID())};
      testErr = new ByteArrayOutputStream();
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      exitCode = runCommandAndGetOutput(delCmd, out, testErr);
@@@ -275,26 -267,24 +272,27 @@@
          EXECUTION_ERROR, exitCode);
  
      // Create an empty container.
-     ContainerInfo container = containerOperationClient
+     ContainerWithPipeline container = containerOperationClient
          .createContainer(xceiverClientManager.getType(),
              HddsProtos.ReplicationFactor.ONE, containerOwner);
 -    ContainerData data = ContainerData.getFromProtBuf(containerOperationClient
 -        .readContainer(container.getContainerInfo().getContainerID()), conf);
 -
 +    KeyValueContainerData data = KeyValueContainerData
 +        .getFromProtoBuf(containerOperationClient.
-             readContainer(container.getContainerID(),
++            readContainer(container.getContainerInfo().getContainerID(),
 +                container.getPipeline()));
- 
-     info = new String[] { "-container", "-info", "-c",
-         Long.toString(container.getContainerID()) };
+     info = new String[]{"-container", "-info", "-c",
+         Long.toString(container.getContainerInfo().getContainerID())};
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      exitCode = runCommandAndGetOutput(info, out, null);
      assertEquals("Expected Success, did not find it.", ResultCode.SUCCESS,
-             exitCode);
+         exitCode);
  
      String openStatus = data.isOpen() ? "OPEN" : "CLOSED";
 -    String expected = String.format(formatStr, container.getContainerInfo()
 -            .getContainerID(), openStatus, data.getDBPath(),
 -        data.getContainerPath(), "", datanodeDetails.getHostName(),
 -        datanodeDetails.getHostName());
 +    String expected =
-         String.format(formatStr, container.getContainerID(), openStatus,
-         data.getDbFile().getPath(), data.getContainerPath(), "",
-         datanodeDetails.getHostName(), datanodeDetails.getHostName());
++        String.format(formatStr, container.getContainerInfo().getContainerID
++                (), openStatus, data.getDbFile().getPath(), data
++                .getContainerPath(), "", datanodeDetails.getHostName(),
++            datanodeDetails.getHostName());
++
      assertEquals(expected, out.toString());
  
      out.reset();
@@@ -303,9 -293,9 +301,10 @@@
      container = containerOperationClient
          .createContainer(xceiverClientManager.getType(),
              HddsProtos.ReplicationFactor.ONE, containerOwner);
 -    data = ContainerData
 -        .getFromProtBuf(containerOperationClient.readContainer(
 -            container.getContainerInfo().getContainerID()), conf);
 +    data = KeyValueContainerData
 +        .getFromProtoBuf(containerOperationClient.readContainer(
-             container.getContainerID(), container.getPipeline()));
++            container.getContainerInfo().getContainerID(), container
++                .getPipeline()));
      KeyUtils.getDB(data, conf)
          .put(containerID.getBytes(), "someKey".getBytes());
  
@@@ -315,25 -305,24 +314,27 @@@
      assertEquals(ResultCode.SUCCESS, exitCode);
  
      openStatus = data.isOpen() ? "OPEN" : "CLOSED";
-     expected = String.format(formatStr, container.getContainerID(), openStatus,
-         data.getDbFile().getPath(), data.getContainerPath(), "",
-         datanodeDetails.getHostName(), datanodeDetails.getHostName());
 -    expected = String.format(formatStr, container.getContainerInfo().
 -            getContainerID(), openStatus, data.getDBPath(),
 -        data.getContainerPath(), "", datanodeDetails.getHostName(),
++
++    expected = String.format(formatStr, container.getContainerInfo()
++            .getContainerID(), openStatus, data.getDbFile().getPath(), data
++            .getContainerPath(), "", datanodeDetails.getHostName(),
+         datanodeDetails.getHostName());
      assertEquals(expected, out.toString());
  
      out.reset();
  
- 
      // Close last container and test info again.
-     containerOperationClient.closeContainer(
-         container.getContainerID(), container.getPipeline());
+     containerOperationClient
+         .closeContainer(container.getContainerInfo().getContainerID());
  
-     info = new String[] { "-container", "-info", "-c",
-         Long.toString(container.getContainerID()) };
+     info = new String[]{"-container", "-info", "-c",
+         Long.toString(container.getContainerInfo().getContainerID())};
      exitCode = runCommandAndGetOutput(info, out, null);
      assertEquals(ResultCode.SUCCESS, exitCode);
 -    data = ContainerData.getFromProtBuf(containerOperationClient
 -        .readContainer(container.getContainerInfo().getContainerID()), conf);
 +    data = KeyValueContainerData
 +        .getFromProtoBuf(containerOperationClient.readContainer(
-             container.getContainerID(), container.getPipeline()));
++            container.getContainerInfo().getContainerID(), container
++                .getPipeline()));
  
      openStatus = data.isOpen() ? "OPEN" : "CLOSED";
      expected = String

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 2f592c1,a95bd0e..c144db2
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@@ -44,18 -44,17 +44,18 @@@ import org.apache.hadoop.ozone.client.i
  import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
  import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
  import org.apache.hadoop.ozone.client.rpc.RpcClient;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 -import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
  import org.apache.hadoop.ozone.container.common.helpers.KeyData;
 +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
 +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 +import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
  import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
- import org.apache.hadoop.ozone.ksm.KeySpaceManager;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
- import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
- import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
+ import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
      .Status;
  import org.apache.hadoop.ozone.client.rest.OzoneException;
  import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@@@ -663,12 -661,11 +663,12 @@@ public class TestKeys 
    }
  
    @Test
 +  @Ignore("Needs to be fixed for new SCM and Storage design")
    public void testDeleteKey() throws Exception {
-     KeySpaceManager ksm = ozoneCluster.getKeySpaceManager();
+     OzoneManager ozoneManager = ozoneCluster.getOzoneManager();
      // To avoid interference from other test cases,
      // we collect number of existing keys at the beginning
-     int numOfExistedKeys = countKsmKeys(ksm);
+     int numOfExistedKeys = countOmKeys(ozoneManager);
  
      // Keep tracking bucket keys info while creating them
      PutHelper helper = new PutHelper(client, path);
@@@ -697,20 -694,17 +697,20 @@@
  
        // Memorize chunks that has been created,
        // so we can verify actual deletions at DN side later.
-       for (KsmKeyInfo keyInfo : createdKeys) {
-         List<KsmKeyLocationInfo> locations =
+       for (OmKeyInfo keyInfo : createdKeys) {
+         List<OmKeyLocationInfo> locations =
              keyInfo.getLatestVersionLocations().getLocationList();
-         for (KsmKeyLocationInfo location : locations) {
+         for (OmKeyLocationInfo location : locations) {
 -          KeyData keyData = new KeyData(location.getBlockID());
 -          KeyData blockInfo = cm.getContainerManager()
 -              .getKeyManager().getKey(keyData);
 -          ContainerData containerData = cm.getContainerManager()
 -              .readContainer(keyData.getContainerID());
 -          File dataDir = ContainerUtils
 -              .getDataDirectory(containerData).toFile();
 +          KeyValueHandler  keyValueHandler = (KeyValueHandler) cm
 +              .getDispatcher().getHandler(ContainerProtos.ContainerType
 +                  .KeyValueContainer);
 +          KeyValueContainer container = (KeyValueContainer) cm.getContainerSet()
 +              .getContainer(location.getBlockID().getContainerID());
 +          KeyData blockInfo = keyValueHandler
 +              .getKeyManager().getKey(container, location.getBlockID());
 +          KeyValueContainerData containerData = (KeyValueContainerData) container
 +              .getContainerData();
 +          File dataDir = new File(containerData.getChunksPath());
            for (ContainerProtos.ChunkInfo chunkInfo : blockInfo.getChunks()) {
              File chunkFile = dataDir.toPath()
                  .resolve(chunkInfo.getChunkName()).toFile();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message