From common-commits-return-85249-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Mon Jul 9 20:26:16 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 06099180676 for ; Mon, 9 Jul 2018 20:26:14 +0200 (CEST) Received: (qmail 22691 invoked by uid 500); 9 Jul 2018 18:25:56 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 21062 invoked by uid 99); 9 Jul 2018 18:25:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Jul 2018 18:25:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4FFE6E11D6; Mon, 9 Jul 2018 18:25:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: virajith@apache.org To: common-commits@hadoop.apache.org Date: Mon, 09 Jul 2018 18:26:33 -0000 Message-Id: In-Reply-To: <2566e4db169f4d678fd99ea25429929b@git.apache.org> References: <2566e4db169f4d678fd99ea25429929b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [41/50] [abbrv] hadoop git commit: HDDS-167. Rename KeySpaceManager to OzoneManager. Contributed by Arpit Agarwal. HDDS-167. Rename KeySpaceManager to OzoneManager. Contributed by Arpit Agarwal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/061b1685 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/061b1685 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/061b1685 Branch: refs/heads/HDFS-12090 Commit: 061b168529a9cd5d6a3a482c890bacdb49186368 Parents: e4bf38c Author: Arpit Agarwal Authored: Fri Jul 6 12:09:05 2018 -0700 Committer: Arpit Agarwal Committed: Fri Jul 6 12:09:05 2018 -0700 ---------------------------------------------------------------------- .gitignore | 4 + dev-support/bin/ozone-dist-layout-stitching | 2 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 6 +- .../src/main/compose/ozone/docker-compose.yaml | 6 +- .../src/main/compose/ozone/docker-config | 2 +- .../src/main/compose/ozoneperf/README.md | 4 +- .../main/compose/ozoneperf/docker-compose.yaml | 6 +- .../src/main/compose/ozoneperf/docker-config | 2 +- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- .../org/apache/hadoop/ozone/OzoneConsts.java | 22 +- .../org/apache/hadoop/ozone/common/Storage.java | 6 +- .../main/proto/ScmBlockLocationProtocol.proto | 10 +- hadoop-hdds/common/src/main/proto/hdds.proto | 6 +- .../common/src/main/resources/ozone-default.xml | 120 +- .../src/main/resources/webapps/static/ozone.js | 4 +- .../webapps/static/templates/config.html | 4 +- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 2 +- .../scm/container/states/ContainerStateMap.java | 6 +- .../hadoop/hdds/scm/node/CommandQueue.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 2 +- .../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 2 +- .../src/test/acceptance/basic/basic.robot | 6 +- .../test/acceptance/basic/docker-compose.yaml | 8 +- .../src/test/acceptance/basic/docker-config | 4 +- .../src/test/acceptance/basic/ozone-shell.robot | 18 +- .../src/test/acceptance/commonlib.robot | 4 +- .../test/acceptance/ozonefs/docker-compose.yaml | 8 +- .../src/test/acceptance/ozonefs/docker-config | 4 +- .../src/test/acceptance/ozonefs/ozonefs.robot | 6 +- .../apache/hadoop/ozone/client/BucketArgs.java | 4 +- .../hadoop/ozone/client/OzoneClientFactory.java | 86 +- .../apache/hadoop/ozone/client/OzoneKey.java | 2 +- .../apache/hadoop/ozone/client/VolumeArgs.java | 4 +- .../ozone/client/io/ChunkGroupInputStream.java | 24 +- .../ozone/client/io/ChunkGroupOutputStream.java | 54 +- .../client/rest/DefaultRestServerSelector.java | 2 +- .../hadoop/ozone/client/rest/RestClient.java | 15 +- .../ozone/client/rest/RestServerSelector.java | 2 +- .../hadoop/ozone/client/rpc/RpcClient.java | 142 +- .../ozone/client/TestHddsClientUtils.java | 24 +- hadoop-ozone/common/pom.xml | 2 +- hadoop-ozone/common/src/main/bin/ozone | 9 +- hadoop-ozone/common/src/main/bin/start-ozone.sh | 16 +- hadoop-ozone/common/src/main/bin/stop-ozone.sh | 16 +- .../java/org/apache/hadoop/ozone/KsmUtils.java | 95 -- .../java/org/apache/hadoop/ozone/OmUtils.java | 94 ++ .../apache/hadoop/ozone/freon/OzoneGetConf.java | 16 +- .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 -- .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 --- .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 --- .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 -- .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 277 ---- .../ozone/ksm/helpers/KsmKeyLocationInfo.java | 129 -- .../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 -- .../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 -- .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 --- .../ozone/ksm/helpers/OpenKeySession.java | 50 - .../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 --- .../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 -- .../hadoop/ozone/ksm/helpers/package-info.java | 18 - .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../ksm/protocol/KeySpaceManagerProtocol.java | 252 ---- .../hadoop/ozone/ksm/protocol/package-info.java | 19 - ...ceManagerProtocolClientSideTranslatorPB.java | 769 ---------- .../protocolPB/KeySpaceManagerProtocolPB.java | 34 - .../ozone/ksm/protocolPB/package-info.java | 19 - .../apache/hadoop/ozone/om/OMConfigKeys.java | 81 ++ .../hadoop/ozone/om/helpers/OmBucketArgs.java | 233 +++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 235 +++ .../hadoop/ozone/om/helpers/OmKeyArgs.java | 119 ++ .../hadoop/ozone/om/helpers/OmKeyInfo.java | 277 ++++ .../ozone/om/helpers/OmKeyLocationInfo.java | 129 ++ .../om/helpers/OmKeyLocationInfoGroup.java | 118 ++ .../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 110 ++ .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 223 +++ .../hadoop/ozone/om/helpers/OpenKeySession.java | 50 + .../hadoop/ozone/om/helpers/ServiceInfo.java | 237 +++ .../hadoop/ozone/om/helpers/VolumeArgs.java | 140 ++ .../hadoop/ozone/om/helpers/package-info.java | 18 + .../apache/hadoop/ozone/om/package-info.java | 21 + .../ozone/om/protocol/OzoneManagerProtocol.java | 252 ++++ .../hadoop/ozone/om/protocol/package-info.java | 19 + ...neManagerProtocolClientSideTranslatorPB.java | 769 ++++++++++ .../om/protocolPB/OzoneManagerProtocolPB.java | 34 + .../ozone/om/protocolPB/package-info.java | 19 + .../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 -- .../hadoop/ozone/protocolPB/OMPBHelper.java | 113 ++ .../main/proto/KeySpaceManagerProtocol.proto | 474 ------ .../src/main/proto/OzoneManagerProtocol.proto | 480 +++++++ hadoop-ozone/docs/content/GettingStarted.md | 18 +- hadoop-ozone/docs/content/Metrics.md | 10 +- hadoop-ozone/docs/content/_index.md | 12 +- hadoop-ozone/docs/static/OzoneOverview.svg | 2 +- .../apache/hadoop/ozone/MiniOzoneCluster.java | 24 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 66 +- .../ozone/TestOzoneConfigurationFields.java | 4 +- .../ozone/TestStorageContainerManager.java | 18 +- .../TestStorageContainerManagerHelper.java | 12 +- .../ozone/client/rest/TestOzoneRestClient.java | 6 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 18 +- .../commandhandler/TestBlockDeletion.java | 45 +- .../TestCloseContainerByPipeline.java | 35 +- .../TestCloseContainerHandler.java | 14 +- .../ozone/ksm/TestContainerReportWithKeys.java | 143 -- .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 306 ---- .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 284 ---- .../hadoop/ozone/ksm/TestKeySpaceManager.java | 1350 ------------------ .../ksm/TestKeySpaceManagerRestInterface.java | 135 -- .../ozone/ksm/TestKsmBlockVersioning.java | 253 ---- .../ksm/TestMultipleContainerReadWrite.java | 215 --- .../ozone/om/TestContainerReportWithKeys.java | 143 ++ .../om/TestMultipleContainerReadWrite.java | 215 +++ .../hadoop/ozone/om/TestOmBlockVersioning.java | 253 ++++ .../apache/hadoop/ozone/om/TestOmMetrics.java | 313 ++++ .../apache/hadoop/ozone/om/TestOmSQLCli.java | 284 ++++ .../hadoop/ozone/om/TestOzoneManager.java | 1349 +++++++++++++++++ .../ozone/om/TestOzoneManagerRestInterface.java | 135 ++ .../hadoop/ozone/ozShell/TestOzoneShell.java | 14 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 3 +- .../ozone/web/TestDistributedOzoneVolumes.java | 12 +- .../hadoop/ozone/web/client/TestKeys.java | 58 +- .../src/test/resources/webapps/ksm/.gitkeep | 15 - .../resources/webapps/ozoneManager/.gitkeep | 15 + .../server/datanode/ObjectStoreHandler.java | 33 +- .../ozone/web/handlers/KeyProcessTemplate.java | 4 +- .../web/handlers/VolumeProcessTemplate.java | 4 +- .../web/storage/DistributedStorageHandler.java | 153 +- .../apache/hadoop/ozone/ksm/BucketManager.java | 79 - .../hadoop/ozone/ksm/BucketManagerImpl.java | 315 ---- .../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 - .../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ---- .../ozone/ksm/KSMMetadataManagerImpl.java | 526 ------- .../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 459 ------ .../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 -- .../hadoop/ozone/ksm/KeyDeletingService.java | 142 -- .../org/apache/hadoop/ozone/ksm/KeyManager.java | 175 --- .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 566 -------- .../hadoop/ozone/ksm/KeySpaceManager.java | 912 ------------ .../ozone/ksm/KeySpaceManagerHttpServer.java | 78 - .../hadoop/ozone/ksm/OpenKeyCleanupService.java | 117 -- .../ozone/ksm/ServiceListJSONServlet.java | 103 -- .../apache/hadoop/ozone/ksm/VolumeManager.java | 100 -- .../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ----- .../ozone/ksm/exceptions/KSMException.java | 118 -- .../ozone/ksm/exceptions/package-info.java | 19 - .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../apache/hadoop/ozone/om/BucketManager.java | 79 + .../hadoop/ozone/om/BucketManagerImpl.java | 315 ++++ .../hadoop/ozone/om/KeyDeletingService.java | 142 ++ .../org/apache/hadoop/ozone/om/KeyManager.java | 175 +++ .../apache/hadoop/ozone/om/KeyManagerImpl.java | 566 ++++++++ .../org/apache/hadoop/ozone/om/OMMXBean.java | 31 + .../hadoop/ozone/om/OMMetadataManager.java | 253 ++++ .../org/apache/hadoop/ozone/om/OMMetrics.java | 459 ++++++ .../org/apache/hadoop/ozone/om/OMStorage.java | 90 ++ .../hadoop/ozone/om/OmMetadataManagerImpl.java | 526 +++++++ .../hadoop/ozone/om/OpenKeyCleanupService.java | 117 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 911 ++++++++++++ .../hadoop/ozone/om/OzoneManagerHttpServer.java | 78 + .../hadoop/ozone/om/ServiceListJSONServlet.java | 103 ++ .../apache/hadoop/ozone/om/VolumeManager.java | 100 ++ .../hadoop/ozone/om/VolumeManagerImpl.java | 390 +++++ .../hadoop/ozone/om/exceptions/OMException.java | 118 ++ .../ozone/om/exceptions/package-info.java | 19 + .../apache/hadoop/ozone/om/package-info.java | 21 + ...ceManagerProtocolServerSideTranslatorPB.java | 559 -------- ...neManagerProtocolServerSideTranslatorPB.java | 571 ++++++++ .../hadoop/ozone/protocolPB/package-info.java | 2 +- .../src/main/webapps/ksm/index.html | 70 - .../src/main/webapps/ksm/ksm-metrics.html | 44 - .../ozone-manager/src/main/webapps/ksm/ksm.js | 110 -- .../ozone-manager/src/main/webapps/ksm/main.css | 23 - .../src/main/webapps/ksm/main.html | 18 - .../src/main/webapps/ozoneManager/index.html | 70 + .../src/main/webapps/ozoneManager/main.css | 23 + .../src/main/webapps/ozoneManager/main.html | 18 + .../main/webapps/ozoneManager/om-metrics.html | 44 + .../main/webapps/ozoneManager/ozoneManager.js | 110 ++ .../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ----- .../hadoop/ozone/ksm/TestChunkStreams.java | 234 --- .../ksm/TestKeySpaceManagerHttpServer.java | 141 -- .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../hadoop/ozone/om/TestBucketManagerImpl.java | 394 +++++ .../hadoop/ozone/om/TestChunkStreams.java | 234 +++ .../ozone/om/TestOzoneManagerHttpServer.java | 141 ++ .../apache/hadoop/ozone/om/package-info.java | 21 + .../hadoop/fs/ozone/contract/OzoneContract.java | 4 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 48 +- 188 files changed, 13252 insertions(+), 13237 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 428950b..d555036 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,10 @@ patchprocess/ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log +# Ignore files generated by HDDS acceptance tests. +hadoop-ozone/acceptance-test/docker-compose.log +hadoop-ozone/acceptance-test/junit-results.xml + #robotframework outputs log.html output.xml http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/dev-support/bin/ozone-dist-layout-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching index be330d5..c30a37d 100755 --- a/dev-support/bin/ozone-dist-layout-stitching +++ b/dev-support/bin/ozone-dist-layout-stitching @@ -148,7 +148,7 @@ run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" mkdir -p "./share/hadoop/ozonefs" cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar" # Optional documentation, could be missing -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/ +cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/ cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index 6573a81..3826f67 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -404,13 +404,13 @@ esac # export HDFS_DFSROUTER_OPTS="" ### -# HDFS Key Space Manager specific parameters +# Ozone Manager specific parameters ### -# Specify the JVM options to be used when starting the HDFS Key Space Manager. +# Specify the JVM options to be used when starting the Ozone Manager. # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HDFS_KSM_OPTS="" +# export HDFS_OM_OPTS="" ### # HDFS StorageContainerManager specific parameters http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-dist/src/main/compose/ozone/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml index 512c649..bb5e8dd 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml @@ -25,17 +25,17 @@ services: command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - ./docker-config - ksm: + ozoneManager: image: apache/hadoop-runner volumes: - ../../ozone:/opt/hadoop ports: - 9874:9874 environment: - ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","ksm"] + command: ["/opt/hadoop/bin/ozone","om"] scm: image: apache/hadoop-runner volumes: http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-dist/src/main/compose/ozone/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config index 632f870..50abb18 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-config +++ b/hadoop-dist/src/main/compose/ozone/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.om.address=ozoneManager OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-dist/src/main/compose/ozoneperf/README.md ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-dist/src/main/compose/ozoneperf/README.md index a78f208..527ff41 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/README.md +++ b/hadoop-dist/src/main/compose/ozoneperf/README.md @@ -67,7 +67,7 @@ http://localhost:9090/graph Example queries: ``` -Hadoop_KeySpaceManager_NumKeyCommits -rate(Hadoop_KeySpaceManager_NumKeyCommits[10m]) +Hadoop_OzoneManager_NumKeyCommits +rate(Hadoop_OzoneManager_NumKeyCommits[10m]) rate(Hadoop_Ozone_BYTES_WRITTEN[10m]) ``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml index 3233c11..6d1d9ca 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -26,7 +26,7 @@ services: command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - ./docker-config - ksm: + ozoneManager: image: apache/hadoop-runner volumes: - ../../ozone:/opt/hadoop @@ -34,10 +34,10 @@ services: ports: - 9874:9874 environment: - ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","ksm"] + command: ["/opt/hadoop/bin/ozone","om"] scm: image: apache/hadoop-runner volumes: http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-dist/src/main/compose/ozoneperf/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config index 2be22a7..2539950 100644 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.om.address=ozoneManager OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index ad326dc..4f1b1c8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -94,7 +94,7 @@ public final class ScmConfigKeys { "ozone.scm.datanode.port"; public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861; - // OZONE_KSM_PORT_DEFAULT = 9862 + // OZONE_OM_PORT_DEFAULT = 9862 public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY = "ozone.scm.block.client.port"; public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863; http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 08a5ffd..4fad5d8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -93,7 +93,7 @@ public final class OzoneConsts { public static final String BLOCK_DB = "block.db"; public static final String OPEN_CONTAINERS_DB = "openContainers.db"; public static final String DELETED_BLOCK_DB = "deletedBlock.db"; - public static final String KSM_DB_NAME = "ksm.db"; + public static final String OM_DB_NAME = "om.db"; /** * Supports Bucket Versioning. @@ -119,13 +119,13 @@ public final class OzoneConsts { public static final String OPEN_KEY_ID_DELIMINATOR = "#"; /** - * KSM LevelDB prefixes. + * OM LevelDB prefixes. * - * KSM DB stores metadata as KV pairs with certain prefixes, + * OM DB stores metadata as KV pairs with certain prefixes, * prefix is used to improve the performance to get related * metadata. * - * KSM DB Schema: + * OM DB Schema: * ---------------------------------------------------------- * | KEY | VALUE | * ---------------------------------------------------------- @@ -140,13 +140,13 @@ public final class OzoneConsts { * | #deleting#/volumeName/bucketName/keyName | KeyInfo | * ---------------------------------------------------------- */ - public static final String KSM_VOLUME_PREFIX = "/#"; - public static final String KSM_BUCKET_PREFIX = "/#"; - public static final String KSM_KEY_PREFIX = "/"; - public static final String KSM_USER_PREFIX = "$"; + public static final String OM_VOLUME_PREFIX = "/#"; + public static final String OM_BUCKET_PREFIX = "/#"; + public static final String OM_KEY_PREFIX = "/"; + public static final String OM_USER_PREFIX = "$"; /** - * Max KSM Quota size of 1024 PB. + * Max OM Quota size of 1024 PB. */ public static final long MAX_QUOTA_IN_BYTES = 1024L * 1024 * TB; @@ -168,9 +168,9 @@ public final class OzoneConsts { public static final int INVALID_PORT = -1; - // The ServiceListJSONServlet context attribute where KeySpaceManager + // The ServiceListJSONServlet context attribute where OzoneManager // instance gets stored. - public static final String KSM_CONTEXT_ATTRIBUTE = "ozone.ksm"; + public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om"; private OzoneConsts() { // Never Constructed http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java index fb30d92..a32d559 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java @@ -38,7 +38,7 @@ import java.util.Properties; * Local storage information is stored in a separate file VERSION. * It contains type of the node, * the storage layout version, the SCM id, and - * the KSM/SCM state creation time. + * the OM/SCM state creation time. * */ @InterfaceAudience.Private @@ -127,7 +127,7 @@ public abstract class Storage { abstract protected Properties getNodeProperties(); /** - * Sets the Node properties spaecific to KSM/SCM. + * Sets the Node properties spaecific to OM/SCM. */ private void setNodeProperties() { Properties nodeProperties = getNodeProperties(); @@ -152,7 +152,7 @@ public abstract class Storage { * File {@code VERSION} contains the following fields: *
    *
  1. node type
    2. - *
  2. KSM/SCM state creation time
    3. + *
  3. OM/SCM state creation time
    4. *
  4. other fields specific for this node type
    5. *
* The version file is always written last during storage directory updates. http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto index 7bea82a..53f408a 100644 --- a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto @@ -46,7 +46,7 @@ message AllocateScmBlockRequestProto { } /** - * A delete key request sent by KSM to SCM, it contains + * A delete key request sent by OM to SCM, it contains * multiple number of keys (and their blocks). */ message DeleteScmKeyBlocksRequestProto { @@ -56,9 +56,9 @@ message DeleteScmKeyBlocksRequestProto { /** * A object key and all its associated blocks. * We need to encapsulate object key name plus the blocks in this potocol - * because SCM needs to response KSM with the keys it has deleted. + * because SCM needs to response OM with the keys it has deleted. * If the response only contains blocks, it will be very expensive for - * KSM to figure out what keys have been deleted. + * OM to figure out what keys have been deleted. */ message KeyBlocks { required string key = 1; @@ -66,7 +66,7 @@ message KeyBlocks { } /** - * A delete key response from SCM to KSM, it contains multiple child-results. + * A delete key response from SCM to OM, it contains multiple child-results. * Each child-result represents a key deletion result, only if all blocks of * a key are successfully deleted, this key result is considered as succeed. */ @@ -111,7 +111,7 @@ message AllocateScmBlockResponseProto { } /** - * Protocol used from KeySpaceManager to StorageContainerManager. + * Protocol used from OzoneManager to StorageContainerManager. * See request and response messages for details of the RPC calls. */ service ScmBlockLocationProtocolService { http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/common/src/main/proto/hdds.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto index b9def2a..a5ce994 100644 --- a/hadoop-hdds/common/src/main/proto/hdds.proto +++ b/hadoop-hdds/common/src/main/proto/hdds.proto @@ -58,9 +58,9 @@ message KeyValue { * Type of the node. */ enum NodeType { - KSM = 1; - SCM = 2; - DATANODE = 3; + OM = 1; // Ozone Manager + SCM = 2; // Storage Container Manager + DATANODE = 3; // DataNode } // Should we rename NodeState to DatanodeState? http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 568d267..530fb09 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -21,7 +21,7 @@ - + @@ -254,122 +254,122 @@ Tells ozone which storage handler to use. The possible values are: distributed - The Ozone distributed storage handler, which speaks to - KSM/SCM on the backend and provides REST services to clients. + OM/SCM on the backend and provides REST services to clients. local - Local Storage handler strictly for testing - To be removed. ozone.key.deleting.limit.per.task 1000 - KSM, PERFORMANCE + OM, PERFORMANCE A maximum number of keys to be scanned by key deleting service - per time interval in KSM. Those keys are sent to delete metadata and + per time interval in OM. Those keys are sent to delete metadata and generate transactions in SCM for next async deletion between SCM and DataNode. - ozone.ksm.address + ozone.om.address - KSM, REQUIRED + OM, REQUIRED - The address of the Ozone KSM service. This allows clients to discover - the KSMs address. + The address of the Ozone OM service. This allows clients to discover + the address of the OM. - ozone.ksm.group.rights + ozone.om.group.rights READ_WRITE - KSM, SECURITY + OM, SECURITY - Default group permissions in Ozone KSM. + Default group permissions in Ozone OM. - ozone.ksm.handler.count.key + ozone.om.handler.count.key 20 - KSM, PERFORMANCE + OM, PERFORMANCE - The number of RPC handler threads for KSM service endpoints. + The number of RPC handler threads for OM service endpoints. - ozone.ksm.http-address + ozone.om.http-address 0.0.0.0:9874 - KSM, MANAGEMENT + OM, MANAGEMENT - The address and the base port where the KSM web UI will listen on. + The address and the base port where the OM web UI will listen on. If the port is 0, then the server will start on a free port. However, it is best to specify a well-known port, so it is easy to connect and see - the KSM management UI. + the OM management UI. - ozone.ksm.http-bind-host + ozone.om.http-bind-host 0.0.0.0 - KSM, MANAGEMENT + OM, MANAGEMENT - The actual address the KSM web server will bind to. If this optional + The actual address the OM web server will bind to. If this optional the address is set, it overrides only the hostname portion of - ozone.ksm.http-address. + ozone.om.http-address. - ozone.ksm.http.enabled + ozone.om.http.enabled true - KSM, MANAGEMENT + OM, MANAGEMENT - Property to enable or disable KSM web user interface. + Property to enable or disable OM web user interface. - ozone.ksm.https-address + ozone.om.https-address 0.0.0.0:9875 - KSM, MANAGEMENT, SECURITY + OM, MANAGEMENT, SECURITY - The address and the base port where the KSM web UI will listen + The address and the base port where the OM web UI will listen on using HTTPS. If the port is 0 then the server will start on a free port. - ozone.ksm.https-bind-host + ozone.om.https-bind-host 0.0.0.0 - KSM, MANAGEMENT, SECURITY + OM, MANAGEMENT, SECURITY - The actual address the KSM web server will bind to using HTTPS. + The actual address the OM web server will bind to using HTTPS. If this optional address is set, it overrides only the hostname portion of - ozone.ksm.http-address. + ozone.om.http-address. - ozone.ksm.keytab.file + ozone.om.keytab.file - KSM, SECURITY + OM, SECURITY - The keytab file for Kerberos authentication in KSM. + The keytab file for Kerberos authentication in OM. - ozone.ksm.db.cache.size.mb + ozone.om.db.cache.size.mb 128 - KSM, PERFORMANCE + OM, PERFORMANCE - The size of KSM DB cache in MB that used for caching files. + The size of OM DB cache in MB that used for caching files. This value is set to an abnormally low value in the default configuration. That is to make unit testing easy. Generally, this value should be set to something like 16GB or more, if you intend to use Ozone at scale. - A large value for this key allows a proportionally larger amount of KSM - metadata to be cached in memory. This makes KSM operations faster. + A large value for this key allows a proportionally larger amount of OM + metadata to be cached in memory. This makes OM operations faster. - ozone.ksm.user.max.volume + ozone.om.user.max.volume 1024 - KSM, MANAGEMENT + OM, MANAGEMENT The maximum number of volumes a user can have on a cluster.Increasing or decreasing this number has no real impact on ozone cluster. This is @@ -379,11 +379,11 @@ - ozone.ksm.user.rights + ozone.om.user.rights READ_WRITE - KSM, SECURITY + OM, SECURITY - Default user permissions used in KSM. + Default user permissions used in OM. @@ -393,20 +393,20 @@ This is used only for testing purposes. This value is used by the local storage handler to simulate a REST backend. This is useful only when - debugging the REST front end independent of KSM and SCM. To be removed. + debugging the REST front end independent of OM and SCM. To be removed. ozone.metadata.dirs - OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE + OZONE, OM, SCM, CONTAINER, REQUIRED, STORAGE - Ozone metadata is shared among KSM, which acts as the namespace + Ozone metadata is shared among OM, which acts as the namespace manager for ozone, SCM which acts as the block manager and data nodes which maintain the name of the key(Key Name and BlockIDs). This replicated and distributed metadata store is maintained under the directory pointed by this key. Since metadata can be I/O intensive, at - least on KSM and SCM we recommend having SSDs. If you have the luxury + least on OM and SCM we recommend having SSDs. If you have the luxury of mapping this path to SSDs on all machines in the cluster, that will be excellent. @@ -417,10 +417,10 @@ ozone.metastore.impl RocksDB - OZONE, KSM, SCM, CONTAINER, STORAGE + OZONE, OM, SCM, CONTAINER, STORAGE Ozone metadata store implementation. Ozone metadata are well - distributed to multiple services such as ksm, scm. They are stored in + distributed to multiple services such as ozoneManager, scm. They are stored in some local key-value databases. This property determines which database library to use. Supported value is either LevelDB or RocksDB. @@ -429,7 +429,7 @@ ozone.metastore.rocksdb.statistics ALL - OZONE, KSM, SCM, STORAGE, PERFORMANCE + OZONE, OM, SCM, STORAGE, PERFORMANCE The statistics level of the rocksdb store. If you use any value from org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb @@ -672,7 +672,7 @@ The heartbeat interval from a data node to SCM. Yes, it is not three but 30, since most data nodes will heart beating via Ratis heartbeats. If a client is not able to talk to a data node, it will notify - KSM/SCM eventually. So a 30 second HB seems to work. This assumes that + OM/SCM eventually. So a 30 second HB seems to work. This assumes that replication strategy used is Ratis if not, this value should be set to something smaller like 3 seconds. @@ -808,7 +808,7 @@ OZONE, SECURITY - The server principal used by the SCM and KSM for web UI SPNEGO + The server principal used by the SCM and OM for web UI SPNEGO authentication when Kerberos security is enabled. This is typically set to HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix HTTP/ by convention. @@ -867,9 +867,9 @@ ozone.key.preallocation.maxsize 134217728 - OZONE, KSM, PERFORMANCE + OZONE, OM, PERFORMANCE - When a new key write request is sent to KSM, if a size is requested, at most + When a new key write request is sent to OM, if a size is requested, at most 128MB of size is allocated at request time. If client needs more space for the write, separate block allocation requests will be made. @@ -938,7 +938,7 @@ ozone.open.key.cleanup.service.interval.seconds 86400 - OZONE, KSM, PERFORMANCE + OZONE, OM, PERFORMANCE A background job periodically checks open key entries and delete the expired ones. This entry controls the interval of this cleanup check. @@ -948,7 +948,7 @@ ozone.open.key.expire.threshold 86400 - OZONE, KSM, PERFORMANCE + OZONE, OM, PERFORMANCE Controls how long an open key operation is considered active. Specifically, if a key has been open longer than the value of this config entry, that open key is considered as @@ -958,12 +958,12 @@ hadoop.tags.custom - OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,KSM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE + OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE ozone.tags.system - OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,KSM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE + OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js ---------------------------------------------------------------------- diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js b/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js index 411438a..c2ed2ad 100644 --- a/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js +++ b/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js @@ -270,7 +270,7 @@ $http.get("conf?cmd=getOzoneTags") .then(function(response) { ctrl.tags = response.data; - var excludedTags = ['CBLOCK', 'KSM', 'SCM']; + var excludedTags = ['CBLOCK', 'OM', 'SCM']; for (var i = 0; i < excludedTags.length; i++) { var idx = ctrl.tags.indexOf(excludedTags[i]); // Remove CBLOCK related properties @@ -302,7 +302,7 @@ } ctrl.loadAll = function() { - $http.get("conf?cmd=getPropertyByTag&tags=KSM,SCM," + ctrl.tags) + $http.get("conf?cmd=getPropertyByTag&tags=OM,SCM," + ctrl.tags) .then(function(response) { ctrl.convertToArray(response.data); http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/framework/src/main/resources/webapps/static/templates/config.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/config.html b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/config.html index 6825750..b52f653 100644 --- a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/config.html +++ b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/config.html @@ -27,8 +27,8 @@ ng-click="$ctrl.switchto('All')">All KSM + ng-class="$ctrl.allSelected('OM') ? 'btn-primary' :'btn-secondary'" + ng-click="$ctrl.switchto('OM')">OM SCM http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java index 4f4c755..28103be 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java @@ -28,7 +28,7 @@ import java.util.Map; /** * The DeletedBlockLog is a persisted log in SCM to keep tracking * container blocks which are under deletion. It maintains info - * about under-deletion container blocks that notified by KSM, + * about under-deletion container blocks that notified by OM, * and the state how it is processed. */ public interface DeletedBlockLog extends Closeable { http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java index 3ada8fe..c23b1fd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java @@ -53,9 +53,9 @@ import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes * client to able to write to it. *

* 2. Owners - Each instance of Name service, for example, Namenode of HDFS or - * Key Space Manager (KSM) of Ozone or CBlockServer -- is an owner. It is - * possible to have many KSMs for a Ozone cluster and only one SCM. But SCM - * keeps the data from each KSM in separate bucket, never mixing them. To + * Ozone Manager (OM) of Ozone or CBlockServer -- is an owner. It is + * possible to have many OMs for a Ozone cluster and only one SCM. But SCM + * keeps the data from each OM in separate bucket, never mixing them. To * write data, often we have to find all open containers for a specific owner. *

* 3. ReplicationType - The clients are allowed to specify what kind of http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java index edbcfa1..996478c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java @@ -33,7 +33,7 @@ import java.util.concurrent.locks.ReentrantLock; /** * Command Queue is queue of commands for the datanode. *

- * Node manager, container Manager and key space managers can queue commands for + * Node manager, container Manager and Ozone managers can queue commands for * datanodes into this queue. These commands will be send in the order in which * there where queued. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 98fe9a1..3bb284e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -152,7 +152,7 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol { @Override public List deleteKeyBlocks( List keyBlocksInfoList) throws IOException { - LOG.info("SCM is informed by KSM to delete {} blocks", keyBlocksInfoList + LOG.info("SCM is informed by OM to delete {} blocks", keyBlocksInfoList .size()); List results = new ArrayList<>(); for (BlockGroup keyBlocks : keyBlocksInfoList) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/OzoneBaseCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/OzoneBaseCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/OzoneBaseCLI.java index 727c81a..7828445 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/OzoneBaseCLI.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/OzoneBaseCLI.java @@ -27,7 +27,7 @@ import java.io.IOException; import java.net.URISyntaxException; /** - * This class is the base CLI for scm, ksm and scmadm. + * This class is the base CLI for scm, om and scmadm. */ public abstract class OzoneBaseCLI extends Configured implements Tool { http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot index c741588..6d6fea0 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot @@ -39,12 +39,12 @@ Test rest interface Should contain ${result} 200 OK Check webui static resources - ${result} = Execute on scm curl -s -I http://localhost:9876/static/bootstrap-3.3.7/js/bootstrap.min.js + ${result} = Execute on scm curl -s -I http://localhost:9876/static/bootstrap-3.3.7/js/bootstrap.min.js Should contain ${result} 200 - ${result} = Execute on ksm curl -s -I http://localhost:9874/static/bootstrap-3.3.7/js/bootstrap.min.js + ${result} = Execute on ozoneManager curl -s -I http://localhost:9874/static/bootstrap-3.3.7/js/bootstrap.min.js Should contain ${result} 200 Start freon testing - ${result} = Execute on ksm ozone freon -numOfVolumes 5 -numOfBuckets 5 -numOfKeys 5 -numOfThreads 10 + ${result} = Execute on ozoneManager ozone freon -numOfVolumes 5 -numOfBuckets 5 -numOfKeys 5 -numOfThreads 10 Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125 Should Not Contain ${result} ERROR http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml index b50f42d..99f2831 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml @@ -25,18 +25,18 @@ services: command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - ./docker-config - ksm: + ozoneManager: image: apache/hadoop-runner - hostname: ksm + hostname: ozoneManager volumes: - ${OZONEDIR}:/opt/hadoop ports: - 9874 environment: - ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","ksm"] + command: ["/opt/hadoop/bin/ozone","om"] scm: image: apache/hadoop-runner volumes: http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config index c3ec2ef..b72085b 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.ksm.address=ksm -OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874 +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot index 9521ad6..f4be3e0 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot @@ -28,22 +28,22 @@ ${PROJECTDIR} ${CURDIR}/../../../../../.. *** Test Cases *** RestClient without http port - Test ozone shell http:// ksm restwoport True + Test ozone shell http:// ozoneManager restwoport True RestClient with http port - Test ozone shell http:// ksm:9874 restwport True + Test ozone shell http:// ozoneManager:9874 restwport True RestClient without host name - Test ozone shell http:// ${EMPTY} restwohost True + Test ozone shell http:// ${EMPTY} restwohost True RpcClient with port - Test ozone shell o3:// ksm:9862 rpcwoport False + Test ozone shell o3:// ozoneManager:9862 rpcwoport False RpcClient without host - Test ozone shell o3:// ${EMPTY} rpcwport False + Test ozone shell o3:// ${EMPTY} rpcwport False RpcClient without scheme - Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme False + Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme False *** Keywords *** @@ -52,7 +52,7 @@ Test ozone shell ${result} = Execute on datanode ozone oz -createVolume ${protocol}${server}/${volume} -user bilbo -quota 100TB -root Should not contain ${result} Failed Should contain ${result} Creating Volume: ${volume} - ${result} = Execute on datanode ozone oz -listVolume o3://ksm -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")' + ${result} = Execute on datanode ozone oz -listVolume o3://ozoneManager -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")' Should contain ${result} createdOn Execute on datanode ozone oz -updateVolume ${protocol}${server}/${volume} -user bill -quota 10TB ${result} = Execute on datanode ozone oz -infoVolume ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name' @@ -66,7 +66,7 @@ Test ozone shell Should Be Equal ${result} GROUP ${result} = Execute on datanode ozone oz -updateBucket ${protocol}${server}/${volume}/bb1 -removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type' Should Be Equal ${result} USER - ${result} = Execute on datanode ozone oz -listBucket o3://ksm/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName' + ${result} = Execute on datanode ozone oz -listBucket o3://ozoneManager/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName' Should Be Equal ${result} ${volume} Run Keyword and Return If ${withkeytest} Test key handling ${protocol} ${server} ${volume} Execute on datanode ozone oz -deleteBucket ${protocol}${server}/${volume}/bb1 @@ -80,6 +80,6 @@ Test key handling Execute on datanode ls -l NOTICE.txt.1 ${result} = Execute on datanode ozone oz -infoKey ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")' Should contain ${result} createdOn - ${result} = Execute on datanode ozone oz -listKey o3://ksm/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName' + ${result} = Execute on datanode ozone oz -listKey o3://ozoneManager/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName' Should Be Equal ${result} key1 Execute on datanode ozone oz -deleteKey ${protocol}${server}/${volume}/bb1/key1 -v http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot index a5ea30a..9235cd9 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot @@ -21,12 +21,12 @@ Startup Ozone cluster with size Run echo "Starting new docker-compose environment" >> docker-compose.log ${rc} ${output} = Run docker compose up -d Should Be Equal As Integers ${rc} 0 - Wait Until Keyword Succeeds 1min 5sec Is Daemon started ksm HTTP server of KSM is listening + Wait Until Keyword Succeeds 1min 5sec Is Daemon started ozoneManager HTTP server of OZONEMANAGER is listening Daemons are running without error Scale datanodes up 5 Daemons are running without error - Is daemon running without error ksm + Is daemon running without error ozoneManager Is daemon running without error scm Is daemon running without error datanode http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml index 12022df..6b7b7bd 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml @@ -25,18 +25,18 @@ services: command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - ./docker-config - ksm: + ozoneManager: image: apache/hadoop-runner - hostname: ksm + hostname: ozoneManager volumes: - ${OZONEDIR}:/opt/hadoop ports: - 9874 environment: - ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","ksm"] + command: ["/opt/hadoop/bin/ozone","om"] scm: image: apache/hadoop-runner volumes: http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config index e06d434..b0129bc 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config @@ -15,8 +15,8 @@ # limitations under the License. CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem -OZONE-SITE.XML_ozone.ksm.address=ksm -OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874 +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot index 9e8a5d2..ea473c0 100644 --- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot @@ -27,13 +27,13 @@ ${PROJECTDIR} ${CURDIR}/../../../../../.. *** Test Cases *** Create volume and bucket - Execute on datanode ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root - Execute on datanode ozone oz -createBucket http://ksm/fstest/bucket1 + Execute on datanode ozone oz -createVolume http://ozoneManager/fstest -user bilbo -quota 100TB -root + Execute on datanode ozone oz -createBucket http://ozoneManager/fstest/bucket1 Check volume from ozonefs ${result} = Execute on hadooplast hdfs dfs -ls o3://bucket1.fstest/ Create directory from ozonefs Execute on hadooplast hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep - ${result} = Execute on ksm ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + ${result} = Execute on ozoneManager ozone oz -listKey o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} testdir/deep http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java index 39b7bb8..0da52dc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java @@ -81,7 +81,7 @@ public final class BucketArgs { } /** - * Returns new builder class that builds a KsmBucketInfo. + * Returns new builder class that builds a OmBucketInfo. * * @return Builder */ @@ -90,7 +90,7 @@ public final class BucketArgs { } /** - * Builder for KsmBucketInfo. + * Builder for OmBucketInfo. */ public static class Builder { private Boolean versioning; http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java index 3085b0d..de0d166 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java @@ -21,7 +21,7 @@ package org.apache.hadoop.ozone.client; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.KsmUtils; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.client.rest.RestClient; import org.apache.hadoop.ozone.client.rpc.RpcClient; @@ -34,11 +34,9 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_CLIENT_PROTOCOL; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys - .OZONE_KSM_HTTP_ADDRESS_KEY; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_PROTOCOL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY; /** * Factory class to create different types of OzoneClients. @@ -97,46 +95,46 @@ public final class OzoneClientFactory { /** * Returns an OzoneClient which will use RPC protocol. * - * @param ksmHost - * hostname of KeySpaceManager to connect. + * @param omHost + * hostname of OzoneManager to connect. * * @return OzoneClient * * @throws IOException */ - public static OzoneClient getRpcClient(String ksmHost) + public static OzoneClient getRpcClient(String omHost) throws IOException { Configuration config = new OzoneConfiguration(); - int port = KsmUtils.getKsmRpcPort(config); - return getRpcClient(ksmHost, port, config); + int port = OmUtils.getOmRpcPort(config); + return getRpcClient(omHost, port, config); } /** * Returns an OzoneClient which will use RPC protocol. * - * @param ksmHost - * hostname of KeySpaceManager to connect. + * @param omHost + * hostname of OzoneManager to connect. * - * @param ksmRpcPort - * RPC port of KeySpaceManager. + * @param omRpcPort + * RPC port of OzoneManager. * * @return OzoneClient * * @throws IOException */ - public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort) + public static OzoneClient getRpcClient(String omHost, Integer omRpcPort) throws IOException { - return getRpcClient(ksmHost, ksmRpcPort, new OzoneConfiguration()); + return getRpcClient(omHost, omRpcPort, new OzoneConfiguration()); } /** * Returns an OzoneClient which will use RPC protocol. * - * @param ksmHost - * hostname of KeySpaceManager to connect. + * @param omHost + * hostname of OzoneManager to connect. * - * @param ksmRpcPort - * RPC port of KeySpaceManager. + * @param omRpcPort + * RPC port of OzoneManager. * * @param config * Configuration to be used for OzoneClient creation @@ -145,13 +143,13 @@ public final class OzoneClientFactory { * * @throws IOException */ - public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort, + public static OzoneClient getRpcClient(String omHost, Integer omRpcPort, Configuration config) throws IOException { - Preconditions.checkNotNull(ksmHost); - Preconditions.checkNotNull(ksmRpcPort); + Preconditions.checkNotNull(omHost); + Preconditions.checkNotNull(omRpcPort); Preconditions.checkNotNull(config); - config.set(OZONE_KSM_ADDRESS_KEY, ksmHost + ":" + ksmRpcPort); + config.set(OZONE_OM_ADDRESS_KEY, omHost + ":" + omRpcPort); return getRpcClient(config); } @@ -175,46 +173,46 @@ public final class OzoneClientFactory { /** * Returns an OzoneClient which will use REST protocol. * - * @param ksmHost - * hostname of KeySpaceManager to connect. + * @param omHost + * hostname of OzoneManager to connect. * * @return OzoneClient * * @throws IOException */ - public static OzoneClient getRestClient(String ksmHost) + public static OzoneClient getRestClient(String omHost) throws IOException { Configuration config = new OzoneConfiguration(); - int port = KsmUtils.getKsmRestPort(config); - return getRestClient(ksmHost, port, config); + int port = OmUtils.getOmRestPort(config); + return getRestClient(omHost, port, config); } /** * Returns an OzoneClient which will use REST protocol. * - * @param ksmHost - * hostname of KeySpaceManager to connect. + * @param omHost + * hostname of OzoneManager to connect. * - * @param ksmHttpPort - * HTTP port of KeySpaceManager. + * @param omHttpPort + * HTTP port of OzoneManager. * * @return OzoneClient * * @throws IOException */ - public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort) + public static OzoneClient getRestClient(String omHost, Integer omHttpPort) throws IOException { - return getRestClient(ksmHost, ksmHttpPort, new OzoneConfiguration()); + return getRestClient(omHost, omHttpPort, new OzoneConfiguration()); } /** * Returns an OzoneClient which will use REST protocol. * - * @param ksmHost - * hostname of KeySpaceManager to connect. + * @param omHost + * hostname of OzoneManager to connect. * - * @param ksmHttpPort - * HTTP port of KeySpaceManager. + * @param omHttpPort + * HTTP port of OzoneManager. * * @param config * Configuration to be used for OzoneClient creation @@ -223,13 +221,13 @@ public final class OzoneClientFactory { * * @throws IOException */ - public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort, + public static OzoneClient getRestClient(String omHost, Integer omHttpPort, Configuration config) throws IOException { - Preconditions.checkNotNull(ksmHost); - Preconditions.checkNotNull(ksmHttpPort); + Preconditions.checkNotNull(omHost); + Preconditions.checkNotNull(omHttpPort); Preconditions.checkNotNull(config); - config.set(OZONE_KSM_HTTP_ADDRESS_KEY, ksmHost + ":" + ksmHttpPort); + config.set(OZONE_OM_HTTP_ADDRESS_KEY, omHost + ":" + omHttpPort); return getRestClient(config); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 0c723dd..7c93146 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -49,7 +49,7 @@ public class OzoneKey { private long modificationTime; /** - * Constructs OzoneKey from KsmKeyInfo. + * Constructs OzoneKey from OmKeyInfo. * */ public OzoneKey(String volumeName, String bucketName, http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java index f1aa031..ae1cfcc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java @@ -77,7 +77,7 @@ public final class VolumeArgs { return acls; } /** - * Returns new builder class that builds a KsmVolumeArgs. + * Returns new builder class that builds a OmVolumeArgs. * * @return Builder */ @@ -86,7 +86,7 @@ public final class VolumeArgs { } /** - * Builder for KsmVolumeArgs. + * Builder for OmVolumeArgs. */ public static class Builder { private String adminName; http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java index edd85aa..b3a566e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java @@ -23,8 +23,8 @@ import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; @@ -255,28 +255,29 @@ public class ChunkGroupInputStream extends InputStream implements Seekable { } } - public static LengthInputStream getFromKsmKeyInfo(KsmKeyInfo keyInfo, + public static LengthInputStream getFromOmKeyInfo( + OmKeyInfo keyInfo, XceiverClientManager xceiverClientManager, StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient, String requestId) - throws IOException { + storageContainerLocationClient, + String requestId) throws IOException { long length = 0; long containerKey; ChunkGroupInputStream groupInputStream = new ChunkGroupInputStream(); groupInputStream.key = keyInfo.getKeyName(); - List keyLocationInfos = + List keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly(); groupInputStream.streamOffset = new long[keyLocationInfos.size()]; for (int i = 0; i < keyLocationInfos.size(); i++) { - KsmKeyLocationInfo ksmKeyLocationInfo = keyLocationInfos.get(i); - BlockID blockID = ksmKeyLocationInfo.getBlockID(); + OmKeyLocationInfo omKeyLocationInfo = keyLocationInfos.get(i); + BlockID blockID = omKeyLocationInfo.getBlockID(); long containerID = blockID.getContainerID(); ContainerWithPipeline containerWithPipeline = storageContainerLocationClient.getContainerWithPipeline(containerID); XceiverClientSpi xceiverClient = xceiverClientManager .acquireClient(containerWithPipeline.getPipeline(), containerID); boolean success = false; - containerKey = ksmKeyLocationInfo.getLocalID(); + containerKey = omKeyLocationInfo.getLocalID(); try { LOG.debug("get key accessing {} {}", containerID, containerKey); @@ -292,11 +293,10 @@ public class ChunkGroupInputStream extends InputStream implements Seekable { } success = true; ChunkInputStream inputStream = new ChunkInputStream( - ksmKeyLocationInfo.getBlockID(), xceiverClientManager, - xceiverClient, + omKeyLocationInfo.getBlockID(), xceiverClientManager, xceiverClient, chunks, requestId); groupInputStream.addStream(inputStream, - ksmKeyLocationInfo.getLength()); + omKeyLocationInfo.getLength()); } finally { if (!success) { xceiverClientManager.releaseClient(xceiverClient); http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java index d1a3b46..9443317 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java @@ -24,15 +24,15 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo; -import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession; -import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolClientSideTranslatorPB; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; +import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers @@ -67,10 +67,10 @@ public class ChunkGroupOutputStream extends OutputStream { private final ArrayList streamEntries; private int currentStreamIndex; private long byteOffset; - private final KeySpaceManagerProtocolClientSideTranslatorPB ksmClient; + private final OzoneManagerProtocolClientSideTranslatorPB omClient; private final StorageContainerLocationProtocolClientSideTranslatorPB scmClient; - private final KsmKeyArgs keyArgs; + private final OmKeyArgs keyArgs; private final int openID; private final XceiverClientManager xceiverClientManager; private final int chunkSize; @@ -83,7 +83,7 @@ public class ChunkGroupOutputStream extends OutputStream { @VisibleForTesting public ChunkGroupOutputStream() { streamEntries = new ArrayList<>(); - ksmClient = null; + omClient = null; scmClient = null; keyArgs = null; openID = -1; @@ -113,16 +113,16 @@ public class ChunkGroupOutputStream extends OutputStream { public ChunkGroupOutputStream( OpenKeySession handler, XceiverClientManager xceiverClientManager, StorageContainerLocationProtocolClientSideTranslatorPB scmClient, - KeySpaceManagerProtocolClientSideTranslatorPB ksmClient, + OzoneManagerProtocolClientSideTranslatorPB omClient, int chunkSize, String requestId, ReplicationFactor factor, ReplicationType type) throws IOException { this.streamEntries = new ArrayList<>(); this.currentStreamIndex = 0; this.byteOffset = 0; - this.ksmClient = ksmClient; + this.omClient = omClient; this.scmClient = scmClient; - KsmKeyInfo info = handler.getKeyInfo(); - this.keyArgs = new KsmKeyArgs.Builder() + OmKeyInfo info = handler.getKeyInfo(); + this.keyArgs = new OmKeyArgs.Builder() .setVolumeName(info.getVolumeName()) .setBucketName(info.getBucketName()) .setKeyName(info.getKeyName()) @@ -150,19 +150,19 @@ public class ChunkGroupOutputStream extends OutputStream { * @param openVersion the version corresponding to the pre-allocation. * @throws IOException */ - public void addPreallocateBlocks(KsmKeyLocationInfoGroup version, + public void addPreallocateBlocks(OmKeyLocationInfoGroup version, long openVersion) throws IOException { // server may return any number of blocks, (0 to any) // only the blocks allocated in this open session (block createVersion // equals to open session version) - for (KsmKeyLocationInfo subKeyInfo : version.getLocationList()) { + for (OmKeyLocationInfo subKeyInfo : version.getLocationList()) { if (subKeyInfo.getCreateVersion() == openVersion) { checkKeyLocationInfo(subKeyInfo); } } } - private void checkKeyLocationInfo(KsmKeyLocationInfo subKeyInfo) + private void checkKeyLocationInfo(OmKeyLocationInfo subKeyInfo) throws IOException { ContainerWithPipeline containerWithPipeline = scmClient .getContainerWithPipeline(subKeyInfo.getContainerID()); @@ -210,7 +210,7 @@ public class ChunkGroupOutputStream extends OutputStream { checkNotClosed(); if (streamEntries.size() <= currentStreamIndex) { - Preconditions.checkNotNull(ksmClient); + Preconditions.checkNotNull(omClient); // allocate a new block, if a exception happens, log an error and // throw exception to the caller directly, and the write fails. try { @@ -258,7 +258,7 @@ public class ChunkGroupOutputStream extends OutputStream { int succeededAllocates = 0; while (len > 0) { if (streamEntries.size() <= currentStreamIndex) { - Preconditions.checkNotNull(ksmClient); + Preconditions.checkNotNull(omClient); // allocate a new block, if a exception happens, log an error and // throw exception to the caller directly, and the write fails. try { @@ -286,7 +286,7 @@ public class ChunkGroupOutputStream extends OutputStream { } /** - * Contact KSM to get a new block. Set the new block with the index (e.g. + * Contact OM to get a new block. Set the new block with the index (e.g. * first block has index = 0, second has index = 1 etc.) * * The returned block is made to new ChunkOutputStreamEntry to write. @@ -295,7 +295,7 @@ public class ChunkGroupOutputStream extends OutputStream { * @throws IOException */ private void allocateNewBlock(int index) throws IOException { - KsmKeyLocationInfo subKeyInfo = ksmClient.allocateBlock(keyArgs, openID); + OmKeyLocationInfo subKeyInfo = omClient.allocateBlock(keyArgs, openID); checkKeyLocationInfo(subKeyInfo); } @@ -311,7 +311,7 @@ public class ChunkGroupOutputStream extends OutputStream { } /** - * Commit the key to KSM, this will add the blocks as the new key blocks. + * Commit the key to OM, this will add the blocks as the new key blocks. * * @throws IOException */ @@ -329,7 +329,7 @@ public class ChunkGroupOutputStream extends OutputStream { if (keyArgs != null) { // in test, this could be null keyArgs.setDataSize(byteOffset); - ksmClient.commitKey(keyArgs, openID); + omClient.commitKey(keyArgs, openID); } else { LOG.warn("Closing ChunkGroupOutputStream, but key args is null"); } @@ -342,7 +342,7 @@ public class ChunkGroupOutputStream extends OutputStream { private OpenKeySession openHandler; private XceiverClientManager xceiverManager; private StorageContainerLocationProtocolClientSideTranslatorPB scmClient; - private KeySpaceManagerProtocolClientSideTranslatorPB ksmClient; + private OzoneManagerProtocolClientSideTranslatorPB omClient; private int chunkSize; private String requestID; private ReplicationType type; @@ -364,9 +364,9 @@ public class ChunkGroupOutputStream extends OutputStream { return this; } - public Builder setKsmClient( - KeySpaceManagerProtocolClientSideTranslatorPB client) { - this.ksmClient = client; + public Builder setOmClient( + OzoneManagerProtocolClientSideTranslatorPB client) { + this.omClient = client; return this; } @@ -392,7 +392,7 @@ public class ChunkGroupOutputStream extends OutputStream { public ChunkGroupOutputStream build() throws IOException { return new ChunkGroupOutputStream(openHandler, xceiverManager, scmClient, - ksmClient, chunkSize, requestID, factor, type); + omClient, chunkSize, requestID, factor, type); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java index 93b3417..abdc2fb 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.client.rest; -import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo; +import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import java.util.List; import java.util.Random; http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java index 6e3f617..78fbe8d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java @@ -45,10 +45,9 @@ import org.apache.hadoop.ozone.client.rest.headers.Header; import org.apache.hadoop.ozone.client.rest.response.BucketInfo; import org.apache.hadoop.ozone.client.rest.response.KeyInfo; import org.apache.hadoop.ozone.client.rest.response.VolumeInfo; -import org.apache.hadoop.ozone.ksm.KSMConfigKeys; -import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.ServicePort; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.ServiceInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort; import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.ListKeys; import org.apache.hadoop.ozone.web.response.ListVolumes; @@ -152,8 +151,8 @@ public class RestClient implements ClientProtocol { .build()) .build(); this.ugi = UserGroupInformation.getCurrentUser(); - this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS, - KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT); + this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS, + OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT); // TODO: Add new configuration parameter to configure RestServerSelector. RestServerSelector defaultSelector = new DefaultRestServerSelector(); @@ -171,11 +170,11 @@ public class RestClient implements ClientProtocol { private InetSocketAddress getOzoneRestServerAddress( RestServerSelector selector) throws IOException { - String httpAddress = conf.get(KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY); + String httpAddress = conf.get(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY); if (httpAddress == null) { throw new IllegalArgumentException( - KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY + " must be defined. See" + + OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY + " must be defined. See" + " https://wiki.apache.org/hadoop/Ozone#Configuration for" + " details on configuring Ozone."); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org