hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [44/51] [partial] hadoop git commit: HDFS-13405. Ozone: Rename HDSL to HDDS. Contributed by Ajay Kumar, Elek Marton, Mukul Kumar Singh, Shashikant Banerjee and Anu Engineer.
Date Thu, 05 Apr 2018 18:33:41 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
new file mode 100644
index 0000000..8018d29
--- /dev/null
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -0,0 +1,1031 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!-- Do not modify this file directly.  Instead, copy entries that you -->
+<!-- wish to modify from this file into ozone-site.xml and change them -->
+<!-- there.  If ozone-site.xml does not already exist, create it.      -->
+
+<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE,   -->
+<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
+<!--REST, STORAGE, PIPELINE, STANDALONE                                    -->
+
+<configuration>
+
+  <!--Container Settings used by Datanode-->
+  <property>
+    <name>ozone.container.cache.size</name>
+    <value>1024</value>
+    <tag>PERFORMANCE, CONTAINER, STORAGE</tag>
+    <description>The open container is cached on the data node side. We maintain
+      an LRU
+      cache for caching the recently used containers. This setting controls the
+      size of that cache.
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ipc</name>
+    <value>9859</value>
+    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+    <description>The ipc port number of container.</description>
+  </property>
+  <property>
+    <name>dfs.container.ipc.random.port</name>
+    <value>false</value>
+    <tag>OZONE, DEBUG, CONTAINER</tag>
+    <description>Allocates a random free port for ozone container. This is used
+      only while
+      running unit tests.
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.datanode.storage.dir</name>
+    <value/>
+    <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
+    <description>This directory is used for storing Ratis metadata like logs. If
+      this is
+      not set then default metadata dirs is used. A warning will be logged if
+      this not set. Ideally, this should be mapped to a fast disk like an SSD.
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.enabled</name>
+    <value>false</value>
+    <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
+    <description>Ozone supports different kinds of replication pipelines. Ratis
+      is one of
+      the replication pipeline supported by ozone.
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.ipc</name>
+    <value>9858</value>
+    <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
+    <description>The ipc port number of container.</description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.ipc.random.port</name>
+    <value>false</value>
+    <tag>OZONE,DEBUG</tag>
+    <description>Allocates a random free port for ozone ratis port for the
+      container. This
+      is used only while running unit tests.
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.rpc.type</name>
+    <value>GRPC</value>
+    <tag>OZONE, RATIS, MANAGEMENT</tag>
+    <description>Ratis supports different kinds of transports like netty, GRPC,
+      Hadoop RPC
+      etc. This picks one of those for this cluster.
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.num.write.chunk.threads</name>
+    <value>60</value>
+    <tag>OZONE, RATIS, PERFORMANCE</tag>
+    <description>Maximum number of threads in the thread pool that Ratis
+      will use for writing chunks (60 by default).
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.segment.size</name>
+    <value>1073741824</value>
+    <tag>OZONE, RATIS, PERFORMANCE</tag>
+    <description>The size of the raft segment used by Apache Ratis on datanodes.
+      (1 GB by default)
+    </description>
+  </property>
+  <property>
+    <name>dfs.container.ratis.segment.preallocated.size</name>
+    <value>134217728</value>
+    <tag>OZONE, RATIS, PERFORMANCE</tag>
+    <description>The size of the buffer which is preallocated for raft segment
+      used by Apache Ratis on datanodes.(128 MB by default)
+    </description>
+  </property>
+  <property>
+    <name>ozone.container.report.interval</name>
+    <value>60000ms</value>
+    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+    <description>Time interval of the datanode to send container report. Each
+      datanode periodically send container report upon receive
+      sendContainerReport from SCM. Unit could be defined with
+      postfix (ns,ms,s,m,h,d)</description>
+  </property>
+  <!--Ozone Settings-->
+  <property>
+    <name>ozone.administrators</name>
+    <value/>
+    <tag>OZONE, SECURITY</tag>
+    <description>Ozone administrator users delimited by the comma.
+      If not set, only the user who launches an ozone service will be the admin
+      user. This property must be set if ozone services are started by different
+      users. Otherwise, the RPC layer will reject calls from other servers which
+      are started by users not in the list.
+    </description>
+  </property>
+  <property>
+    <name>ozone.block.deleting.container.limit.per.interval</name>
+    <value>10</value>
+    <tag>OZONE, PERFORMANCE, SCM</tag>
+    <description>A maximum number of containers to be scanned by block deleting
+      service per
+      time interval. The block deleting service spawns a thread to handle block
+      deletions in a container. This property is used to throttle the number of
+      threads spawned for block deletions.
+    </description>
+  </property>
+  <property>
+    <name>ozone.block.deleting.limit.per.task</name>
+    <value>1000</value>
+    <tag>OZONE, PERFORMANCE, SCM</tag>
+    <description>A maximum number of blocks to be deleted by block deleting
+      service per
+      time interval. This property is used to throttle the actual number of
+      block deletions on a data node per container.
+    </description>
+  </property>
+  <property>
+    <name>ozone.block.deleting.service.interval</name>
+    <value>1m</value>
+    <tag>OZONE, PERFORMANCE, SCM</tag>
+    <description>Time interval of the block deleting service.
+      The block deleting service runs on each datanode periodically and
+      deletes blocks queued for deletion. Unit could be defined with
+      postfix (ns,ms,s,m,h,d)
+    </description>
+  </property>
+  <property>
+    <name>ozone.block.deleting.service.timeout</name>
+    <value>300000ms</value>
+    <tag>OZONE, PERFORMANCE, SCM</tag>
+    <description>A timeout value of block deletion service. If this is set
+      greater than 0,
+      the service will stop waiting for the block deleting completion after this
+      time. If timeout happens to a large proportion of block deletion, this
+      needs to be increased with ozone.block.deleting.limit.per.task. This
+      setting supports multiple time unit suffixes as described in
+      dfs.heartbeat.interval. If no suffix is specified, then milliseconds is
+      assumed.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.connection.timeout</name>
+    <value>5000ms</value>
+    <tag>OZONE, PERFORMANCE, CLIENT</tag>
+    <description>Connection timeout for Ozone client in milliseconds.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.protocol</name>
+    <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value>
+    <tag>OZONE, CLIENT, MANAGEMENT</tag>
+    <description>Protocol class to be used by the client to connect to ozone
+      cluster.
+      The build-in implementation includes:
+      org.apache.hadoop.ozone.client.rpc.RpcClient for RPC
+      org.apache.hadoop.ozone.client.rest.RestClient for REST
+      The default is the RpClient. Please do not change this unless you have a
+      very good understanding of what you are doing.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.socket.timeout</name>
+    <value>5000ms</value>
+    <tag>OZONE, CLIENT</tag>
+    <description>Socket timeout for Ozone client. Unit could be defined with
+      postfix (ns,ms,s,m,h,d)</description>
+  </property>
+  <property>
+    <name>ozone.enabled</name>
+    <value>false</value>
+    <tag>OZONE, REQUIRED</tag>
+    <description>
+      Status of the Ozone Object Storage service is enabled.
+      Set to true to enable Ozone.
+      Set to false to disable Ozone.
+      Unless this value is set to true, Ozone services will not be started in
+      the cluster.
+
+      Please note: By default ozone is disabled on a hadoop cluster.
+    </description>
+  </property>
+  <property>
+    <name>ozone.handler.type</name>
+    <value>distributed</value>
+    <tag>OZONE, REST</tag>
+    <description>
+      Tells ozone which storage handler to use. The possible values are:
+      distributed - The Ozone distributed storage handler, which speaks to
+      KSM/SCM on the backend and provides REST services to clients.
+      local - Local Storage handler strictly for testing - To be removed.
+    </description>
+  </property>
+  <property>
+    <name>ozone.key.deleting.limit.per.task</name>
+    <value>1000</value>
+    <tag>KSM, PERFORMANCE</tag>
+    <description>
+      A maximum number of keys to be scanned by key deleting service
+      per time interval in KSM. Those keys are sent to delete metadata and
+      generate transactions in SCM for next async deletion between SCM
+      and DataNode.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.address</name>
+    <value/>
+    <tag>KSM, REQUIRED</tag>
+    <description>
+      The address of the Ozone KSM service. This allows clients to discover
+      the KSMs address.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.group.rights</name>
+    <value>READ_WRITE</value>
+    <tag>KSM, SECURITY</tag>
+    <description>
+      Default group permissions in Ozone KSM.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.handler.count.key</name>
+    <value>20</value>
+    <tag>KSM, PERFORMANCE</tag>
+    <description>
+      The number of RPC handler threads for KSM service endpoints.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.http-address</name>
+    <value>0.0.0.0:9874</value>
+    <tag>KSM, MANAGEMENT</tag>
+    <description>
+      The address and the base port where the KSM web UI will listen on.
+
+      If the port is 0, then the server will start on a free port. However, it
+      is best to specify a well-known port, so it is easy to connect and see
+      the KSM management UI.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.http-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>KSM, MANAGEMENT</tag>
+    <description>
+      The actual address the KSM web server will bind to. If this optional
+      the address is set, it overrides only the hostname portion of
+      ozone.ksm.http-address.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.http.enabled</name>
+    <value>true</value>
+    <tag>KSM, MANAGEMENT</tag>
+    <description>
+      Property to enable or disable KSM web user interface.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.https-address</name>
+    <value>0.0.0.0:9875</value>
+    <tag>KSM, MANAGEMENT, SECURITY</tag>
+    <description>
+      The address and the base port where the KSM web UI will listen
+      on using HTTPS.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.https-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>KSM, MANAGEMENT, SECURITY</tag>
+    <description>
+      The actual address the KSM web server will bind to using HTTPS.
+      If this optional address is set, it overrides only the hostname portion of
+      ozone.ksm.http-address.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.keytab.file</name>
+    <value/>
+    <tag>KSM, SECURITY</tag>
+    <description>
+      The keytab file for Kerberos authentication in KSM.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.db.cache.size.mb</name>
+    <value>128</value>
+    <tag>KSM, PERFORMANCE</tag>
+    <description>
+      The size of KSM DB cache in MB that used for caching files.
+      This value is set to an abnormally low value in the default configuration.
+      That is to make unit testing easy. Generally, this value should be set to
+      something like 16GB or more, if you intend to use Ozone at scale.
+
+      A large value for this key allows a proportionally larger amount of KSM
+      metadata to be cached in memory. This makes KSM operations faster.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.user.max.volume</name>
+    <value>1024</value>
+    <tag>KSM, MANAGEMENT</tag>
+    <description>
+      The maximum number of volumes a user can have on a cluster.Increasing or
+      decreasing this number has no real impact on ozone cluster. This is
+      defined only for operational purposes. Only an administrator can create a
+      volume, once a volume is created there are no restrictions on the number
+      of buckets or keys inside each bucket a user can create.
+    </description>
+  </property>
+  <property>
+    <name>ozone.ksm.user.rights</name>
+    <value>READ_WRITE</value>
+    <tag>KSM, SECURITY</tag>
+    <description>
+      Default user permissions used in KSM.
+    </description>
+  </property>
+  <property>
+    <name>ozone.localstorage.root</name>
+    <value>${hadoop.tmp.dir}/ozone</value>
+    <tag>OZONE, DEBUG</tag>
+    <description>
+      This is used only for testing purposes. This value is used by the local
+      storage handler to simulate a REST backend. This is useful only when
+      debugging the REST front end independent of KSM and SCM. To be removed.
+    </description>
+  </property>
+  <property>
+    <name>ozone.metadata.dirs</name>
+    <value/>
+    <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
+    <description>
+      Ozone metadata is shared among KSM, which acts as the namespace
+      manager for ozone, SCM which acts as the block manager and data nodes
+      which maintain the name of the key(Key Name and BlockIDs). This
+      replicated and distributed metadata store is maintained under the
+      directory pointed by this key. Since metadata can be I/O intensive, at
+      least on KSM and SCM we recommend having SSDs. If you have the luxury
+      of mapping this path to SSDs on all machines in the cluster, that will
+      be excellent.
+
+      If Ratis metadata directories are not specified, Ratis server will emit a
+      warning and use this path for storing its metadata too.
+    </description>
+  </property>
+  <property>
+    <name>ozone.metastore.impl</name>
+    <value>RocksDB</value>
+    <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag>
+    <description>
+      Ozone metadata store implementation. Ozone metadata are well
+      distributed to multiple services such as ksm, scm. They are stored in
+      some local key-value databases. This property determines which database
+      library to use. Supported value is either LevelDB or RocksDB.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.metastore.rocksdb.statistics</name>
+    <value>ALL</value>
+    <tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
+    <description>
+      The statistics level of the rocksdb store. If you use any value from
+      org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
+      statistics will be exposed over JMX bean with the choosed setting. Set
+      it to OFF to not initialize rocksdb statistics at all. Please note that
+      collection of statistics could have 5-10% performance penalty.
+      Check the rocksdb documentation for more details.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.scm.block.client.address</name>
+    <value/>
+    <tag>OZONE, SCM</tag>
+    <description>The address of the Ozone SCM block client service. If not
+      defined value of ozone.scm.client.address is used.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.block.client.bind.host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      The hostname or IP address used by the SCM block client
+      endpoint to bind.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.block.client.port</name>
+    <value>9863</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      The port number of the Ozone SCM block client service.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.block.deletion.max.retry</name>
+    <value>4096</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      SCM wraps up many blocks in a deletion transaction and sends that to data
+      node for physical deletion periodically. This property determines how many
+      times SCM is going to retry sending a deletion operation to the data node.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.block.size.in.mb</name>
+    <value>256</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      The default size of a scm block in bytes. This is maps to the default
+      Ozone block size.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.chunk.size</name>
+    <value>16777216</value>
+    <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
+    <description>
+      The chunk size for reading/writing chunk operations in bytes.
+
+      The chunk size defaults to 8MB. If the value configured is more than the
+      maximum size (16MB), it will be reset to the maximum size. This maps to
+      the network packet sizes and file write operations in the client to
+      datanode protocol.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.client.address</name>
+    <value/>
+    <tag>OZONE, SCM, REQUIRED</tag>
+    <description>
+      The address of the Ozone SCM client service. This is a required setting.
+
+      It is a string in the host:port format. The port number is optional
+      and defaults to 9860.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.client.bind.host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, SCM, MANAGEMENT</tag>
+    <description>The hostname or IP address used by the SCM client endpoint to
+      bind.
+      This setting is used by the SCM only and never used by clients.
+
+      The setting can be useful in multi-homed setups to restrict the
+      availability of the SCM client service to a specific interface.
+
+      The default is appropriate for most clusters.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.client.port</name>
+    <value>9860</value>
+    <tag>OZONE, SCM, MANAGEMENT</tag>
+    <description>The port number of the Ozone SCM client service.</description>
+  </property>
+  <property>
+    <name>ozone.scm.container.deletion-choosing.policy</name>
+    <value>
+      org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
+    </value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The policy used for choosing desire containers for block deletion.
+      Datanode selects some containers to process block deletion
+      in a certain interval defined by ozone.block.deleting.service.interval.
+      The number of containers to process in each interval is defined
+      by ozone.block.deleting.container.limit.per.interval. This property is
+      used to configure the policy applied while selecting containers.
+      There are two policies supporting now:
+      RandomContainerDeletionChoosingPolicy and
+      TopNOrderedContainerDeletionChoosingPolicy.
+      org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy
+      implements a simply random policy that to return a random list of
+      containers.
+      org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
+      implements a policy that choosing top count number of containers in a
+      pending-deletion-blocks's num
+      based descending order.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.placement.impl</name>
+    <value>
+      org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom
+    </value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>Placement policy class for containers.
+      Defaults to SCMContainerPlacementRandom.class
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.provision_batch_size</name>
+    <value>20</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>Pre-provision specified number of containers for block
+      allocation.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.report.processing.interval</name>
+    <value>60s</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>Time interval for scm to process container reports
+      for a node pool. Scm handles node pool reports in a cyclic clock
+      manner, it fetches pools periodically with this time interval.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.reports.wait.timeout</name>
+    <value>300s</value>
+    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
+    <description>Maximum time to wait in seconds for processing all container
+      reports from
+      a node pool. It determines the timeout for a
+      node pool report.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.size.gb</name>
+    <value>5</value>
+    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
+    <description>
+      Default container size used by Ozone. This value is specified
+      in GB.
+      There are two considerations while picking this number. The speed at which
+      a container can be replicated, determined by the network speed and the
+      metadata that each container generates. So selecting a large number
+      creates less SCM metadata, but recovery time will be more. 5GB is a number
+      that maps to quick replication times in gigabit networks, but still
+      balances the amount of metadata.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.datanode.address</name>
+    <value/>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The address of the Ozone SCM service used for internal
+      communication between the DataNodes and the SCM.
+
+      It is a string in the host:port format. The port number is optional
+      and defaults to 9861.
+
+      This setting is optional. If unspecified then the hostname portion
+      is picked from the ozone.scm.client.address setting and the
+      default service port of 9861 is chosen.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.datanode.bind.host</name>
+    <value/>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The hostname or IP address used by the SCM service endpoint to
+      bind.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.datanode.id</name>
+    <value/>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>The path that datanodes will use to store the datanode ID.
+      If this value is not set, then datanode ID is created under the
+      metadata directory.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.datanode.port</name>
+    <value>9861</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The port number of the Ozone SCM service.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.db.cache.size.mb</name>
+    <value>128</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>SCM keeps track of the Containers in the cluster. This DB holds
+      the container metadata. This value is set to a small value to make the
+      unit
+      testing runs smooth. In production, we recommend a value of 16GB or
+      higher. This allows SCM to avoid disk I/O's while looking up the container
+      location.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.dead.node.interval</name>
+    <value>10m</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The interval between heartbeats before a node is tagged as dead.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.handler.count.key</name>
+    <value>10</value>
+    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+    <description>
+      The number of RPC handler threads for each SCM service
+      endpoint.
+
+      The default is appropriate for small clusters (tens of nodes).
+
+      Set a value that is appropriate for the cluster size. Generally, HDFS
+      recommends RPC handler count is set to 20 * log2(Cluster Size) with an
+      upper limit of 200. However, SCM will not have the same amount of
+      traffic as Namenode, so a value much smaller than that will work well too.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.heartbeat.interval</name>
+    <value>30s</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The heartbeat interval from a data node to SCM. Yes,
+      it is not three but 30, since most data nodes will heart beating via Ratis
+      heartbeats. If a client is not able to talk to a data node, it will notify
+      KSM/SCM eventually. So a 30 second HB seems to work. This assumes that
+      replication strategy used is Ratis if not, this value should be set to
+      something smaller like 3 seconds.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.heartbeat.log.warn.interval.count</name>
+    <value>10</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      Defines how frequently we will log the missing of a heartbeat to SCM.
+      For example in the default case, we will write a warning message for each
+      ten consecutive heartbeats that we miss to SCM. This helps in reducing
+      clutter in a data node log, but trade off is that logs will have less of
+      this statement.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.heartbeat.rpc-timeout</name>
+    <value>1000</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      Timeout value for the RPC from Datanode to SCM in milliseconds.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.heartbeat.thread.interval</name>
+    <value>3s</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      When a heartbeat from the data node arrives on SCM, It is queued for
+      processing with the time stamp of when the heartbeat arrived. There is a
+      heartbeat processing thread inside SCM that runs at a specified interval.
+      This value controls how frequently this thread is run.
+
+      There are some assumptions build into SCM such as this value should allow
+      the heartbeat processing thread to run at least three times more
+      frequently than heartbeats and at least five times more than stale node
+      detection time. If you specify a wrong value, SCM will gracefully refuse
+      to run. For more info look at the node manager tests in SCM.
+
+      In short, you don't need to change this.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.http-address</name>
+    <value>0.0.0.0:9876</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The address and the base port where the SCM web ui will listen on.
+
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.http-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The actual address the SCM web server will bind to. If this
+      optional address is set, it overrides only the hostname portion of
+      ozone.scm.http-address.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.http.enabled</name>
+    <value>true</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      Property to enable or disable SCM web ui.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.https-address</name>
+    <value>0.0.0.0:9877</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The address and the base port where the SCM web UI will listen
+      on using HTTPS.
+
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.https-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The actual address the SCM web server will bind to using HTTPS.
+      If this optional address is set, it overrides only the hostname portion of
+      ozone.scm.http-address.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.keytab.file</name>
+    <value/>
+    <tag>OZONE, SECURITY</tag>
+    <description>
+      The keytab file for Kerberos authentication in SCM.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.max.container.report.threads</name>
+    <value>100</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>
+      Maximum number of threads to process container reports in scm.
+      Each container report from a data node is processed by scm in a worker
+      thread, fetched from a thread pool. This property is used to control the
+      maximum size of the thread pool.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.max.hb.count.to.process</name>
+    <value>5000</value>
+    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+    <description>
+      The maximum number of heartbeat to process per loop of the
+      heartbeat process thread. Please see
+      ozone.scm.heartbeat.thread.interval
+      for more info.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.max.nodepool.processing.threads</name>
+    <value>1</value>
+    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+    <description>
+      Number of node pools to process in parallel.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.names</name>
+    <value/>
+    <tag>OZONE</tag>
+    <description>
+      The value of this property is a set of DNS | DNS:PORT | IP
+      Address | IP:PORT. Written as a comma separated string. e.g. scm1,
+      scm2:8020, 7.7.7.7:7777.
+      This property allows datanodes to discover where SCM is, so that
+      datanodes can send heartbeat to SCM.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.stale.node.interval</name>
+    <value>90s</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      The interval for stale node flagging. Please
+      see ozone.scm.heartbeat.thread.interval before changing this value.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.max.nodepool.processing.threads</name>
+    <value>1</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      Controls the number of node pools that can be processed in parallel by
+      Container Supervisor.
+    </description>
+  </property>
+  <property>
+    <name>ozone.trace.enabled</name>
+    <value>false</value>
+    <tag>OZONE, DEBUG</tag>
+    <description>
+      Setting this flag to true dumps the HTTP request/ response in
+      the logs. Very useful when debugging REST protocol.
+    </description>
+  </property>
+  <property>
+    <name>ozone.web.authentication.kerberos.principal</name>
+    <value/>
+    <tag>OZONE, SECURITY</tag>
+    <description>
+      The server principal used by the SCM and KSM for web UI SPNEGO
+      authentication when Kerberos security is enabled. This is typically set to
+      HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
+      HTTP/ by convention.
+
+      If the value is '*', the web server will attempt to login with
+      every principal specified in the keytab file.
+    </description>
+  </property>
+
+  <!--Client Settings-->
+  <property>
+    <name>scm.container.client.idle.threshold</name>
+    <value>10s</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>
+      In the standalone pipelines, the SCM clients use netty to
+      communicate with the container. It also uses connection pooling to
+      reduce client side overheads. This allows a connection to stay idle for
+      a while before the connection is closed.
+    </description>
+  </property>
+  <property>
+    <name>scm.container.client.max.size</name>
+    <value>256</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>
+      Controls the maximum number of connections that we cached via
+      clientconnection pooling. If the number of connection
+      exceed this count then the oldest idle connection is evicted.
+    </description>
+  </property>
+
+  <property>
+    <name>scm.container.client.max.outstanding.requests</name>
+    <value>100</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>
+      Controls the maximum number of outstanding async requests that can be
+      handled by the Standalone as well as Ratis client.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.scm.container.creation.lease.timeout</name>
+    <value>60s</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      Container creation timeout in milliseconds to be used by SCM. When
+      BEGIN_CREATE event happens the container is moved from ALLOCATED to
+      CREATING state, SCM will now wait for the configured amount of time
+      to get COMPLETE_CREATE event if it doesn't receive it will move the
+      container to DELETING.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.key.preallocation.maxsize</name>
+    <value>134217728</value>
+    <tag>OZONE, KSM, PERFORMANCE</tag>
+    <description>
+      When a new key write request is sent to KSM, if a size is requested, at most
+      128MB of size is allocated at request time. If client needs more space for the
+      write, separate block allocation requests will be made.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.client.list.cache</name>
+    <value>1000</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>
+      Configuration property to configure the cache size of client list calls.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.replication</name>
+    <value>3</value>
+    <tag>OZONE, CLIENT</tag>
+    <description>
+      Default replication value. The actual number of replications can be
+      specified when writing the key. The default is used if replication
+      is not specified. Supported values: 1 and 3.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.replication.type</name>
+    <value>RATIS</value>
+    <tag>OZONE, CLIENT</tag>
+    <description>
+      Default replication type to be used while writing key into ozone. The
+      value can be specified when writing the key, default is used when
+      nothing is specified. Supported values: RATIS, STAND_ALONE and CHAINED.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.close.threshold</name>
+    <value>0.9f</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      This determines the threshold to be used for closing a container.
+      When the container used percentage reaches this threshold,
+      the container will be closed. Value should be a positive, non-zero
+      percentage in float notation (X.Yf), with 1.0f meaning 100%.
+    </description>
+  </property>
+  <property>
+    <name>ozone.rest.client.http.connection.max</name>
+    <value>100</value>
+    <tag>OZONE, CLIENT</tag>
+    <description>
+      This defines the overall connection limit for the connection pool used in
+      RestClient.
+    </description>
+  </property>
+  <property>
+    <name>ozone.rest.client.http.connection.per-route.max</name>
+    <value>20</value>
+    <tag>OZONE, CLIENT</tag>
+    <description>
+      This defines the connection limit per one HTTP route/host. Total max
+      connection is limited by ozone.rest.client.http.connection.max property.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.open.key.cleanup.service.interval.seconds</name>
+    <value>86400</value>
+    <tag>OZONE, KSM, PERFORMANCE</tag>
+    <description>
+      A background job periodically checks open key entries and delete the expired ones. This entry controls the
+      interval of this cleanup check.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.open.key.expire.threshold</name>
+    <value>86400</value>
+    <tag>OZONE, KSM, PERFORMANCE</tag>
+    <description>
+      Controls how long an open key operation is considered active. Specifically, if a key
+      has been open longer than the value of this config entry, that open key is considered as
+      expired (e.g. due to client crash). Default to 24 hours.
+    </description>
+  </property>
+
+
+  <property>
+    <name>hdds.rest.rest-csrf.enabled</name>
+    <value>false</value>
+    <description>
+      If true, then enables Object Store REST server protection against
+      cross-site request forgery (CSRF).
+    </description>
+  </property>
+
+  <property>
+    <name>hdds.rest.http-address</name>
+    <value>0.0.0.0:9880</value>
+    <description>The http address of Object Store REST server inside the
+      datanode.</description>
+  </property>
+
+
+  <property>
+    <name>hdds.rest.netty.high.watermark</name>
+    <value>65535</value>
+    <description>
+      High watermark configuration to Netty for Object Store REST server.
+    </description>
+  </property>
+
+  <property>
+    <name>hdds.rest.netty.low.watermark</name>
+    <value>32768</value>
+    <description>
+      Low watermark configuration to Netty for Object Store REST server.
+    </description>
+  </property>
+
+
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java
new file mode 100644
index 0000000..f53f770
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestArchive.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+
+/**
+ * Test archive creation and unpacking.
+ */
+public class TestArchive {
+  private static final int DIR_COUNT = 10;
+  private static final int SUB_DIR_COUNT = 3;
+  private static final int FILE_COUNT = 10;
+  private long checksumWrite = 0L;
+  private long checksumRead = 0L;
+  private long tmp = 0L;
+  private Checksum crc = new Adler32();
+
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+
+  @Rule
+  public TemporaryFolder outputFolder = new TemporaryFolder();
+
+
+  @Before
+  public void setUp() throws Exception {
+    Random r = new Random();
+    final int megaByte = 1024 * 1024;
+
+    for (int x = 0; x < DIR_COUNT; x++) {
+      File subdir = folder.newFolder(String.format("dir%d", x));
+      for (int y = 0; y < SUB_DIR_COUNT; y++) {
+        File targetDir = new File(subdir.getPath().concat(File.separator)
+            .concat(String.format("subdir%d%d", x, y)));
+        if(!targetDir.mkdirs()) {
+          throw new IOException("Failed to create subdirectory. " +
+              targetDir.toString());
+        }
+        for (int z = 0; z < FILE_COUNT; z++) {
+          Path temp = Paths.get(targetDir.getPath().concat(File.separator)
+              .concat(String.format("File%d.txt", z)));
+          byte[] buf = RandomUtils.nextBytes(r.nextInt(megaByte));
+          Files.write(temp, buf);
+          crc.reset();
+          crc.update(buf, 0, buf.length);
+          tmp = crc.getValue();
+          checksumWrite +=tmp;
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testArchive() throws Exception {
+    File archiveFile = new File(outputFolder.getRoot() + File.separator
+        + "test.container.zip");
+    long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile);
+    Assert.assertTrue(zipCheckSum > 0);
+    File decomp = new File(outputFolder.getRoot() + File.separator +
+        "decompress");
+    if (!decomp.exists() && !decomp.mkdirs()) {
+      throw new IOException("Unable to create the destination directory. " +
+          decomp.getPath());
+    }
+
+    FileUtil.unZip(archiveFile, decomp);
+    String[] patterns = {"txt"};
+    Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true);
+    int count = 0;
+    while (iter.hasNext()) {
+      count++;
+      byte[] buf = Files.readAllBytes(iter.next().toPath());
+      crc.reset();
+      crc.update(buf, 0, buf.length);
+      tmp = crc.getValue();
+      checksumRead += tmp;
+    }
+    Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count);
+    Assert.assertEquals(checksumWrite, checksumRead);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
new file mode 100644
index 0000000..7966941
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+/**
+ Test cases for SCM client classes.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
new file mode 100644
index 0000000..6b26b60
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java
@@ -0,0 +1,414 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.utils.BatchOperation;
+import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
+import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Test class for ozone metadata store.
+ */
+@RunWith(Parameterized.class)
+public class TestMetadataStore {
+
+  private final String storeImpl;
+
+  public TestMetadataStore(String metadataImpl) {
+    this.storeImpl = metadataImpl;
+  }
+
+  @Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][] {
+        {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+        {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+    });
+  }
+
+  private MetadataStore store;
+  private File testDir;
+  private final static int MAX_GETRANGE_LENGTH = 100;
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  @Before
+  public void init() throws IOException {
+    testDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+        + "-" + storeImpl.toLowerCase());
+
+    Configuration conf = new OzoneConfiguration();
+    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+    store = MetadataStoreBuilder.newBuilder()
+        .setConf(conf)
+        .setCreateIfMissing(true)
+        .setDbFile(testDir)
+        .build();
+
+    // Add 20 entries.
+    // {a0 : a-value0} to {a9 : a-value9}
+    // {b0 : b-value0} to {b9 : b-value9}
+    for (int i=0; i<10; i++) {
+      store.put(getBytes("a" + i), getBytes("a-value" + i));
+      store.put(getBytes("b" + i), getBytes("b-value" + i));
+    }
+  }
+
+  @After
+  public void cleanup() throws IOException {
+    store.close();
+    store.destroy();
+    FileUtils.deleteDirectory(testDir);
+  }
+
+  private byte[] getBytes(String str) {
+    return str == null ? null :
+        DFSUtilClient.string2Bytes(str);
+  }
+
+  private String getString(byte[] bytes) {
+    return bytes == null ? null :
+        DFSUtilClient.bytes2String(bytes);
+  }
+
+  @Test
+  public void testGetDelete() throws IOException {
+    for (int i=0; i<10; i++) {
+      byte[] va = store.get(getBytes("a" + i));
+      Assert.assertEquals("a-value" + i, getString(va));
+
+      byte[] vb = store.get(getBytes("b" + i));
+      Assert.assertEquals("b-value" + i, getString(vb));
+    }
+
+    String keyToDel = "del-" + UUID.randomUUID().toString();
+    store.put(getBytes(keyToDel), getBytes(keyToDel));
+    Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
+    store.delete(getBytes(keyToDel));
+    Assert.assertEquals(null, store.get(getBytes(keyToDel)));
+  }
+
+  @Test
+  public void testPeekFrom() throws IOException {
+    // Test peek from an element that has prev as well as next
+    testPeek("a3", "a2", "a4");
+
+    // Test peek from an element that only has prev
+    testPeek("b9", "b8", null);
+
+    // Test peek from an element that only has next
+    testPeek("a0", null, "a1");
+  }
+
+  private String getExpectedValue(String key) {
+    if (key == null) {
+      return null;
+    }
+    char[] arr = key.toCharArray();
+    return new StringBuffer().append(arr[0]).append("-value")
+        .append(arr[arr.length - 1]).toString();
+  }
+
+  private void testPeek(String peekKey, String prevKey, String nextKey)
+      throws IOException {
+    // Look for current
+    String k = null;
+    String v = null;
+    ImmutablePair<byte[], byte[]> current =
+        store.peekAround(0, getBytes(peekKey));
+    if (current != null) {
+      k = getString(current.getKey());
+      v = getString(current.getValue());
+    }
+    Assert.assertEquals(peekKey, k);
+    Assert.assertEquals(v, getExpectedValue(peekKey));
+
+    // Look for prev
+    k = null;
+    v = null;
+    ImmutablePair<byte[], byte[]> prev =
+        store.peekAround(-1, getBytes(peekKey));
+    if (prev != null) {
+      k = getString(prev.getKey());
+      v = getString(prev.getValue());
+    }
+    Assert.assertEquals(prevKey, k);
+    Assert.assertEquals(v, getExpectedValue(prevKey));
+
+    // Look for next
+    k = null;
+    v = null;
+    ImmutablePair<byte[], byte[]> next =
+        store.peekAround(1, getBytes(peekKey));
+    if (next != null) {
+      k = getString(next.getKey());
+      v = getString(next.getValue());
+    }
+    Assert.assertEquals(nextKey, k);
+    Assert.assertEquals(v, getExpectedValue(nextKey));
+  }
+
+  @Test
+  public void testIterateKeys() throws IOException {
+    // iterate keys from b0
+    ArrayList<String> result = Lists.newArrayList();
+    store.iterate(getBytes("b0"), (k, v) -> {
+      // b-value{i}
+      String value = getString(v);
+      char num = value.charAt(value.length() - 1);
+      // each value adds 1
+      int i = Character.getNumericValue(num) + 1;
+      value =  value.substring(0, value.length() - 1) + i;
+      result.add(value);
+      return true;
+    });
+
+    Assert.assertFalse(result.isEmpty());
+    for (int i=0; i<result.size(); i++) {
+      Assert.assertEquals("b-value" + (i+1), result.get(i));
+    }
+
+    // iterate from a non exist key
+    result.clear();
+    store.iterate(getBytes("xyz"), (k, v) -> {
+      result.add(getString(v));
+      return true;
+    });
+    Assert.assertTrue(result.isEmpty());
+
+    // iterate from the beginning
+    result.clear();
+    store.iterate(null, (k, v) -> {
+      result.add(getString(v));
+      return true;
+    });
+    Assert.assertEquals(20, result.size());
+  }
+
+  @Test
+  public void testGetRangeKVs() throws IOException {
+    List<Map.Entry<byte[], byte[]>> result = null;
+
+    // Set empty startKey will return values from beginning.
+    result = store.getRangeKVs(null, 5);
+    Assert.assertEquals(5, result.size());
+    Assert.assertEquals("a-value2", getString(result.get(2).getValue()));
+
+    // Empty list if startKey doesn't exist.
+    result = store.getRangeKVs(getBytes("a12"), 5);
+    Assert.assertEquals(0, result.size());
+
+    // Returns max available entries after a valid startKey.
+    result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
+    Assert.assertEquals(10, result.size());
+    Assert.assertEquals("b0", getString(result.get(0).getKey()));
+    Assert.assertEquals("b-value0", getString(result.get(0).getValue()));
+    result = store.getRangeKVs(getBytes("b0"), 5);
+    Assert.assertEquals(5, result.size());
+
+    // Both startKey and count are honored.
+    result = store.getRangeKVs(getBytes("a9"), 2);
+    Assert.assertEquals(2, result.size());
+    Assert.assertEquals("a9", getString(result.get(0).getKey()));
+    Assert.assertEquals("a-value9", getString(result.get(0).getValue()));
+    Assert.assertEquals("b0", getString(result.get(1).getKey()));
+    Assert.assertEquals("b-value0", getString(result.get(1).getValue()));
+
+    // Filter keys by prefix.
+    // It should returns all "b*" entries.
+    MetadataKeyFilter filter1 = new KeyPrefixFilter("b");
+    result = store.getRangeKVs(null, 100, filter1);
+    Assert.assertEquals(10, result.size());
+    Assert.assertTrue(result.stream().allMatch(entry ->
+        new String(entry.getKey()).startsWith("b")
+    ));
+    Assert.assertEquals(20, filter1.getKeysScannedNum());
+    Assert.assertEquals(10, filter1.getKeysHintedNum());
+    result = store.getRangeKVs(null, 3, filter1);
+    Assert.assertEquals(3, result.size());
+    result = store.getRangeKVs(getBytes("b3"), 1, filter1);
+    Assert.assertEquals("b-value3", getString(result.get(0).getValue()));
+
+    // Define a customized filter that filters keys by suffix.
+    // Returns all "*2" entries.
+    MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
+        -> getString(currentKey).endsWith("2");
+    result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
+    Assert.assertEquals(2, result.size());
+    Assert.assertEquals("a2", getString(result.get(0).getKey()));
+    Assert.assertEquals("b2", getString(result.get(1).getKey()));
+    result = store.getRangeKVs(null, 1, filter2);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals("a2", getString(result.get(0).getKey()));
+
+    // Apply multiple filters.
+    result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals("b2", getString(result.get(0).getKey()));
+    Assert.assertEquals("b-value2", getString(result.get(0).getValue()));
+
+    // If filter is null, no effect.
+    result = store.getRangeKVs(null, 1, null);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals("a0", getString(result.get(0).getKey()));
+  }
+
+  @Test
+  public void testGetSequentialRangeKVs() throws IOException {
+    MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey)
+        -> DFSUtil.bytes2String(currentKey).endsWith("2");
+    // Suppose to return a2 and b2
+    List<Map.Entry<byte[], byte[]>> result =
+        store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
+    Assert.assertEquals(2, result.size());
+    Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
+    Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey()));
+
+    // Suppose to return just a2, because when it iterates to a3,
+    // the filter no long matches and it should stop from there.
+    result = store.getSequentialRangeKVs(null,
+        MAX_GETRANGE_LENGTH, suffixFilter);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
+  }
+
+  @Test
+  public void testGetRangeLength() throws IOException {
+    List<Map.Entry<byte[], byte[]>> result = null;
+
+    result = store.getRangeKVs(null, 0);
+    Assert.assertEquals(0, result.size());
+
+    result = store.getRangeKVs(null, 1);
+    Assert.assertEquals(1, result.size());
+
+    // Count less than zero is invalid.
+    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expectMessage("Invalid count given");
+    store.getRangeKVs(null, -1);
+  }
+
+  @Test
+  public void testInvalidStartKey() throws IOException {
+    // If startKey is invalid, the returned list should be empty.
+    List<Map.Entry<byte[], byte[]>> kvs =
+        store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
+    Assert.assertEquals(kvs.size(), 0);
+  }
+
+  @Test
+  public void testDestroyDB() throws IOException {
+    // create a new DB to test db destroy
+    Configuration conf = new OzoneConfiguration();
+    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+    File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+        + "-" + storeImpl.toLowerCase() + "-toDestroy");
+    MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+        .setConf(conf)
+        .setCreateIfMissing(true)
+        .setDbFile(dbDir)
+        .build();
+
+    dbStore.put(getBytes("key1"), getBytes("value1"));
+    dbStore.put(getBytes("key2"), getBytes("value2"));
+
+    Assert.assertFalse(dbStore.isEmpty());
+    Assert.assertTrue(dbDir.exists());
+    Assert.assertTrue(dbDir.listFiles().length > 0);
+
+    dbStore.destroy();
+
+    Assert.assertFalse(dbDir.exists());
+  }
+
+  @Test
+  public void testBatchWrite() throws IOException {
+    Configuration conf = new OzoneConfiguration();
+    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+    File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+        + "-" + storeImpl.toLowerCase() + "-batchWrite");
+    MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+        .setConf(conf)
+        .setCreateIfMissing(true)
+        .setDbFile(dbDir)
+        .build();
+
+    List<String> expectedResult = Lists.newArrayList();
+    for (int i = 0; i<10; i++) {
+      dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i));
+      expectedResult.add("batch-" + i);
+    }
+
+    BatchOperation batch = new BatchOperation();
+    batch.delete(getBytes("batch-2"));
+    batch.delete(getBytes("batch-3"));
+    batch.delete(getBytes("batch-4"));
+    batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2"));
+
+    expectedResult.remove("batch-2");
+    expectedResult.remove("batch-3");
+    expectedResult.remove("batch-4");
+    expectedResult.add("batch-new-2");
+
+    dbStore.writeBatch(batch);
+
+    Iterator<String> it = expectedResult.iterator();
+    AtomicInteger count = new AtomicInteger(0);
+    dbStore.iterate(null, (key, value) -> {
+      count.incrementAndGet();
+      return it.hasNext() && it.next().equals(getString(key));
+    });
+
+    Assert.assertEquals(8, count.get());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
new file mode 100644
index 0000000..03c45c5
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * This class is to test acl stoarge and retreival in ozone store.
+ */
+public class TestOzoneAcls {
+
+  @Test
+  public void testAclParse() {
+    HashMap<String, Boolean> testMatrix;
+    testMatrix = new HashMap<>();
+
+    testMatrix.put("user:bilbo:r", Boolean.TRUE);
+    testMatrix.put("user:bilbo:w", Boolean.TRUE);
+    testMatrix.put("user:bilbo:rw", Boolean.TRUE);
+    testMatrix.put("user:bilbo:wr", Boolean.TRUE);
+    testMatrix.put("    user:bilbo:wr   ", Boolean.TRUE);
+
+
+    // ACLs makes no judgement on the quality of
+    // user names. it is for the userAuth interface
+    // to determine if a user name is really a name
+    testMatrix.put(" user:*:rw", Boolean.TRUE);
+    testMatrix.put(" user:~!:rw", Boolean.TRUE);
+
+
+    testMatrix.put("", Boolean.FALSE);
+    testMatrix.put(null, Boolean.FALSE);
+    testMatrix.put(" user:bilbo:", Boolean.FALSE);
+    testMatrix.put(" user:bilbo:rx", Boolean.FALSE);
+    testMatrix.put(" user:bilbo:mk", Boolean.FALSE);
+    testMatrix.put(" user::rw", Boolean.FALSE);
+    testMatrix.put("user11:bilbo:rw", Boolean.FALSE);
+    testMatrix.put(" user:::rw", Boolean.FALSE);
+
+    testMatrix.put(" group:hobbit:r", Boolean.TRUE);
+    testMatrix.put(" group:hobbit:w", Boolean.TRUE);
+    testMatrix.put(" group:hobbit:rw", Boolean.TRUE);
+    testMatrix.put(" group:hobbit:wr", Boolean.TRUE);
+    testMatrix.put(" group:*:rw", Boolean.TRUE);
+    testMatrix.put(" group:~!:rw", Boolean.TRUE);
+
+    testMatrix.put(" group:hobbit:", Boolean.FALSE);
+    testMatrix.put(" group:hobbit:rx", Boolean.FALSE);
+    testMatrix.put(" group:hobbit:mk", Boolean.FALSE);
+    testMatrix.put(" group::", Boolean.FALSE);
+    testMatrix.put(" group::rw", Boolean.FALSE);
+    testMatrix.put(" group22:hobbit:", Boolean.FALSE);
+    testMatrix.put(" group:::rw", Boolean.FALSE);
+
+    testMatrix.put("JUNK group:hobbit:r", Boolean.FALSE);
+    testMatrix.put("JUNK group:hobbit:w", Boolean.FALSE);
+    testMatrix.put("JUNK group:hobbit:rw", Boolean.FALSE);
+    testMatrix.put("JUNK group:hobbit:wr", Boolean.FALSE);
+    testMatrix.put("JUNK group:*:rw", Boolean.FALSE);
+    testMatrix.put("JUNK group:~!:rw", Boolean.FALSE);
+
+    testMatrix.put(" world::r", Boolean.TRUE);
+    testMatrix.put(" world::w", Boolean.TRUE);
+    testMatrix.put(" world::rw", Boolean.TRUE);
+    testMatrix.put(" world::wr", Boolean.TRUE);
+
+    testMatrix.put(" world:bilbo:w", Boolean.FALSE);
+    testMatrix.put(" world:bilbo:rw", Boolean.FALSE);
+
+    Set<String> keys = testMatrix.keySet();
+    for (String key : keys) {
+      if (testMatrix.get(key)) {
+        OzoneAcl.parseAcl(key);
+      } else {
+        try {
+          OzoneAcl.parseAcl(key);
+          // should never get here since parseAcl will throw
+          fail("An exception was expected but did not happen.");
+        } catch (IllegalArgumentException e) {
+          // nothing to do
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testAclValues() {
+    OzoneAcl acl = OzoneAcl.parseAcl("user:bilbo:rw");
+    assertEquals(acl.getName(), "bilbo");
+    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+    acl = OzoneAcl.parseAcl("user:bilbo:wr");
+    assertEquals("bilbo", acl.getName());
+    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+    acl = OzoneAcl.parseAcl("user:bilbo:r");
+    assertEquals("bilbo", acl.getName());
+    assertEquals(OzoneAcl.OzoneACLRights.READ, acl.getRights());
+    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+    acl = OzoneAcl.parseAcl("user:bilbo:w");
+    assertEquals("bilbo", acl.getName());
+    assertEquals(OzoneAcl.OzoneACLRights.WRITE, acl.getRights());
+    assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
+
+    acl = OzoneAcl.parseAcl("group:hobbit:wr");
+    assertEquals(acl.getName(), "hobbit");
+    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+    assertEquals(OzoneAcl.OzoneACLType.GROUP, acl.getType());
+
+    acl = OzoneAcl.parseAcl("world::wr");
+    assertEquals(acl.getName(), "");
+    assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
+    assertEquals(OzoneAcl.OzoneACLType.WORLD, acl.getType());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
new file mode 100644
index 0000000..c1470bb
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.common;
+
+import org.apache.commons.collections.SetUtils;
+import org.apache.hadoop.ozone.common.statemachine
+    .InvalidStateTransitionException;
+import org.apache.hadoop.ozone.common.statemachine.StateMachine;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT;
+import static org.apache.hadoop.ozone.common.TestStateMachine.STATES
+    .OPERATIONAL;
+
+/**
+ * This class is to test ozone common state machine.
+ */
+public class TestStateMachine {
+
+  /**
+   * STATES used by the test state machine.
+   */
+  public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL};
+
+  /**
+   * EVENTS used by the test state machine.
+   */
+  public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT};
+
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
+
+  @Test
+  public void testStateMachineStates() throws InvalidStateTransitionException {
+    Set<STATES> finals = new HashSet<>();
+    finals.add(FINAL);
+
+    StateMachine<STATES, EVENTS> stateMachine =
+        new StateMachine<>(INIT, finals);
+
+    stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE);
+    stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE);
+    stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE);
+    stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE);
+    stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE);
+    stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT);
+
+    // Initial and Final states
+    Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState());
+    Assert.assertTrue("Final States", SetUtils.isEqualSet(finals,
+        stateMachine.getFinalStates()));
+
+    // Valid state transitions
+    Assert.assertEquals("STATE should be OPERATIONAL after being created",
+        OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE));
+    Assert.assertEquals("STATE should be OPERATIONAL after being updated",
+        OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE));
+    Assert.assertEquals("STATE should be CLEANUP after being deleted",
+        CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE));
+    Assert.assertEquals("STATE should be CLEANUP after being timeout",
+        CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT));
+    Assert.assertEquals("STATE should be CLOSED after being closed",
+        CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE));
+
+    // Negative cases: invalid transition
+    expectException();
+    stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE);
+
+    expectException();
+    stateMachine.getNextState(CREATING, EVENTS.CLOSE);
+  }
+
+  /**
+   * We expect an InvalidStateTransitionException.
+   */
+  private void expectException() {
+    exception.expect(InvalidStateTransitionException.class);
+    exception.expectMessage("Invalid event");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
new file mode 100644
index 0000000..517c1a7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * A generic lease management API which can be used if a service
+ * needs any kind of lease management.
+ */
+
+package org.apache.hadoop.ozone.lease;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test class to check functionality and consistency of LeaseManager.
+ */
+public class TestLeaseManager {
+
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
+
+  /**
+   * Dummy resource on which leases can be acquired.
+   */
+  private final class DummyResource {
+
+    private final String name;
+
+    private DummyResource(String name) {
+      this.name = name;
+    }
+
+    @Override
+    public int hashCode() {
+      return name.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if(obj instanceof DummyResource) {
+        return name.equals(((DummyResource) obj).name);
+      }
+      return false;
+    }
+  }
+
+  @Test
+  public void testLeaseAcquireAndRelease() throws LeaseException {
+    //It is assumed that the test case execution won't take more than 5 seconds,
+    //if it takes more time increase the defaultTimeout value of LeaseManager.
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    DummyResource resourceTwo = new DummyResource("two");
+    DummyResource resourceThree = new DummyResource("three");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+    Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+    Assert.assertEquals(leaseThree, manager.get(resourceThree));
+    Assert.assertFalse(leaseOne.hasExpired());
+    Assert.assertFalse(leaseTwo.hasExpired());
+    Assert.assertFalse(leaseThree.hasExpired());
+    //The below releases should not throw LeaseNotFoundException.
+    manager.release(resourceOne);
+    manager.release(resourceTwo);
+    manager.release(resourceThree);
+    Assert.assertTrue(leaseOne.hasExpired());
+    Assert.assertTrue(leaseTwo.hasExpired());
+    Assert.assertTrue(leaseThree.hasExpired());
+    manager.shutdown();
+  }
+
+  @Test
+  public void testLeaseAlreadyExist() throws LeaseException {
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    DummyResource resourceTwo = new DummyResource("two");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+
+    exception.expect(LeaseAlreadyExistException.class);
+    exception.expectMessage("Resource: " + resourceOne);
+    manager.acquire(resourceOne);
+
+    manager.release(resourceOne);
+    manager.release(resourceTwo);
+    manager.shutdown();
+  }
+
+  @Test
+  public void testLeaseNotFound() throws LeaseException, InterruptedException {
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    DummyResource resourceTwo = new DummyResource("two");
+    DummyResource resourceThree = new DummyResource("three");
+
+    //Case 1: lease was never acquired.
+    exception.expect(LeaseNotFoundException.class);
+    exception.expectMessage("Resource: " + resourceOne);
+    manager.get(resourceOne);
+
+    //Case 2: lease is acquired and released.
+    Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+    Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+    Assert.assertFalse(leaseTwo.hasExpired());
+    manager.release(resourceTwo);
+    Assert.assertTrue(leaseTwo.hasExpired());
+    exception.expect(LeaseNotFoundException.class);
+    exception.expectMessage("Resource: " + resourceTwo);
+    manager.get(resourceTwo);
+
+    //Case 3: lease acquired and timed out.
+    Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
+    Assert.assertEquals(leaseThree, manager.get(resourceThree));
+    Assert.assertFalse(leaseThree.hasExpired());
+    long sleepTime = leaseThree.getRemainingTime() + 1000;
+    try {
+      Thread.sleep(sleepTime);
+    } catch (InterruptedException ex) {
+      //even in case of interrupt we have to wait till lease times out.
+      Thread.sleep(sleepTime);
+    }
+    Assert.assertTrue(leaseThree.hasExpired());
+    exception.expect(LeaseNotFoundException.class);
+    exception.expectMessage("Resource: " + resourceThree);
+    manager.get(resourceThree);
+    manager.shutdown();
+  }
+
+  @Test
+  public void testCustomLeaseTimeout() throws LeaseException {
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    DummyResource resourceTwo = new DummyResource("two");
+    DummyResource resourceThree = new DummyResource("three");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo, 10000);
+    Lease<DummyResource> leaseThree = manager.acquire(resourceThree, 50000);
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertEquals(leaseTwo, manager.get(resourceTwo));
+    Assert.assertEquals(leaseThree, manager.get(resourceThree));
+    Assert.assertFalse(leaseOne.hasExpired());
+    Assert.assertFalse(leaseTwo.hasExpired());
+    Assert.assertFalse(leaseThree.hasExpired());
+    Assert.assertEquals(5000, leaseOne.getLeaseLifeTime());
+    Assert.assertEquals(10000, leaseTwo.getLeaseLifeTime());
+    Assert.assertEquals(50000, leaseThree.getLeaseLifeTime());
+    // Releasing of leases is done in shutdown, so don't have to worry about
+    // lease release
+    manager.shutdown();
+  }
+
+  @Test
+  public void testLeaseCallback() throws LeaseException, InterruptedException {
+    Map<DummyResource, String> leaseStatus = new HashMap<>();
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    leaseStatus.put(resourceOne, "lease in use");
+    leaseOne.registerCallBack(() -> {
+      leaseStatus.put(resourceOne, "lease expired");
+      return null;
+    });
+    // wait for lease to expire
+    long sleepTime = leaseOne.getRemainingTime() + 1000;
+    try {
+      Thread.sleep(sleepTime);
+    } catch (InterruptedException ex) {
+      //even in case of interrupt we have to wait till lease times out.
+      Thread.sleep(sleepTime);
+    }
+    Assert.assertTrue(leaseOne.hasExpired());
+    exception.expect(LeaseNotFoundException.class);
+    exception.expectMessage("Resource: " + resourceOne);
+    manager.get(resourceOne);
+    // check if callback has been executed
+    Assert.assertEquals("lease expired", leaseStatus.get(resourceOne));
+  }
+
+  @Test
+  public void testCallbackExecutionInCaseOfLeaseRelease()
+      throws LeaseException, InterruptedException {
+    // Callbacks should not be executed in case of lease release
+    Map<DummyResource, String> leaseStatus = new HashMap<>();
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    leaseStatus.put(resourceOne, "lease in use");
+    leaseOne.registerCallBack(() -> {
+      leaseStatus.put(resourceOne, "lease expired");
+      return null;
+    });
+    leaseStatus.put(resourceOne, "lease released");
+    manager.release(resourceOne);
+    Assert.assertTrue(leaseOne.hasExpired());
+    exception.expect(LeaseNotFoundException.class);
+    exception.expectMessage("Resource: " + resourceOne);
+    manager.get(resourceOne);
+    Assert.assertEquals("lease released", leaseStatus.get(resourceOne));
+  }
+
+  @Test
+  public void testLeaseCallbackWithMultipleLeases()
+      throws LeaseException, InterruptedException {
+    Map<DummyResource, String> leaseStatus = new HashMap<>();
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    DummyResource resourceTwo = new DummyResource("two");
+    DummyResource resourceThree = new DummyResource("three");
+    DummyResource resourceFour = new DummyResource("four");
+    DummyResource resourceFive = new DummyResource("five");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo);
+    Lease<DummyResource> leaseThree = manager.acquire(resourceThree);
+    Lease<DummyResource> leaseFour = manager.acquire(resourceFour);
+    Lease<DummyResource> leaseFive = manager.acquire(resourceFive);
+    leaseStatus.put(resourceOne, "lease in use");
+    leaseStatus.put(resourceTwo, "lease in use");
+    leaseStatus.put(resourceThree, "lease in use");
+    leaseStatus.put(resourceFour, "lease in use");
+    leaseStatus.put(resourceFive, "lease in use");
+    leaseOne.registerCallBack(() -> {
+      leaseStatus.put(resourceOne, "lease expired");
+      return null;
+    });
+    leaseTwo.registerCallBack(() -> {
+      leaseStatus.put(resourceTwo, "lease expired");
+      return null;
+    });
+    leaseThree.registerCallBack(() -> {
+      leaseStatus.put(resourceThree, "lease expired");
+      return null;
+    });
+    leaseFour.registerCallBack(() -> {
+      leaseStatus.put(resourceFour, "lease expired");
+      return null;
+    });
+    leaseFive.registerCallBack(() -> {
+      leaseStatus.put(resourceFive, "lease expired");
+      return null;
+    });
+
+    // release lease one, two and three
+    leaseStatus.put(resourceOne, "lease released");
+    manager.release(resourceOne);
+    leaseStatus.put(resourceTwo, "lease released");
+    manager.release(resourceTwo);
+    leaseStatus.put(resourceThree, "lease released");
+    manager.release(resourceThree);
+
+    // wait for other leases to expire
+    long sleepTime = leaseFive.getRemainingTime() + 1000;
+
+    try {
+      Thread.sleep(sleepTime);
+    } catch (InterruptedException ex) {
+      //even in case of interrupt we have to wait till lease times out.
+      Thread.sleep(sleepTime);
+    }
+    Assert.assertTrue(leaseOne.hasExpired());
+    Assert.assertTrue(leaseTwo.hasExpired());
+    Assert.assertTrue(leaseThree.hasExpired());
+    Assert.assertTrue(leaseFour.hasExpired());
+    Assert.assertTrue(leaseFive.hasExpired());
+
+    Assert.assertEquals("lease released", leaseStatus.get(resourceOne));
+    Assert.assertEquals("lease released", leaseStatus.get(resourceTwo));
+    Assert.assertEquals("lease released", leaseStatus.get(resourceThree));
+    Assert.assertEquals("lease expired", leaseStatus.get(resourceFour));
+    Assert.assertEquals("lease expired", leaseStatus.get(resourceFive));
+    manager.shutdown();
+  }
+
+  @Test
+  public void testReuseReleasedLease() throws LeaseException {
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertFalse(leaseOne.hasExpired());
+
+    manager.release(resourceOne);
+    Assert.assertTrue(leaseOne.hasExpired());
+
+    Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne);
+    Assert.assertEquals(sameResourceLease, manager.get(resourceOne));
+    Assert.assertFalse(sameResourceLease.hasExpired());
+
+    manager.release(resourceOne);
+    Assert.assertTrue(sameResourceLease.hasExpired());
+    manager.shutdown();
+  }
+
+  @Test
+  public void testReuseTimedOutLease()
+      throws LeaseException, InterruptedException {
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertFalse(leaseOne.hasExpired());
+
+    // wait for lease to expire
+    long sleepTime = leaseOne.getRemainingTime() + 1000;
+    try {
+      Thread.sleep(sleepTime);
+    } catch (InterruptedException ex) {
+      //even in case of interrupt we have to wait till lease times out.
+      Thread.sleep(sleepTime);
+    }
+    Assert.assertTrue(leaseOne.hasExpired());
+
+    Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne);
+    Assert.assertEquals(sameResourceLease, manager.get(resourceOne));
+    Assert.assertFalse(sameResourceLease.hasExpired());
+
+    manager.release(resourceOne);
+    Assert.assertTrue(sameResourceLease.hasExpired());
+    manager.shutdown();
+  }
+
+  @Test
+  public void testRenewLease() throws LeaseException, InterruptedException {
+    LeaseManager<DummyResource> manager = new LeaseManager<>(5000);
+    manager.start();
+    DummyResource resourceOne = new DummyResource("one");
+    Lease<DummyResource> leaseOne = manager.acquire(resourceOne);
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertFalse(leaseOne.hasExpired());
+
+    // add 5 more seconds to the lease
+    leaseOne.renew(5000);
+
+    Thread.sleep(5000);
+
+    // lease should still be active
+    Assert.assertEquals(leaseOne, manager.get(resourceOne));
+    Assert.assertFalse(leaseOne.hasExpired());
+    manager.release(resourceOne);
+    manager.shutdown();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
new file mode 100644
index 0000000..1071309
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.lease;
+/*
+ This package contains lease management unit test classes.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java
new file mode 100644
index 0000000..0030d2e
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+/**
+ * Ozone related test helper classes and tests of common utils.
+ */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message