hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [2/2] hadoop git commit: HDDS-18. Ozone Shell should use RestClient and RpcClient. Contributed by Lokesh Jain.
Date Thu, 10 May 2018 06:00:31 GMT
HDDS-18. Ozone Shell should use RestClient and RpcClient. Contributed by Lokesh Jain.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46e0f278
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46e0f278
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46e0f278

Branch: refs/heads/trunk
Commit: 46e0f2786259ecd25e9a5a1e51b667f9c32e5c56
Parents: 3c95ca4
Author: Anu Engineer <aengineer@apache.org>
Authored: Wed May 9 21:23:33 2018 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Wed May 9 23:00:13 2018 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/ozone/OzoneConsts.java    |  12 +-
 .../src/test/compose/docker-compose.yaml        |   1 +
 .../test/robotframework/acceptance/ozone.robot  |  13 +-
 .../apache/hadoop/ozone/client/ObjectStore.java |  49 +--
 .../apache/hadoop/ozone/client/OzoneBucket.java |  25 +-
 .../ozone/client/OzoneClientException.java      |  54 ++++
 .../hadoop/ozone/client/OzoneClientFactory.java |   2 +-
 .../hadoop/ozone/client/OzoneClientUtils.java   |  86 ++++++
 .../apache/hadoop/ozone/client/OzoneVolume.java |  26 +-
 .../hadoop/ozone/client/rest/RestClient.java    |   8 +-
 .../hadoop/ozone/web/client/OzoneBucket.java    |  47 +--
 .../ozone/web/client/OzoneRestClient.java       |  43 +--
 .../web/client/OzoneRestClientException.java    |  54 ----
 .../hadoop/ozone/web/client/OzoneVolume.java    |  23 +-
 .../hadoop/ozone/ozShell/TestOzoneShell.java    | 298 +++++++++++++------
 .../hadoop/ozone/web/client/TestBuckets.java    |   3 +-
 .../hadoop/ozone/web/client/TestVolume.java     |   3 +-
 .../hadoop/ozone/web/ozShell/Handler.java       |  59 +++-
 .../web/ozShell/bucket/CreateBucketHandler.java |  32 +-
 .../web/ozShell/bucket/DeleteBucketHandler.java |  20 +-
 .../web/ozShell/bucket/InfoBucketHandler.java   |  25 +-
 .../web/ozShell/bucket/ListBucketHandler.java   |  51 ++--
 .../web/ozShell/bucket/UpdateBucketHandler.java |  41 +--
 .../web/ozShell/keys/DeleteKeyHandler.java      |  26 +-
 .../ozone/web/ozShell/keys/GetKeyHandler.java   |  46 +--
 .../ozone/web/ozShell/keys/InfoKeyHandler.java  |  31 +-
 .../ozone/web/ozShell/keys/ListKeyHandler.java  |  48 +--
 .../ozone/web/ozShell/keys/PutKeyHandler.java   |  49 ++-
 .../web/ozShell/volume/CreateVolumeHandler.java |  28 +-
 .../web/ozShell/volume/DeleteVolumeHandler.java |  18 +-
 .../web/ozShell/volume/InfoVolumeHandler.java   |  38 +--
 .../web/ozShell/volume/ListVolumeHandler.java   |  58 ++--
 .../web/ozShell/volume/UpdateVolumeHandler.java |  24 +-
 .../org/apache/hadoop/fs/ozone/Constants.java   |   6 -
 .../java/org/apache/hadoop/fs/ozone/OzFs.java   |   3 +-
 .../apache/hadoop/fs/ozone/OzoneFileSystem.java |   4 +-
 .../hadoop/fs/ozone/TestOzoneFSInputStream.java |   3 +-
 .../fs/ozone/TestOzoneFileInterfaces.java       |   5 +-
 .../hadoop/fs/ozone/contract/OzoneContract.java |   6 +-
 39 files changed, 796 insertions(+), 572 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 2f9e469..451a08f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -58,6 +58,10 @@ public final class OzoneConsts {
   public static final String OZONE_USER = "user";
   public static final String OZONE_REQUEST = "request";
 
+  public static final String OZONE_URI_SCHEME = "o3";
+  public static final String OZONE_HTTP_SCHEME = "http";
+  public static final String OZONE_URI_DELIMITER = "/";
+
   public static final String CONTAINER_EXTENSION = ".container";
   public static final String CONTAINER_META = ".meta";
 
@@ -95,7 +99,13 @@ public final class OzoneConsts {
   /**
    * Supports Bucket Versioning.
    */
-  public enum Versioning {NOT_DEFINED, ENABLED, DISABLED}
+  public enum Versioning {
+    NOT_DEFINED, ENABLED, DISABLED;
+
+    public static Versioning getVersioning(boolean versioning) {
+      return versioning ? ENABLED : DISABLED;
+    }
+  }
 
   /**
    * Ozone handler types.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
index 8350eae..da63f84 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
@@ -39,6 +39,7 @@ services:
         - ./docker-config
    ksm:
       image: apache/hadoop-runner
+      hostname: ksm
       volumes:
          - ${HADOOPDIR}:/opt/hadoop
       ports:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
index ea9131e..1a9cee7 100644
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
@@ -49,14 +49,14 @@ Test rest interface
                     Should contain      ${result}       200 OK
 
 Test ozone cli
-                    Execute on          datanode        ozone oz -createVolume http://localhost:9880/hive -user bilbo -quota 100TB -root
-    ${result} =     Execute on          datanode        ozone oz -listVolume http://localhost:9880/ -user bilbo | grep -v Removed | jq '.[] | select(.volumeName=="hive")'
+                    Execute on          datanode        ozone oz -createVolume http://ksm/hive -user bilbo -quota 100TB -root
+    ${result} =     Execute on          datanode        ozone oz -listVolume o3://ksm -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
                     Should contain      ${result}       createdOn
-                    Execute on          datanode        ozone oz -createBucket http://localhost:9880/hive/bb1
-    ${result}       Execute on          datanode        ozone oz -listBucket http://localhost:9880/hive/ | grep -v Removed | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
+                    Execute on          datanode        ozone oz -createBucket http://ksm/hive/bb1
+    ${result}       Execute on          datanode        ozone oz -listBucket o3://ksm/hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
                     Should Be Equal     ${result}       hive
-                    Execute on          datanode        ozone oz -deleteBucket http://localhost:9880/hive/bb1
-                    Execute on          datanode        ozone oz -deleteVolume http://localhost:9880/hive -user bilbo
+                    Execute on          datanode        ozone oz -deleteBucket http://ksm/hive/bb1
+                    Execute on          datanode        ozone oz -deleteVolume http://ksm/hive -user bilbo
 
 
 
@@ -106,6 +106,7 @@ Scale datanodes up
 Execute on
     [arguments]     ${componentname}    ${command}
     ${rc}           ${return} =         Run docker compose          exec ${componentname} ${command}
+    Log             ${return}
     [return]        ${return}
 
 Run docker compose

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index b915213..d8b3011 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -106,25 +106,42 @@ public class ObjectStore {
    */
   public Iterator<OzoneVolume> listVolumes(String volumePrefix)
       throws IOException {
-    return new VolumeIterator(volumePrefix);
+    return listVolumes(volumePrefix, null);
   }
 
   /**
-   * Returns Iterator to iterate over the List of volumes owned by a specific
-   * user. The result can be restricted using volume prefix, will return all
-   * volumes if volume prefix is null. If user is null, returns the volume of
-   * current user.
+   * Returns Iterator to iterate over all the volumes after prevVolume in object
+   * store. If prevVolume is null it iterates from the first volume.
+   * The result can be restricted using volume prefix, will return all
+   * volumes if volume prefix is null.
+   *
+   * @param volumePrefix Volume prefix to match
+   * @param prevVolume Volumes will be listed after this volume name
+   * @return {@code Iterator<OzoneVolume>}
+   */
+  public Iterator<OzoneVolume> listVolumes(String volumePrefix,
+      String prevVolume) throws IOException {
+    return new VolumeIterator(null, volumePrefix, prevVolume);
+  }
+
+  /**
+   * Returns Iterator to iterate over the list of volumes after prevVolume owned
+   * by a specific user. The result can be restricted using volume prefix, will
+   * return all volumes if volume prefix is null. If user is not null, returns
+   * the volume of current user.
    *
    * @param user User Name
    * @param volumePrefix Volume prefix to match
+   * @param prevVolume Volumes will be listed after this volume name
    * @return {@code Iterator<OzoneVolume>}
    */
-  public Iterator<OzoneVolume> listVolumes(String user, String volumePrefix)
+  public Iterator<OzoneVolume> listVolumesByUser(String user,
+      String volumePrefix, String prevVolume)
       throws IOException {
     if(Strings.isNullOrEmpty(user)) {
       user = UserGroupInformation.getCurrentUser().getShortUserName();
     }
-    return new VolumeIterator(user, volumePrefix);
+    return new VolumeIterator(user, volumePrefix, prevVolume);
   }
 
   /**
@@ -150,25 +167,17 @@ public class ObjectStore {
     private OzoneVolume currentValue;
 
     /**
-     * Creates an Iterator to iterate over all volumes in the cluster,
-     * which matches the volume prefix.
-     * @param volPrefix prefix to match
-     */
-    VolumeIterator(String volPrefix) {
-      this(null, volPrefix);
-    }
-
-    /**
-     * Creates an Iterator to iterate over all volumes of the user,
-     * which matches volume prefix.
+     * Creates an Iterator to iterate over all volumes after prevVolume of the user.
+     * If prevVolume is null it iterates from the first volume. The returned volumes
+     * match volume prefix.
      * @param user user name
      * @param volPrefix volume prefix to match
      */
-    VolumeIterator(String user, String volPrefix) {
+    VolumeIterator(String user, String volPrefix, String prevVolume) {
       this.user = user;
       this.volPrefix = volPrefix;
       this.currentValue = null;
-      this.currentIterator = getNextListOfVolumes(null).iterator();
+      this.currentIterator = getNextListOfVolumes(prevVolume).iterator();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 1712979..5df0254 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -291,7 +291,21 @@ public class OzoneBucket {
    * @return {@code Iterator<OzoneKey>}
    */
   public Iterator<OzoneKey> listKeys(String keyPrefix) {
-    return new KeyIterator(keyPrefix);
+    return listKeys(keyPrefix, null);
+  }
+
+  /**
+   * Returns Iterator to iterate over all keys after prevKey in the bucket.
+   * If prevKey is null it iterates from the first key in the bucket.
+   * The result can be restricted using key prefix, will return all
+   * keys if key prefix is null.
+   *
+   * @param keyPrefix Bucket prefix to match
+   * @param prevKey Keys will be listed after this key name
+   * @return {@code Iterator<OzoneKey>}
+   */
+  public Iterator<OzoneKey> listKeys(String keyPrefix, String prevKey) {
+    return new KeyIterator(keyPrefix, prevKey);
   }
 
   /**
@@ -325,14 +339,15 @@ public class OzoneBucket {
 
 
     /**
-     * Creates an Iterator to iterate over all keys in the bucket,
-     * which matches volume prefix.
+     * Creates an Iterator to iterate over all keys after prevKey in the bucket.
+     * If prevKey is null it iterates from the first key in the bucket.
+     * The returned keys match key prefix.
      * @param keyPrefix
      */
-    KeyIterator(String keyPrefix) {
+    KeyIterator(String keyPrefix, String prevKey) {
       this.keyPrefix = keyPrefix;
       this.currentValue = null;
-      this.currentIterator = getNextListOfKeys(null).iterator();
+      this.currentIterator = getNextListOfKeys(prevKey).iterator();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java
new file mode 100644
index 0000000..de3116a
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+
+/**
+ * This exception is thrown by the Ozone Clients.
+ */
+public class OzoneClientException extends OzoneException {
+  /**
+   * Constructor that allows the shortMessage.
+   *
+   * @param shortMessage Short Message
+   */
+  public OzoneClientException(String shortMessage) {
+    super(0, shortMessage, shortMessage);
+  }
+
+  /**
+   * Constructor that allows a shortMessage and an exception.
+   *
+   * @param shortMessage short message
+   * @param ex exception
+   */
+  public OzoneClientException(String shortMessage, Exception ex) {
+    super(0, shortMessage, shortMessage, ex);
+  }
+
+  /**
+   * Constructor that allows the shortMessage and a longer message.
+   *
+   * @param shortMessage Short Message
+   * @param message long error message
+   */
+  public OzoneClientException(String shortMessage, String message) {
+    super(0, shortMessage, message);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
index 6b24f2a..dae94aa 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
@@ -292,7 +292,7 @@ public final class OzoneClientFactory {
       return ctor.newInstance(config);
     } catch (Exception e) {
       final String message = "Couldn't create protocol " + protocolClass;
-      LOG.error(message + " exception:" + e);
+      LOG.error(message + " exception: ", e);
       if (e.getCause() instanceof IOException) {
         throw (IOException) e.getCause();
       } else if (e instanceof InvocationTargetException) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
new file mode 100644
index 0000000..5c83d9b
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
+import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
+import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
+import org.apache.hadoop.ozone.client.rest.response.VolumeOwner;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+
+import static org.apache.hadoop.ozone.web.utils.OzoneUtils.formatTime;
+
+/** A utility class for OzoneClient. */
+public final class OzoneClientUtils {
+
+  private OzoneClientUtils() {}
+
+  /**
+   * Returns a BucketInfo object constructed using fields of the input
+   * OzoneBucket object.
+   *
+   * @param bucket OzoneBucket instance from which BucketInfo object needs to
+   *               be created.
+   * @return BucketInfo instance
+   */
+  public static BucketInfo asBucketInfo(OzoneBucket bucket) {
+    BucketInfo bucketInfo =
+        new BucketInfo(bucket.getVolumeName(), bucket.getName());
+    bucketInfo.setCreatedOn(OzoneUtils.formatTime(bucket.getCreationTime()));
+    bucketInfo.setStorageType(bucket.getStorageType());
+    bucketInfo.setVersioning(
+        OzoneConsts.Versioning.getVersioning(bucket.getVersioning()));
+    return bucketInfo;
+  }
+
+  /**
+   * Returns a VolumeInfo object constructed using fields of the input
+   * OzoneVolume object.
+   *
+   * @param volume OzoneVolume instance from which VolumeInfo object needs to
+   *               be created.
+   * @return VolumeInfo instance
+   */
+  public static VolumeInfo asVolumeInfo(OzoneVolume volume) {
+    VolumeInfo volumeInfo =
+        new VolumeInfo(volume.getName(), formatTime(volume.getCreationTime()),
+            volume.getOwner());
+    volumeInfo.setQuota(OzoneQuota.getOzoneQuota(volume.getQuota()));
+    volumeInfo.setOwner(new VolumeOwner(volume.getOwner()));
+    return volumeInfo;
+  }
+
+  /**
+   * Returns a KeyInfo object constructed using fields of the input
+   * OzoneKey object.
+   *
+   * @param key OzoneKey instance from which KeyInfo object needs to
+   *            be created.
+   * @return KeyInfo instance
+   */
+  public static KeyInfo asKeyInfo(OzoneKey key) {
+    KeyInfo keyInfo = new KeyInfo();
+    keyInfo.setKeyName(key.getName());
+    keyInfo.setCreatedOn(formatTime(key.getCreationTime()));
+    keyInfo.setModifiedOn(formatTime(key.getModificationTime()));
+    keyInfo.setSize(key.getDataSize());
+    return keyInfo;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
index c4e7331..4601f1a 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
@@ -222,7 +222,22 @@ public class OzoneVolume {
    * @return {@code Iterator<OzoneBucket>}
    */
   public Iterator<OzoneBucket> listBuckets(String bucketPrefix) {
-    return new BucketIterator(bucketPrefix);
+    return listBuckets(bucketPrefix, null);
+  }
+
+  /**
+   * Returns Iterator to iterate over all buckets after prevBucket in the volume.
+   * If prevBucket is null it iterates from the first bucket in the volume.
+   * The result can be restricted using bucket prefix, will return all
+   * buckets if bucket prefix is null.
+   *
+   * @param bucketPrefix Bucket prefix to match
+   * @param prevBucket Buckets are listed after this bucket
+   * @return {@code Iterator<OzoneBucket>}
+   */
+  public Iterator<OzoneBucket> listBuckets(String bucketPrefix,
+      String prevBucket) {
+    return new BucketIterator(bucketPrefix, prevBucket);
   }
 
   /**
@@ -250,14 +265,15 @@ public class OzoneVolume {
 
 
     /**
-     * Creates an Iterator to iterate over all buckets in the volume,
-     * which matches volume prefix.
+     * Creates an Iterator to iterate over all buckets after prevBucket in the volume.
+     * If prevBucket is null it iterates from the first bucket in the volume.
+     * The returned buckets match bucket prefix.
      * @param bucketPrefix
      */
-    BucketIterator(String bucketPrefix) {
+    public BucketIterator(String bucketPrefix, String prevBucket) {
       this.bucketPrefix = bucketPrefix;
       this.currentValue = null;
-      this.currentIterator = getNextListOfBuckets(null).iterator();
+      this.currentIterator = getNextListOfBuckets(prevBucket).iterator();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index e9885d1..1fd2091 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.ozone.client.rest.headers.Header;
 import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
 import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
 import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
 import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.protocol.proto
@@ -94,7 +93,7 @@ import static java.net.HttpURLConnection.HTTP_OK;
 public class RestClient implements ClientProtocol {
 
   private static final String PATH_SEPARATOR = "/";
-  private static final Logger LOG = LoggerFactory.getLogger(RpcClient.class);
+  private static final Logger LOG = LoggerFactory.getLogger(RestClient.class);
 
   private final Configuration conf;
   private final URI ozoneRestUri;
@@ -195,8 +194,9 @@ public class RestClient implements ClientProtocol {
 
       ServiceInfo restServer = selector.getRestServer(dataNodeInfos);
 
-      return NetUtils.createSocketAddr(restServer.getHostname() + ":" +
-          restServer.getPort(ServicePort.Type.HTTP));
+      return NetUtils.createSocketAddr(
+          NetUtils.normalizeHostName(restServer.getHostname()) + ":"
+              + restServer.getPort(ServicePort.Type.HTTP));
     } finally {
       EntityUtils.consume(entity);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
index e6fe0ec..3183d03 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.io.IOUtils;
 
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
 import org.apache.hadoop.ozone.client.rest.headers.Header;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -169,11 +170,11 @@ public class OzoneBucket {
    */
   public void putKey(String keyName, String data) throws OzoneException {
     if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name.");
+      throw new OzoneClientException("Invalid key Name.");
     }
 
     if (data == null) {
-      throw new OzoneRestClientException("Invalid data.");
+      throw new OzoneClientException("Invalid data.");
     }
 
     HttpPut putRequest = null;
@@ -195,7 +196,7 @@ public class OzoneBucket {
       }
       executePutKey(putRequest, httpClient);
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       IOUtils.closeStream(is);
       releaseConnection(putRequest);
@@ -211,7 +212,7 @@ public class OzoneBucket {
    */
   public void putKey(File dataFile) throws OzoneException {
     if (dataFile == null) {
-      throw new OzoneRestClientException("Invalid file object.");
+      throw new OzoneClientException("Invalid file object.");
     }
     String keyName = dataFile.getName();
     putKey(keyName, dataFile);
@@ -228,11 +229,11 @@ public class OzoneBucket {
       throws OzoneException {
 
     if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
+      throw new OzoneClientException("Invalid key Name");
     }
 
     if (file == null) {
-      throw new OzoneRestClientException("Invalid data stream");
+      throw new OzoneClientException("Invalid data stream");
     }
 
     HttpPut putRequest = null;
@@ -253,7 +254,7 @@ public class OzoneBucket {
       executePutKey(putRequest, httpClient);
 
     } catch (IOException | URISyntaxException ex) {
-      final OzoneRestClientException orce = new OzoneRestClientException(
+      final OzoneClientException orce = new OzoneClientException(
           "Failed to putKey: keyName=" + keyName + ", file=" + file);
       orce.initCause(ex);
       LOG.trace("", orce);
@@ -285,7 +286,7 @@ public class OzoneBucket {
       }
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
 
       throw OzoneException.parse(EntityUtils.toString(entity));
@@ -306,11 +307,11 @@ public class OzoneBucket {
   public void getKey(String keyName, Path downloadTo) throws OzoneException {
 
     if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
+      throw new OzoneClientException("Invalid key Name");
     }
 
     if (downloadTo == null) {
-      throw new OzoneRestClientException("Invalid download path");
+      throw new OzoneClientException("Invalid download path");
     }
 
     FileOutputStream outPutFile = null;
@@ -326,7 +327,7 @@ public class OzoneBucket {
       executeGetKey(getRequest, httpClient, outPutFile);
       outPutFile.flush();
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       IOUtils.closeStream(outPutFile);
       releaseConnection(getRequest);
@@ -343,7 +344,7 @@ public class OzoneBucket {
   public String getKey(String keyName) throws OzoneException {
 
     if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
+      throw new OzoneClientException("Invalid key Name");
     }
 
     HttpGet getRequest = null;
@@ -360,7 +361,7 @@ public class OzoneBucket {
       executeGetKey(getRequest, httpClient, outPutStream);
       return outPutStream.toString(ENCODING_NAME);
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       IOUtils.closeStream(outPutStream);
       releaseConnection(getRequest);
@@ -394,7 +395,7 @@ public class OzoneBucket {
       }
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
 
       throw OzoneException.parse(EntityUtils.toString(entity));
@@ -414,7 +415,7 @@ public class OzoneBucket {
   public void deleteKey(String keyName) throws OzoneException {
 
     if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException("Invalid key Name");
+      throw new OzoneClientException("Invalid key Name");
     }
 
     HttpDelete deleteRequest = null;
@@ -427,7 +428,7 @@ public class OzoneBucket {
           .getClient().getHttpDelete(builder.toString());
       executeDeleteKey(deleteRequest, httpClient);
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(deleteRequest);
     }
@@ -457,7 +458,7 @@ public class OzoneBucket {
       }
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
 
       throw OzoneException.parse(EntityUtils.toString(entity));
@@ -505,7 +506,7 @@ public class OzoneBucket {
       return executeListKeys(getRequest, httpClient);
 
     } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
+      throw new OzoneClientException(e.getMessage(), e);
     } finally {
       releaseConnection(getRequest);
     }
@@ -534,7 +535,7 @@ public class OzoneBucket {
       getRequest = client.getHttpGet(uri.toString());
       return executeListKeys(getRequest, HttpClientBuilder.create().build());
     } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage());
+      throw new OzoneClientException(e.getMessage());
     } finally {
       releaseConnection(getRequest);
     }
@@ -560,7 +561,7 @@ public class OzoneBucket {
       entity = response.getEntity();
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
       if (errorCode == HTTP_OK) {
         String temp = EntityUtils.toString(entity);
@@ -586,7 +587,7 @@ public class OzoneBucket {
    */
   public OzoneKey getKeyInfo(String keyName) throws OzoneException {
     if ((keyName == null) || keyName.isEmpty()) {
-      throw new OzoneRestClientException(
+      throw new OzoneClientException(
           "Unable to get key info, key name is null or empty");
     }
 
@@ -604,7 +605,7 @@ public class OzoneBucket {
       getRequest = client.getHttpGet(builder.toString());
       return executeGetKeyInfo(getRequest, httpClient);
     } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
+      throw new OzoneClientException(e.getMessage(), e);
     } finally {
       releaseConnection(getRequest);
     }
@@ -627,7 +628,7 @@ public class OzoneBucket {
       int errorCode = response.getStatusLine().getStatusCode();
       entity = response.getEntity();
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
 
       if (errorCode == HTTP_OK) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
index 6d0bbf4..8373f67 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
@@ -24,6 +24,7 @@ import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
 import org.apache.hadoop.ozone.client.rest.headers.Header;
 import org.apache.hadoop.ozone.web.response.ListVolumes;
@@ -111,7 +112,7 @@ public class OzoneRestClient implements Closeable {
    */
   public void setEndPointURI(URI endPointURI) throws OzoneException {
     if ((endPointURI == null) || (endPointURI.toString().isEmpty())) {
-      throw new OzoneRestClientException("Invalid ozone URI");
+      throw new OzoneClientException("Invalid ozone URI");
     }
     this.endPointURI = endPointURI;
   }
@@ -151,7 +152,7 @@ public class OzoneRestClient implements Closeable {
    * @param onBehalfOf - The user on behalf we are making the call for
    * @param quota      - Quota's are specified in a specific format. it is
    *                   integer(MB|GB|TB), for example 100TB.
-   * @throws OzoneRestClientException
+   * @throws OzoneClientException
    */
   public OzoneVolume createVolume(String volumeName, String onBehalfOf,
                                   String quota) throws OzoneException {
@@ -169,7 +170,7 @@ public class OzoneRestClient implements Closeable {
       executeCreateVolume(httpPost, httpClient);
       return getVolume(volumeName);
     } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(httpPost);
     }
@@ -196,7 +197,7 @@ public class OzoneRestClient implements Closeable {
       httpGet = getHttpGet(builder.toString());
       return executeInfoVolume(httpGet, httpClient);
     } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(httpGet);
     }
@@ -247,7 +248,7 @@ public class OzoneRestClient implements Closeable {
       }
       return executeListVolume(httpGet, httpClient);
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(httpGet);
     }
@@ -329,7 +330,7 @@ public class OzoneRestClient implements Closeable {
       return executeListVolume(httpGet, httpClient);
 
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(httpGet);
     }
@@ -351,7 +352,7 @@ public class OzoneRestClient implements Closeable {
       httpDelete = getHttpDelete(builder.toString());
       executeDeleteVolume(httpDelete, httpClient);
     } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(httpDelete);
     }
@@ -368,7 +369,7 @@ public class OzoneRestClient implements Closeable {
       throws OzoneException {
     HttpPut putRequest = null;
     if (newOwner == null || newOwner.isEmpty()) {
-      throw new OzoneRestClientException("Invalid new owner name");
+      throw new OzoneClientException("Invalid new owner name");
     }
     try (CloseableHttpClient httpClient = newHttpClient()) {
       OzoneUtils.verifyResourceName(volumeName);
@@ -380,7 +381,7 @@ public class OzoneRestClient implements Closeable {
       executePutVolume(putRequest, httpClient);
 
     } catch (URISyntaxException | IllegalArgumentException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(putRequest);
     }
@@ -399,7 +400,7 @@ public class OzoneRestClient implements Closeable {
   public void setVolumeQuota(String volumeName, String quota)
       throws OzoneException {
     if (quota == null || quota.isEmpty()) {
-      throw new OzoneRestClientException("Invalid quota");
+      throw new OzoneClientException("Invalid quota");
     }
     HttpPut putRequest = null;
     try (CloseableHttpClient httpClient = newHttpClient()) {
@@ -413,7 +414,7 @@ public class OzoneRestClient implements Closeable {
       executePutVolume(putRequest, httpClient);
 
     } catch (URISyntaxException | IllegalArgumentException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(putRequest);
     }
@@ -443,7 +444,7 @@ public class OzoneRestClient implements Closeable {
       if (entity != null) {
         throw OzoneException.parse(EntityUtils.toString(entity));
       } else {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
     } finally {
       if (entity != null) {
@@ -470,7 +471,7 @@ public class OzoneRestClient implements Closeable {
 
       entity = response.getEntity();
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
 
       if (errorCode == HTTP_OK) {
@@ -531,7 +532,7 @@ public class OzoneRestClient implements Closeable {
       entity = response.getEntity();
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
 
       String temp = EntityUtils.toString(entity);
@@ -595,11 +596,11 @@ public class OzoneRestClient implements Closeable {
     OzoneUtils.verifyResourceName(bucketName);
 
     if (StringUtils.isEmpty(keyName)) {
-      throw new OzoneRestClientException("Invalid key Name");
+      throw new OzoneClientException("Invalid key Name");
     }
 
     if (file == null) {
-      throw new OzoneRestClientException("Invalid data stream");
+      throw new OzoneClientException("Invalid data stream");
     }
 
     HttpPut putRequest = null;
@@ -619,7 +620,7 @@ public class OzoneRestClient implements Closeable {
       putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis));
       OzoneBucket.executePutKey(putRequest, httpClient);
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       IOUtils.closeStream(fis);
       releaseConnection(putRequest);
@@ -641,11 +642,11 @@ public class OzoneRestClient implements Closeable {
     OzoneUtils.verifyResourceName(bucketName);
 
     if (StringUtils.isEmpty(keyName)) {
-      throw new OzoneRestClientException("Invalid key Name");
+      throw new OzoneClientException("Invalid key Name");
     }
 
     if (downloadTo == null) {
-      throw new OzoneRestClientException("Invalid download path");
+      throw new OzoneClientException("Invalid download path");
     }
 
     FileOutputStream outPutFile = null;
@@ -661,7 +662,7 @@ public class OzoneRestClient implements Closeable {
       OzoneBucket.executeGetKey(getRequest, httpClient, outPutFile);
       outPutFile.flush();
     } catch (IOException | URISyntaxException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       IOUtils.closeStream(outPutFile);
       releaseConnection(getRequest);
@@ -706,7 +707,7 @@ public class OzoneRestClient implements Closeable {
       getRequest = getHttpGet(builder.toString());
       return OzoneBucket.executeListKeys(getRequest, httpClient);
     } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
+      throw new OzoneClientException(e.getMessage(), e);
     } finally {
       releaseConnection(getRequest);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
deleted file mode 100644
index dfb2357..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-
-/**
- * This exception is thrown by the Ozone Clients.
- */
-public class OzoneRestClientException extends OzoneException {
-  /**
-   * Constructor that allows the shortMessage.
-   *
-   * @param shortMessage Short Message
-   */
-  public OzoneRestClientException(String shortMessage) {
-    super(0, shortMessage, shortMessage);
-  }
-
-  /**
-   * Constructor that allows a shortMessage and an exception.
-   *
-   * @param shortMessage short message
-   * @param ex exception
-   */
-  public OzoneRestClientException(String shortMessage, Exception ex) {
-    super(0, shortMessage, shortMessage, ex);
-  }
-
-  /**
-   * Constructor that allows the shortMessage and a longer message.
-   *
-   * @param shortMessage Short Message
-   * @param message long error message
-   */
-  public OzoneRestClientException(String shortMessage, String message) {
-    super(0, shortMessage, message);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
index 6728e68..9d3831c 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
 import org.apache.hadoop.ozone.client.rest.headers.Header;
 import org.apache.hadoop.ozone.web.request.OzoneQuota;
@@ -203,7 +204,7 @@ public class OzoneVolume {
       executeCreateBucket(httpPost, httpClient);
       return getBucket(bucketName);
     } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(httpPost);
     }
@@ -276,7 +277,7 @@ public class OzoneVolume {
       if (entity != null) {
         throw OzoneException.parse(EntityUtils.toString(entity));
       } else {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
     } finally {
       if (entity != null) {
@@ -307,7 +308,7 @@ public class OzoneVolume {
       }
       executePutBucket(putRequest, httpClient);
     } catch (URISyntaxException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(putRequest);
     }
@@ -336,7 +337,7 @@ public class OzoneVolume {
       }
       executePutBucket(putRequest, httpClient);
     } catch (URISyntaxException | IOException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(putRequest);
     }
@@ -361,7 +362,7 @@ public class OzoneVolume {
       return executeInfoBucket(getRequest, httpClient);
 
     } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(getRequest);
     }
@@ -388,7 +389,7 @@ public class OzoneVolume {
       int errorCode = response.getStatusLine().getStatusCode();
       entity = response.getEntity();
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
       if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
         OzoneBucket bucket =
@@ -432,7 +433,7 @@ public class OzoneVolume {
         throw OzoneException.parse(EntityUtils.toString(entity));
       }
 
-      throw new OzoneRestClientException("Unexpected null in http result");
+      throw new OzoneClientException("Unexpected null in http result");
     } finally {
       if (entity != null) {
         EntityUtils.consumeQuietly(entity);
@@ -467,7 +468,7 @@ public class OzoneVolume {
       return executeListBuckets(getRequest, httpClient);
 
     } catch (IOException | URISyntaxException e) {
-      throw new OzoneRestClientException(e.getMessage(), e);
+      throw new OzoneClientException(e.getMessage(), e);
     } finally {
       releaseConnection(getRequest);
     }
@@ -496,7 +497,7 @@ public class OzoneVolume {
       entity = response.getEntity();
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload");
+        throw new OzoneClientException("Unexpected null in http payload");
       }
       if (errorCode == HTTP_OK) {
         ListBuckets bucketList =
@@ -535,7 +536,7 @@ public class OzoneVolume {
       executeDeleteBucket(delRequest, httpClient);
 
     } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
-      throw new OzoneRestClientException(ex.getMessage(), ex);
+      throw new OzoneClientException(ex.getMessage(), ex);
     } finally {
       releaseConnection(delRequest);
     }
@@ -564,7 +565,7 @@ public class OzoneVolume {
       }
 
       if (entity == null) {
-        throw new OzoneRestClientException("Unexpected null in http payload.");
+        throw new OzoneClientException("Unexpected null in http payload.");
       }
 
       throw OzoneException.parse(EntityUtils.toString(entity));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 31046f5..8ffe67d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.ozone.ozShell;
 
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -26,15 +27,21 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.PrintStream;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -43,11 +50,16 @@ import org.apache.hadoop.ozone.OzoneAcl.OzoneACLType;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneKey;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.client.rest.RestClient;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.web.ozShell.Shell;
 import org.apache.hadoop.ozone.web.request.OzoneQuota;
 import org.apache.hadoop.ozone.web.response.BucketInfo;
@@ -63,12 +75,20 @@ import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This test class specified for testing Ozone shell command.
  */
+@RunWith(value = Parameterized.class)
 public class TestOzoneShell {
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestOzoneShell.class);
+
   /**
    * Set the timeout for every test.
    */
@@ -79,7 +99,7 @@ public class TestOzoneShell {
   private static File baseDir;
   private static OzoneConfiguration conf = null;
   private static MiniOzoneCluster cluster = null;
-  private static OzoneRestClient client = null;
+  private static ClientProtocol client = null;
   private static Shell shell = null;
 
   private final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -87,6 +107,16 @@ public class TestOzoneShell {
   private static final PrintStream OLD_OUT = System.out;
   private static final PrintStream OLD_ERR = System.err;
 
+  @Parameterized.Parameters
+  public static Collection<Object[]> clientProtocol() {
+    Object[][] params = new Object[][] {
+        {RpcClient.class},
+        {RestClient.class}};
+    return Arrays.asList(params);
+  }
+
+  @Parameterized.Parameter
+  public Class clientProtocol;
   /**
    * Create a MiniDFSCluster for testing with using distributed Ozone
    * handler type.
@@ -110,13 +140,12 @@ public class TestOzoneShell {
     shell = new Shell();
     shell.setConf(conf);
 
-    cluster = MiniOzoneCluster.newBuilder(conf).build();
+    cluster = MiniOzoneCluster.newBuilder(conf)
+        .setNumDatanodes(3)
+        .build();
+    conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
+    client = new RpcClient(conf);
     cluster.waitForClusterToBeReady();
-    final int port = cluster.getHddsDatanodes().get(0).getDatanodeDetails()
-        .getOzoneRestPort();
-    url = String.format("http://localhost:%d", port);
-    client = new OzoneRestClient(String.format("http://localhost:%d", port));
-    client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
   }
 
   /**
@@ -137,6 +166,26 @@ public class TestOzoneShell {
   public void setup() {
     System.setOut(new PrintStream(out));
     System.setErr(new PrintStream(err));
+    if(clientProtocol.equals(RestClient.class)) {
+      String hostName = cluster.getKeySpaceManager().getHttpServer()
+          .getHttpAddress().getHostName();
+      int port = cluster
+          .getKeySpaceManager().getHttpServer().getHttpAddress().getPort();
+      url = String.format("http://" + hostName + ":" + port);
+    } else {
+      List<ServiceInfo> services = null;
+      try {
+        services = cluster.getKeySpaceManager().getServiceList();
+      } catch (IOException e) {
+        LOG.error("Could not get service list from KSM");
+      }
+      String hostName = services.stream().filter(
+          a -> a.getNodeType().equals(HddsProtos.NodeType.KSM))
+          .collect(Collectors.toList()).get(0).getHostname();
+
+      String port = cluster.getKeySpaceManager().getRpcPort();
+      url = String.format("o3://" + hostName + ":" + port);
+    }
   }
 
   @After
@@ -152,22 +201,29 @@ public class TestOzoneShell {
 
   @Test
   public void testCreateVolume() throws Exception {
+    LOG.info("Running testCreateVolume");
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     String userName = "bilbo";
     String[] args = new String[] {"-createVolume", url + "/" + volumeName,
         "-user", userName, "-root"};
 
     assertEquals(0, ToolRunner.run(shell, args));
-    OzoneVolume volumeInfo = client.getVolume(volumeName);
-    assertEquals(volumeName, volumeInfo.getVolumeName());
-    assertEquals(userName, volumeInfo.getOwnerName());
+    OzoneVolume volumeInfo = client.getVolumeDetails(volumeName);
+    assertEquals(volumeName, volumeInfo.getName());
+    assertEquals(userName, volumeInfo.getOwner());
   }
 
   @Test
   public void testDeleteVolume() throws Exception {
+    LOG.info("Running testDeleteVolume");
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
-    assertNotNull(vol);
+    VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+        .setOwner("bilbo")
+        .setQuota("100TB")
+        .build();
+    client.createVolume(volumeName, volumeArgs);
+    OzoneVolume volume = client.getVolumeDetails(volumeName);
+    assertNotNull(volume);
 
     String[] args = new String[] {"-deleteVolume", url + "/" + volumeName,
         "-root"};
@@ -175,9 +231,9 @@ public class TestOzoneShell {
 
     // verify if volume has been deleted
     try {
-      client.getVolume(volumeName);
+      client.getVolumeDetails(volumeName);
       fail("Get volume call should have thrown.");
-    } catch (OzoneException e) {
+    } catch (IOException e) {
       GenericTestUtils.assertExceptionContains(
           "Info Volume failed, error:VOLUME_NOT_FOUND", e);
     }
@@ -185,8 +241,13 @@ public class TestOzoneShell {
 
   @Test
   public void testInfoVolume() throws Exception {
+    LOG.info("Running testInfoVolume");
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    client.createVolume(volumeName, "bilbo", "100TB");
+    VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+        .setOwner("bilbo")
+        .setQuota("100TB")
+        .build();
+    client.createVolume(volumeName, volumeArgs);
 
     String[] args = new String[] {"-infoVolume", url + "/" + volumeName,
         "-root"};
@@ -206,45 +267,53 @@ public class TestOzoneShell {
 
   @Test
   public void testUpdateVolume() throws Exception {
+    LOG.info("Running testUpdateVolume");
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     String userName = "bilbo";
-    OzoneVolume vol = client.createVolume(volumeName, userName, "100TB");
-    assertEquals(userName, vol.getOwnerName());
-    assertEquals(100, vol.getQuota().getSize(), 100);
-    assertEquals(OzoneQuota.Units.TB, vol.getQuota().getUnit());
+    VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+        .setOwner("bilbo")
+        .setQuota("100TB")
+        .build();
+    client.createVolume(volumeName, volumeArgs);
+    OzoneVolume vol = client.getVolumeDetails(volumeName);
+    assertEquals(userName, vol.getOwner());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), vol.getQuota());
 
     String[] args = new String[] {"-updateVolume", url + "/" + volumeName,
         "-quota", "500MB", "-root"};
     assertEquals(0, ToolRunner.run(shell, args));
-    vol = client.getVolume(volumeName);
-    assertEquals(userName, vol.getOwnerName());
-    assertEquals(500, vol.getQuota().getSize(), 500);
-    assertEquals(OzoneQuota.Units.MB, vol.getQuota().getUnit());
+    vol = client.getVolumeDetails(volumeName);
+    assertEquals(userName, vol.getOwner());
+    assertEquals(OzoneQuota.parseQuota("500MB").sizeInBytes(), vol.getQuota());
 
     String newUser = "new-user";
     args = new String[] {"-updateVolume", url + "/" + volumeName,
         "-user", newUser, "-root"};
     assertEquals(0, ToolRunner.run(shell, args));
-    vol = client.getVolume(volumeName);
-    assertEquals(newUser, vol.getOwnerName());
+    vol = client.getVolumeDetails(volumeName);
+    assertEquals(newUser, vol.getOwner());
 
     // test error conditions
     args = new String[] {"-updateVolume", url + "/invalid-volume",
         "-user", newUser, "-root"};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
-        "Volume owner change failed, error:VOLUME_NOT_FOUND"));
+        "Info Volume failed, error:VOLUME_NOT_FOUND"));
 
     err.reset();
     args = new String[] {"-updateVolume", url + "/invalid-volume",
         "-quota", "500MB", "-root"};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
-        "Volume quota change failed, error:VOLUME_NOT_FOUND"));
+        "Info Volume failed, error:VOLUME_NOT_FOUND"));
   }
 
   @Test
   public void testListVolume() throws Exception {
+    LOG.info("Running testListVolume");
+    if (clientProtocol.equals(RestClient.class)) {
+      return;
+    }
     String commandOutput;
     List<VolumeInfo> volumes;
     final int volCount = 20;
@@ -265,7 +334,12 @@ public class TestOzoneShell {
         userName = user2;
         volumeName = "test-vol" + x;
       }
-      OzoneVolume vol = client.createVolume(volumeName, userName, "100TB");
+      VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+          .setOwner(userName)
+          .setQuota("100TB")
+          .build();
+      client.createVolume(volumeName, volumeArgs);
+      OzoneVolume vol = client.getVolumeDetails(volumeName);
       assertNotNull(vol);
     }
 
@@ -343,16 +417,17 @@ public class TestOzoneShell {
 
   @Test
   public void testCreateBucket() throws Exception {
+    LOG.info("Running testCreateBucket");
     OzoneVolume vol = creatVolume();
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
     String[] args = new String[] {"-createBucket",
-        url + "/" + vol.getVolumeName() + "/" + bucketName};
+        url + "/" + vol.getName() + "/" + bucketName};
 
     assertEquals(0, ToolRunner.run(shell, args));
     OzoneBucket bucketInfo = vol.getBucket(bucketName);
-    assertEquals(vol.getVolumeName(),
-        bucketInfo.getBucketInfo().getVolumeName());
-    assertEquals(bucketName, bucketInfo.getBucketName());
+    assertEquals(vol.getName(),
+        bucketInfo.getVolumeName());
+    assertEquals(bucketName, bucketInfo.getName());
 
     // test create a bucket in a non-exist volume
     args = new String[] {"-createBucket",
@@ -365,20 +440,22 @@ public class TestOzoneShell {
 
   @Test
   public void testDeleteBucket() throws Exception {
+    LOG.info("Running testDeleteBucket");
     OzoneVolume vol = creatVolume();
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-    OzoneBucket bucketInfo = vol.createBucket(bucketName);
+    vol.createBucket(bucketName);
+    OzoneBucket bucketInfo = vol.getBucket(bucketName);
     assertNotNull(bucketInfo);
 
     String[] args = new String[] {"-deleteBucket",
-        url + "/" + vol.getVolumeName() + "/" + bucketName};
+        url + "/" + vol.getName() + "/" + bucketName};
     assertEquals(0, ToolRunner.run(shell, args));
 
     // verify if bucket has been deleted in volume
     try {
       vol.getBucket(bucketName);
       fail("Get bucket should have thrown.");
-    } catch (OzoneException e) {
+    } catch (IOException e) {
       GenericTestUtils.assertExceptionContains(
           "Info Bucket failed, error: BUCKET_NOT_FOUND", e);
     }
@@ -393,7 +470,7 @@ public class TestOzoneShell {
     err.reset();
     // test delete non-exist bucket
     args = new String[] {"-deleteBucket",
-        url + "/" + vol.getVolumeName() + "/invalid-bucket"};
+        url + "/" + vol.getName() + "/invalid-bucket"};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
         "Delete Bucket failed, error:BUCKET_NOT_FOUND"));
@@ -401,12 +478,13 @@ public class TestOzoneShell {
 
   @Test
   public void testInfoBucket() throws Exception {
+    LOG.info("Running testInfoBucket");
     OzoneVolume vol = creatVolume();
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
     vol.createBucket(bucketName);
 
     String[] args = new String[] {"-infoBucket",
-        url + "/" + vol.getVolumeName() + "/" + bucketName};
+        url + "/" + vol.getName() + "/" + bucketName};
     assertEquals(0, ToolRunner.run(shell, args));
 
     String output = out.toString();
@@ -416,7 +494,7 @@ public class TestOzoneShell {
 
     // test get info from a non-exist bucket
     args = new String[] {"-infoBucket",
-        url + "/" + vol.getVolumeName() + "/invalid-bucket" + bucketName};
+        url + "/" + vol.getName() + "/invalid-bucket" + bucketName};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
         "Info Bucket failed, error: BUCKET_NOT_FOUND"));
@@ -424,13 +502,15 @@ public class TestOzoneShell {
 
   @Test
   public void testUpdateBucket() throws Exception {
+    LOG.info("Running testUpdateBucket");
     OzoneVolume vol = creatVolume();
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-    OzoneBucket bucket = vol.createBucket(bucketName);
-    assertEquals(0, bucket.getAcls().size());
+    vol.createBucket(bucketName);
+    OzoneBucket bucket = vol.getBucket(bucketName);
+    int aclSize = bucket.getAcls().size();
 
     String[] args = new String[] {"-updateBucket",
-        url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl",
+        url + "/" + vol.getName() + "/" + bucketName, "-addAcl",
         "user:frodo:rw,group:samwise:r"};
     assertEquals(0, ToolRunner.run(shell, args));
     String output = out.toString();
@@ -438,36 +518,40 @@ public class TestOzoneShell {
         && output.contains(OzoneConsts.OZONE_TIME_ZONE));
 
     bucket = vol.getBucket(bucketName);
-    assertEquals(2, bucket.getAcls().size());
+    assertEquals(2 + aclSize, bucket.getAcls().size());
 
-    OzoneAcl acl = bucket.getAcls().get(0);
+    OzoneAcl acl = bucket.getAcls().get(aclSize);
     assertTrue(acl.getName().equals("frodo")
         && acl.getType() == OzoneACLType.USER
         && acl.getRights()== OzoneACLRights.READ_WRITE);
 
     args = new String[] {"-updateBucket",
-        url + "/" + vol.getVolumeName() + "/" + bucketName, "-removeAcl",
+        url + "/" + vol.getName() + "/" + bucketName, "-removeAcl",
         "user:frodo:rw"};
     assertEquals(0, ToolRunner.run(shell, args));
 
     bucket = vol.getBucket(bucketName);
-    acl = bucket.getAcls().get(0);
-    assertEquals(1, bucket.getAcls().size());
+    acl = bucket.getAcls().get(aclSize);
+    assertEquals(1 + aclSize, bucket.getAcls().size());
     assertTrue(acl.getName().equals("samwise")
         && acl.getType() == OzoneACLType.GROUP
         && acl.getRights()== OzoneACLRights.READ);
 
     // test update bucket for a non-exist bucket
     args = new String[] {"-updateBucket",
-        url + "/" + vol.getVolumeName() + "/invalid-bucket", "-addAcl",
+        url + "/" + vol.getName() + "/invalid-bucket", "-addAcl",
         "user:frodo:rw"};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
-        "Setting bucket property failed, error: BUCKET_NOT_FOUND"));
+        "Info Bucket failed, error: BUCKET_NOT_FOUND"));
   }
 
   @Test
   public void testListBucket() throws Exception {
+    LOG.info("Running testListBucket");
+    if (clientProtocol.equals(RestClient.class)) {
+      return;
+    }
     List<BucketInfo> buckets;
     String commandOutput;
     int bucketCount = 11;
@@ -478,13 +562,14 @@ public class TestOzoneShell {
     for (int i = 0; i < bucketCount; i++) {
       String name = "test-bucket" + i;
       bucketNames.add(name);
-      OzoneBucket bucket = vol.createBucket(name);
+      vol.createBucket(name);
+      OzoneBucket bucket = vol.getBucket(name);
       assertNotNull(bucket);
     }
 
     // test -length option
     String[] args = new String[] {"-listBucket",
-        url + "/" + vol.getVolumeName(), "-length", "100"};
+        url + "/" + vol.getName(), "-length", "100"};
     assertEquals(0, ToolRunner.run(shell, args));
     commandOutput = out.toString();
     buckets = (List<BucketInfo>) JsonUtils.toJsonList(commandOutput,
@@ -497,13 +582,13 @@ public class TestOzoneShell {
     // test-bucket10, test-bucket2, ,..., test-bucket9]
     for (int i = 0; i < buckets.size(); i++) {
       assertEquals(buckets.get(i).getBucketName(), bucketNames.get(i));
-      assertEquals(buckets.get(i).getVolumeName(), vol.getVolumeName());
+      assertEquals(buckets.get(i).getVolumeName(), vol.getName());
       assertTrue(buckets.get(i).getCreatedOn()
           .contains(OzoneConsts.OZONE_TIME_ZONE));
     }
 
     out.reset();
-    args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+    args = new String[] {"-listBucket", url + "/" + vol.getName(),
         "-length", "3"};
     assertEquals(0, ToolRunner.run(shell, args));
     commandOutput = out.toString();
@@ -519,7 +604,7 @@ public class TestOzoneShell {
 
     // test -prefix option
     out.reset();
-    args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+    args = new String[] {"-listBucket", url + "/" + vol.getName(),
         "-length", "100", "-prefix", "test-bucket1"};
     assertEquals(0, ToolRunner.run(shell, args));
     commandOutput = out.toString();
@@ -533,7 +618,7 @@ public class TestOzoneShell {
 
     // test -start option
     out.reset();
-    args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+    args = new String[] {"-listBucket", url + "/" + vol.getName(),
         "-length", "100", "-start", "test-bucket7"};
     assertEquals(0, ToolRunner.run(shell, args));
     commandOutput = out.toString();
@@ -546,7 +631,7 @@ public class TestOzoneShell {
 
     // test error conditions
     err.reset();
-    args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+    args = new String[] {"-listBucket", url + "/" + vol.getName(),
         "-length", "-1"};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
@@ -555,9 +640,10 @@ public class TestOzoneShell {
 
   @Test
   public void testPutKey() throws Exception {
+    LOG.info("Running testPutKey");
     OzoneBucket bucket = creatBucket();
-    String volumeName = bucket.getBucketInfo().getVolumeName();
-    String bucketName = bucket.getBucketName();
+    String volumeName = bucket.getVolumeName();
+    String bucketName = bucket.getName();
     String keyName = "key" + RandomStringUtils.randomNumeric(5);
 
     String[] args = new String[] {"-putKey",
@@ -565,8 +651,8 @@ public class TestOzoneShell {
         createTmpFile()};
     assertEquals(0, ToolRunner.run(shell, args));
 
-    OzoneKey keyInfo = bucket.getKeyInfo(keyName);
-    assertEquals(keyName, keyInfo.getObjectInfo().getKeyName());
+    OzoneKey keyInfo = bucket.getKey(keyName);
+    assertEquals(keyName, keyInfo.getName());
 
     // test put key in a non-exist bucket
     args = new String[] {"-putKey",
@@ -574,18 +660,22 @@ public class TestOzoneShell {
         createTmpFile()};
     assertEquals(1, ToolRunner.run(shell, args));
     assertTrue(err.toString().contains(
-        "Create key failed, error:BUCKET_NOT_FOUND"));
+        "Info Bucket failed, error: BUCKET_NOT_FOUND"));
   }
 
   @Test
   public void testGetKey() throws Exception {
+    LOG.info("Running testGetKey");
     String keyName = "key" + RandomStringUtils.randomNumeric(5);
     OzoneBucket bucket = creatBucket();
-    String volumeName = bucket.getBucketInfo().getVolumeName();
-    String bucketName = bucket.getBucketName();
+    String volumeName = bucket.getVolumeName();
+    String bucketName = bucket.getName();
 
     String dataStr = "test-data";
-    bucket.putKey(keyName, dataStr);
+    OzoneOutputStream keyOutputStream =
+        bucket.createKey(keyName, dataStr.length());
+    keyOutputStream.write(dataStr.getBytes());
+    keyOutputStream.close();
 
     String tmpPath = baseDir.getAbsolutePath() + "/testfile-"
         + UUID.randomUUID().toString();
@@ -603,14 +693,19 @@ public class TestOzoneShell {
 
   @Test
   public void testDeleteKey() throws Exception {
+    LOG.info("Running testDeleteKey");
     String keyName = "key" + RandomStringUtils.randomNumeric(5);
     OzoneBucket bucket = creatBucket();
-    String volumeName = bucket.getBucketInfo().getVolumeName();
-    String bucketName = bucket.getBucketName();
-    bucket.putKey(keyName, "test-data");
+    String volumeName = bucket.getVolumeName();
+    String bucketName = bucket.getName();
+    String dataStr = "test-data";
+    OzoneOutputStream keyOutputStream =
+        bucket.createKey(keyName, dataStr.length());
+    keyOutputStream.write(dataStr.getBytes());
+    keyOutputStream.close();
 
-    OzoneKey keyInfo = bucket.getKeyInfo(keyName);
-    assertEquals(keyName, keyInfo.getObjectInfo().getKeyName());
+    OzoneKey keyInfo = bucket.getKey(keyName);
+    assertEquals(keyName, keyInfo.getName());
 
     String[] args = new String[] {"-deleteKey",
         url + "/" + volumeName + "/" + bucketName + "/" + keyName};
@@ -618,9 +713,9 @@ public class TestOzoneShell {
 
     // verify if key has been deleted in the bucket
     try {
-      bucket.getKeyInfo(keyName);
+      bucket.getKey(keyName);
       fail("Get key should have thrown.");
-    } catch (OzoneException e) {
+    } catch (IOException e) {
       GenericTestUtils.assertExceptionContains(
           "Lookup key failed, error:KEY_NOT_FOUND", e);
     }
@@ -643,22 +738,29 @@ public class TestOzoneShell {
 
   @Test
   public void testInfoKey() throws Exception {
+    LOG.info("Running testInfoKey");
     String keyName = "key" + RandomStringUtils.randomNumeric(5);
     OzoneBucket bucket = creatBucket();
-    String volumeName = bucket.getBucketInfo().getVolumeName();
-    String bucketName = bucket.getBucketName();
-    bucket.putKey(keyName, "test-data");
+    String volumeName = bucket.getVolumeName();
+    String bucketName = bucket.getName();
+    String dataStr = "test-data";
+    OzoneOutputStream keyOutputStream =
+        bucket.createKey(keyName, dataStr.length());
+    keyOutputStream.write(dataStr.getBytes());
+    keyOutputStream.close();
 
     String[] args = new String[] {"-infoKey",
         url + "/" + volumeName + "/" + bucketName + "/" + keyName};
 
     // verify the response output
-    assertEquals(0, ToolRunner.run(shell, args));
-
+    int a = ToolRunner.run(shell, args);
     String output = out.toString();
+    assertEquals(0, a);
+
     assertTrue(output.contains(keyName));
-    assertTrue(output.contains("createdOn") && output.contains("modifiedOn")
-        && output.contains(OzoneConsts.OZONE_TIME_ZONE));
+    assertTrue(
+        output.contains("createdOn") && output.contains("modifiedOn") && output
+            .contains(OzoneConsts.OZONE_TIME_ZONE));
 
     // reset stream
     out.reset();
@@ -677,19 +779,27 @@ public class TestOzoneShell {
 
   @Test
   public void testListKey() throws Exception {
+    LOG.info("Running testListKey");
+    if (clientProtocol.equals(RestClient.class)) {
+      return;
+    }
     String commandOutput;
     List<KeyInfo> keys;
     int keyCount = 11;
     OzoneBucket bucket = creatBucket();
-    String volumeName = bucket.getBucketInfo().getVolumeName();
-    String bucketName = bucket.getBucketName();
+    String volumeName = bucket.getVolumeName();
+    String bucketName = bucket.getName();
 
     String keyName;
     List<String> keyNames = new ArrayList<>();
     for (int i = 0; i < keyCount; i++) {
       keyName = "test-key" + i;
       keyNames.add(keyName);
-      bucket.putKey(keyName, "test-data" + i);
+      String dataStr = "test-data";
+      OzoneOutputStream keyOutputStream =
+          bucket.createKey(keyName, dataStr.length());
+      keyOutputStream.write(dataStr.getBytes());
+      keyOutputStream.close();
     }
 
     // test -length option
@@ -763,17 +873,23 @@ public class TestOzoneShell {
         "the vaule should be a positive number"));
   }
 
-  private OzoneVolume creatVolume() throws OzoneException {
-    String volumeName = UUID.randomUUID().toString() + "volume";
-    OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
+  private OzoneVolume creatVolume() throws OzoneException, IOException {
+    String volumeName = RandomStringUtils.randomNumeric(5) + "volume";
+    VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+        .setOwner("bilbo")
+        .setQuota("100TB")
+        .build();
+    client.createVolume(volumeName, volumeArgs);
+    OzoneVolume volume = client.getVolumeDetails(volumeName);
 
-    return vol;
+    return volume;
   }
 
-  private OzoneBucket creatBucket() throws OzoneException {
+  private OzoneBucket creatBucket() throws OzoneException, IOException {
     OzoneVolume vol = creatVolume();
-    String bucketName = UUID.randomUUID().toString() + "bucket";
-    OzoneBucket bucketInfo = vol.createBucket(bucketName);
+    String bucketName = RandomStringUtils.randomNumeric(5) + "bucket";
+    vol.createBucket(bucketName);
+    OzoneBucket bucketInfo = vol.getBucket(bucketName);
 
     return bucketInfo;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
index 46539e7..64e5f71 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
 import org.apache.hadoop.ozone.web.request.OzoneQuota;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@@ -132,7 +133,7 @@ public class TestBuckets {
       fail("Except the bucket creation to be failed because the"
           + " bucket name starts with an invalid char #");
     } catch (Exception e) {
-      assertTrue(e instanceof OzoneRestClientException);
+      assertTrue(e instanceof OzoneClientException);
       assertTrue(e.getMessage().contains("Bucket or Volume name"
           + " has an unsupported character : #"));
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
index 4cd90c9..2d3cea9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
@@ -141,7 +142,7 @@ public class TestVolume {
       fail("Except the volume creation be failed because the"
           + " volume name starts with an invalid char #");
     } catch (Exception e) {
-      assertTrue(e instanceof OzoneRestClientException);
+      assertTrue(e instanceof OzoneClientException);
       assertTrue(e.getMessage().contains("Bucket or Volume name"
           + " has an unsupported character : #"));
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
index 99020c8..7fe6bb8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
@@ -19,28 +19,30 @@
 package org.apache.hadoop.ozone.web.ozShell;
 
 import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
 import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+
 /**
  * Common interface for command handling.
  */
 public abstract class Handler {
 
-  protected OzoneRestClient client;
-
-  /**
-   * Constructs a client object.
-   */
-  public Handler() {
-    client = new OzoneRestClient();
-  }
+  protected static final Logger LOG = LoggerFactory.getLogger(Handler.class);
+  protected OzoneClient client;
 
   /**
    * Executes the Client command.
@@ -61,19 +63,44 @@ public abstract class Handler {
    * @throws URISyntaxException
    * @throws OzoneException
    */
-  protected URI verifyURI(String uri) throws URISyntaxException,
-      OzoneException {
+  protected URI verifyURI(String uri)
+      throws URISyntaxException, OzoneException, IOException {
     if ((uri == null) || uri.isEmpty()) {
-      throw new OzoneRestClientException(
+      throw new OzoneClientException(
           "Ozone URI is needed to execute this command.");
     }
     URIBuilder ozoneURI = new URIBuilder(uri);
-
     if (ozoneURI.getPort() == 0) {
       ozoneURI.setPort(Shell.DEFAULT_OZONE_PORT);
     }
+
+    Configuration conf = new OzoneConfiguration();
+    String scheme = ozoneURI.getScheme();
+    if (scheme.equals(OZONE_HTTP_SCHEME)) {
+      if (ozoneURI.getHost() != null) {
+        if (ozoneURI.getPort() == -1) {
+          client = OzoneClientFactory.getRestClient(ozoneURI.getHost());
+        } else {
+          client = OzoneClientFactory
+              .getRestClient(ozoneURI.getHost(), ozoneURI.getPort(), conf);
+        }
+      } else {
+        client = OzoneClientFactory.getRestClient(conf);
+      }
+    } else if (scheme.equals(OZONE_URI_SCHEME) || scheme.isEmpty()) {
+      if (ozoneURI.getHost() != null) {
+        if (ozoneURI.getPort() == -1) {
+          client = OzoneClientFactory.getRpcClient(ozoneURI.getHost());
+        } else {
+          client = OzoneClientFactory
+              .getRpcClient(ozoneURI.getHost(), ozoneURI.getPort(), conf);
+        }
+      } else {
+        client = OzoneClientFactory.getRpcClient(conf);
+      }
+    } else {
+      throw new OzoneClientException("Invalid URI: " + ozoneURI);
+    }
     return ozoneURI.build();
   }
-
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e0f278/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
index d1c46b5..0788f9e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
@@ -18,9 +18,10 @@
 package org.apache.hadoop.ozone.web.ozShell.bucket;
 
 import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientUtils;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.rest.OzoneException;
 import org.apache.hadoop.ozone.web.ozShell.Handler;
 import org.apache.hadoop.ozone.web.ozShell.Shell;
@@ -39,7 +40,6 @@ public class CreateBucketHandler extends Handler {
 
   private String volumeName;
   private String bucketName;
-  private String rootName;
 
   /**
    * Executes create bucket.
@@ -54,7 +54,7 @@ public class CreateBucketHandler extends Handler {
   protected void execute(CommandLine cmd)
       throws IOException, OzoneException, URISyntaxException {
     if (!cmd.hasOption(Shell.CREATE_BUCKET)) {
-      throw new OzoneRestClientException(
+      throw new OzoneClientException(
           "Incorrect call : createBucket is missing");
     }
 
@@ -62,7 +62,7 @@ public class CreateBucketHandler extends Handler {
     URI ozoneURI = verifyURI(ozoneURIString);
     Path path = Paths.get(ozoneURI.getPath());
     if (path.getNameCount() < 2) {
-      throw new OzoneRestClientException(
+      throw new OzoneClientException(
           "volume and bucket name required in createBucket");
     }
 
@@ -74,23 +74,13 @@ public class CreateBucketHandler extends Handler {
       System.out.printf("Bucket Name : %s%n", bucketName);
     }
 
-    if (cmd.hasOption(Shell.RUNAS)) {
-      rootName = "hdfs";
-    } else {
-      rootName = System.getProperty("user.name");
-    }
-
-
-    client.setEndPointURI(ozoneURI);
-    client.setUserAuth(rootName);
-
-
-    OzoneVolume vol = client.getVolume(volumeName);
-    OzoneBucket bucket = vol.createBucket(bucketName);
+    OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+    vol.createBucket(bucketName);
 
     if (cmd.hasOption(Shell.VERBOSE)) {
-      System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
-          bucket.getBucketInfo().toJsonString()));
+      OzoneBucket bucket = vol.getBucket(bucketName);
+      System.out.printf(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
+          JsonUtils.toJsonString(OzoneClientUtils.asBucketInfo(bucket))));
     }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message