hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDFS-11955. Ozone: Set proper parameter default values for listBuckets http request. Contributed by Weiwei Yang.
Date Sat, 01 Jul 2017 07:15:05 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 2708e6260 -> f4af0c91c


HDFS-11955. Ozone: Set proper parameter default values for listBuckets http request.  Contributed
by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4af0c91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4af0c91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4af0c91

Branch: refs/heads/HDFS-7240
Commit: f4af0c91c514316f8114f279479328bfdb19d4b4
Parents: 2708e62
Author: Anu Engineer <aengineer@apache.org>
Authored: Sat Jul 1 00:08:30 2017 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Sat Jul 1 00:08:30 2017 -0700

----------------------------------------------------------------------
 ...ceManagerProtocolServerSideTranslatorPB.java |  8 ++++++-
 .../hadoop/ozone/web/client/OzoneVolume.java    | 13 +++++++++-
 .../hadoop/ozone/web/interfaces/Volume.java     |  1 +
 .../apache/hadoop/ozone/web/ozShell/Shell.java  | 25 +++++++++++++++++++-
 .../web/ozShell/bucket/ListBucketHandler.java   | 16 ++++++++++++-
 .../hadoop/ozone/web/client/TestBuckets.java    | 16 ++++++++++---
 6 files changed, 72 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
index d25630c..0b131ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
@@ -81,7 +81,8 @@ import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.List
 import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysResponse;
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.Status;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
@@ -93,6 +94,8 @@ import java.util.List;
  */
 public class KeySpaceManagerProtocolServerSideTranslatorPB implements
     KeySpaceManagerProtocolPB {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class);
   private final KeySpaceManagerProtocol impl;
 
   /**
@@ -134,6 +137,9 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
         return Status.INTERNAL_ERROR;
       }
     } else {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Unknown error occurs", ex);
+      }
       return Status.INTERNAL_ERROR;
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
index e1dfec1..91faa04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.web.client;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.ozone.OzoneClientUtils;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@@ -428,11 +429,21 @@ public class OzoneVolume {
    *
    * @throws OzoneException
    */
-  public List<OzoneBucket> listBuckets() throws OzoneException {
+  public List<OzoneBucket> listBuckets(String resultLength,
+      String startBucket, String prefix) throws OzoneException {
     HttpGet getRequest = null;
     try (CloseableHttpClient httpClient = newHttpClient()) {
       URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
       builder.setPath("/" + getVolumeName()).build();
+      if (!Strings.isNullOrEmpty(resultLength)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+      }
+      if (!Strings.isNullOrEmpty(startBucket)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startBucket);
+      }
+      if (!Strings.isNullOrEmpty(prefix)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+      }
 
       getRequest = client.getHttpGet(builder.toString());
       return executeListBuckets(getRequest, httpClient);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
index 8511373..e74ded7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
@@ -136,6 +136,7 @@ public interface Volume {
                          String info,
                          @QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
                          String prefix,
+                         @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
                          @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
                          int keys,
                          @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
index c0d3651..294d8f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
@@ -90,6 +90,11 @@ public class Shell extends Configured implements Tool {
   public static final String LIST_KEY = "listKey";
   public static final String FILE = "file";
 
+  // Listing related command line arguments
+  public static final String LIST_LENGTH = "length";
+  public static final String START = "start";
+  public static final String PREFIX = "prefix";
+
   /**
    * Main for the ozShell Command handling.
    *
@@ -136,6 +141,7 @@ public class Shell extends Configured implements Tool {
     addVolumeCommands(opts);
     addBucketCommands(opts);
     addKeyCommands(opts);
+    addListingCommands(opts);
     return opts;
   }
 
@@ -249,7 +255,6 @@ public class Shell extends Configured implements Tool {
         new Option(REMOVE_ACLS, true, "allows user to remove acls from a " +
             "bucket.");
     opts.addOption(removeAcl);
-
   }
 
   /**
@@ -285,6 +290,24 @@ public class Shell extends Configured implements Tool {
   }
 
   /**
+   * Sub commands for list command.
+   * @param opts
+   */
+  private void addListingCommands(Options opts) {
+    Option maxKeys = new Option(LIST_LENGTH, true,
+        "Specify the max length of listing result.");
+    opts.addOption(maxKeys);
+
+    Option prevKey = new Option(START, true,
+        "Specify the start key where to start listing from.");
+    opts.addOption(prevKey);
+
+    Option prefix = new Option(PREFIX, true,
+        "Specify the prefix to filter the listing result.");
+    opts.addOption(prefix);
+  }
+
+  /**
    * Dispatches calls to the right command Handler classes.
    *
    * @param cmd - CommandLine

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
index 51c8016..ce4a41e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
@@ -82,9 +82,23 @@ public class ListBucketHandler extends Handler {
     client.setEndPointURI(ozoneURI);
     client.setUserAuth(rootName);
 
+    String length = null;
+    if (cmd.hasOption(Shell.LIST_LENGTH)) {
+      length = cmd.getOptionValue(Shell.LIST_LENGTH);
+    }
+
+    String startBucket = null;
+    if (cmd.hasOption(Shell.START)) {
+      startBucket = cmd.getOptionValue(Shell.START);
+    }
+
+    String prefix = null;
+    if (cmd.hasOption(Shell.PREFIX)) {
+      prefix = cmd.getOptionValue(Shell.PREFIX);
+    }
 
     OzoneVolume vol = client.getVolume(volumeName);
-    List<OzoneBucket> bucketList = vol.listBuckets();
+    List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
 
     for (OzoneBucket bucket : bucketList) {
       System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
index 9ea84eb..635e463 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
@@ -72,7 +72,7 @@ public class TestBuckets {
 
     conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
     cluster = new MiniOzoneCluster.Builder(conf)
-        .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
+        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
     DataNode dataNode = cluster.getDataNodes().get(0);
     final int port = dataNode.getInfoPort();
     client = new OzoneRestClient(String.format("http://localhost:%d", port));
@@ -168,11 +168,21 @@ public class TestBuckets {
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
     String[] acls = {"user:frodo:rw", "user:samwise:rw"};
     for (int x = 0; x < 10; x++) {
-      String bucketName = OzoneUtils.getRequestID().toLowerCase();
+      String bucketName = "listbucket-test-" + x;
       vol.createBucket(bucketName, acls);
     }
-    List<OzoneBucket> bucketList = vol.listBuckets();
+    List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
     assertEquals(bucketList.size(), 10);
+
+    bucketList = vol.listBuckets("3", null, null);
+    assertEquals(bucketList.size(), 3);
+
+    bucketList = vol.listBuckets("100", "listbucket-test-5", null);
+    assertEquals(bucketList.size(), 5);
+
+    bucketList = vol.listBuckets("100", null, "listbucket-test-3");
+    assertEquals(bucketList.size(), 1);
+
     client.close();
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message