hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bha...@apache.org
Subject hadoop git commit: HDDS-683. Add a shell command to provide ozone mapping for a S3Bucket. Contributed by Bharat Viswanadham.
Date Wed, 17 Oct 2018 23:47:28 GMT
Repository: hadoop
Updated Branches:
  refs/heads/ozone-0.3 106be1416 -> 3c11dec71


HDDS-683. Add a shell command to provide ozone mapping for a S3Bucket. Contributed by Bharat
Viswanadham.

(cherry picked from commit 4c4364ddd06927fa990d3e66d4eadbd4a7eb9474)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c11dec7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c11dec7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c11dec7

Branch: refs/heads/ozone-0.3
Commit: 3c11dec714b193525ba5e6190407f2156110d71a
Parents: 106be14
Author: Bharat Viswanadham <bharat@apache.org>
Authored: Wed Oct 17 16:46:06 2018 -0700
Committer: Bharat Viswanadham <bharat@apache.org>
Committed: Wed Oct 17 16:47:11 2018 -0700

----------------------------------------------------------------------
 .../hadoop/ozone/ozShell/TestOzoneShell.java    | 42 +++++++++
 .../apache/hadoop/ozone/web/ozShell/Shell.java  |  5 +-
 .../web/ozShell/bucket/BucketCommands.java      |  3 +-
 .../web/ozShell/bucket/S3BucketMapping.java     | 99 ++++++++++++++++++++
 4 files changed, 147 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c11dec7/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 1576d41..c736f81 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -1076,6 +1076,48 @@ public class TestOzoneShell {
     executeWithError(shell, args, "the length should be a positive number");
   }
 
+  @Test
+  public void testS3BucketMapping() throws  IOException {
+    String s3Bucket = "bucket1";
+    String commandOutput;
+    createS3Bucket("ozone", s3Bucket);
+    String volumeName = client.getOzoneVolumeName(s3Bucket);
+    String[] args = new String[] {"bucket", "path", url + "/" + s3Bucket};
+    if (url.startsWith("o3")) {
+      execute(shell, args);
+      commandOutput = out.toString();
+      assertTrue(commandOutput.contains("Volume name for S3Bucket is : " +
+          volumeName));
+      assertTrue(commandOutput.contains(OzoneConsts.OZONE_URI_SCHEME +"://" +
+          s3Bucket + "." + volumeName));
+      out.reset();
+      //Trying to get map for an unknown bucket
+      args = new String[] {"bucket", "path", url + "/" + "unknownbucket"};
+      executeWithError(shell, args, "S3_BUCKET_NOT_FOUND");
+    } else {
+      executeWithError(shell, args, "Ozone REST protocol does not support " +
+          "this operation");
+    }
+
+    // No bucket name
+    args = new String[] {"bucket", "path", url};
+    executeWithError(shell, args, "S3Bucket name is required");
+
+    // Invalid bucket name
+    args = new String[] {"bucket", "path", url + "/" + s3Bucket +
+          "/multipleslash"};
+    executeWithError(shell, args, "Invalid S3Bucket name. Delimiters (/) not" +
+        " allowed");
+  }
+
+  private void createS3Bucket(String userName, String s3Bucket) {
+    try {
+      client.createS3Bucket("ozone", s3Bucket);
+    } catch (IOException ex) {
+      GenericTestUtils.assertExceptionContains("S3_BUCKET_ALREADY_EXISTS", ex);
+    }
+  }
+
   private OzoneVolume creatVolume() throws OzoneException, IOException {
     String volumeName = RandomStringUtils.randomNumeric(5) + "volume";
     VolumeArgs volumeArgs = VolumeArgs.newBuilder()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c11dec7/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
index 0f3969f..28ab900 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
@@ -51,7 +51,7 @@ public class Shell extends GenericCli {
   public static final String OZONE_URI_DESCRIPTION = "Ozone URI could start "
       + "with o3:// or http(s):// or without prefix. REST protocol will "
       + "be used for http(s), RPC otherwise. URI may contain the host and port "
-      + "of the SCM server. Both are optional. "
+      + "of the OM server. Both are optional. "
       + "If they are not specified it will be identified from "
       + "the config files.";
 
@@ -64,6 +64,9 @@ public class Shell extends GenericCli {
   public static final String OZONE_KEY_URI_DESCRIPTION =
       "URI of the volume/bucket/key.\n" + OZONE_URI_DESCRIPTION;
 
+  public static final String OZONE_S3BUCKET_URI_DESCRIPTION = "URI of the " +
+      "S3Bucket.\n" + OZONE_URI_DESCRIPTION;
+
   // General options
   public static final int DEFAULT_OZONE_PORT = 50070;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c11dec7/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
index 870f4d7..617643c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
@@ -38,7 +38,8 @@ import picocli.CommandLine.ParentCommand;
         ListBucketHandler.class,
         CreateBucketHandler.class,
         UpdateBucketHandler.class,
-        DeleteBucketHandler.class
+        DeleteBucketHandler.class,
+        S3BucketMapping.class
     },
     mixinStandardHelpOptions = true,
     versionProvider = HddsVersionProvider.class)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c11dec7/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
new file mode 100644
index 0000000..21ae746
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.web.ozShell.bucket;
+
+import java.net.URI;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.web.ozShell.Handler;
+import org.apache.hadoop.ozone.web.ozShell.Shell;
+
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Parameters;
+
+/**
+ * S3Bucket mapping handler, which returns volume name and Ozone fs uri for
+ * that bucket.
+ */
+@Command(name = "path",
+    description = "Returns the ozone path for S3Bucket")
+public class S3BucketMapping extends Handler {
+
+  @Parameters(arity = "1..1", description = Shell
+      .OZONE_S3BUCKET_URI_DESCRIPTION)
+  private String uri;
+
+  /**
+   * Executes create bucket.
+   */
+  @Override
+  public Void call() throws Exception {
+
+    URI ozoneURI = verifyURI(uri);
+    Path path = Paths.get(ozoneURI.getPath());
+    int pathNameCount = path.getNameCount();
+    String errorMessage;
+
+    // When just uri is given as http://om:9874, we are getting pathCount
+    // still as 1, as getPath() is returning empty string.
+    // So for safer side check, whether it is an empty string
+    if (pathNameCount == 1) {
+      String s3Bucket = path.getName(0).toString();
+      if (StringUtils.isBlank(s3Bucket)) {
+        errorMessage = "S3Bucket name is required to get volume name and " +
+            "Ozone fs Uri";
+        throw new OzoneClientException(errorMessage);
+      }
+    }
+    if (pathNameCount != 1) {
+      if (pathNameCount < 1) {
+        errorMessage = "S3Bucket name is required to get volume name and " +
+            "Ozone fs Uri";
+      } else {
+        errorMessage = "Invalid S3Bucket name. Delimiters (/) not allowed in " +
+            "S3Bucket name";
+      }
+      throw new OzoneClientException(errorMessage);
+    }
+
+    String s3Bucket = path.getName(0).toString();
+    if (isVerbose()) {
+      System.out.printf("S3Bucket Name : %s%n", s3Bucket);
+    }
+
+    String mapping = client.getObjectStore().getOzoneBucketMapping(s3Bucket);
+    String volumeName = client.getObjectStore().getOzoneVolumeName(s3Bucket);
+
+    if (isVerbose()) {
+      System.out.printf("Mapping created for S3Bucket is : %s%n", mapping);
+    }
+
+    System.out.printf("Volume name for S3Bucket is : %s%n", volumeName);
+
+    String ozoneFsUri = String.format("%s://%s.%s", OzoneConsts
+        .OZONE_URI_SCHEME, s3Bucket, volumeName);
+
+    System.out.printf("Ozone FileSystem Uri is : %s%n", ozoneFsUri);
+
+    return null;
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message