hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDFS-12145. Ozone: OzoneFileSystem: Ozone & KSM should support / delimited key names. Contributed by Mukul Kumar Singh.
Date Thu, 27 Jul 2017 05:37:03 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 f6ed18633 -> c651868c4


HDFS-12145. Ozone: OzoneFileSystem: Ozone & KSM should support / delimited key names.
Contributed by Mukul Kumar Singh.

(cherry picked from commit fe0a957e92d506330ba005abba18f665eb5ac1cc)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c651868c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c651868c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c651868c

Branch: refs/heads/HDFS-7240
Commit: c651868c4a317404c1ce5d8394b6007517886415
Parents: f6ed186
Author: Weiwei Yang <wwei@apache.org>
Authored: Thu Jul 27 12:40:40 2017 +0800
Committer: Anu Engineer <aengineer@apache.org>
Committed: Wed Jul 26 22:36:23 2017 -0700

----------------------------------------------------------------------
 .../hadoop/scm/storage/ChunkOutputStream.java   |  4 +-
 .../hadoop/ozone/web/interfaces/Keys.java       |  2 +-
 .../hadoop/ozone/web/client/TestKeys.java       | 56 ++++++++++++++++++--
 .../hadoop/ozone/web/client/TestKeysRatis.java  | 20 +++++++
 4 files changed, 76 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c651868c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
index d93d2cf..8daded5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
@@ -28,6 +28,7 @@ import java.util.UUID;
 
 import com.google.protobuf.ByteString;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
 import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
 import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
@@ -209,7 +210,8 @@ public class ChunkOutputStream extends OutputStream {
     ChunkInfo chunk = ChunkInfo
         .newBuilder()
         .setChunkName(
-            key + "_stream_" + streamId + "_chunk_" + ++chunkIndex)
+            DigestUtils.md5Hex(key) + "_stream_"
+                + streamId + "_chunk_" + ++chunkIndex)
         .setOffset(0)
         .setLen(data.size())
         .build();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c651868c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
index b6885a7..e64c8ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
@@ -40,7 +40,7 @@ import java.io.InputStream;
  * This interface defines operations permitted on a key.
  */
 
-@Path("/{volume}/{bucket}/{keys}")
+@Path("/{volume}/{bucket}/{keys:.*}")
 public interface Keys {
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c651868c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 516c571..d9168ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.ozone.web.client;
 
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.math.RandomUtils;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -28,6 +30,7 @@ import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Stat
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.junit.AfterClass;
@@ -122,17 +125,41 @@ public class TestKeys {
     return tmpFile;
   }
 
+  /**
+   * This function generates multi part key which are delimited by a certain
+   * delimiter. Different parts of key are random string of random length
+   * between 0 - 4. Number of parts of the keys are between 0 and 5.
+   *
+   * @param delimiter delimiter used to delimit parts of string
+   * @return Key composed of multiple parts delimited by "/"
+   */
+  static String getMultiPartKey(String delimiter) {
+    int numParts = RandomUtils.nextInt(5)  + 1;
+    String[] nameParts = new String[numParts];
+    for (int i = 0; i < numParts; i++) {
+      int stringLength = numParts == 1 ? 5 : RandomUtils.nextInt(5);
+      nameParts[i] = RandomStringUtils.randomAlphanumeric(stringLength);
+    }
+    return StringUtils.join(delimiter, nameParts);
+  }
+
   static class PutHelper {
     private final OzoneRestClient client;
     private final String dir;
+    private final String keyName;
 
     OzoneVolume vol;
     OzoneBucket bucket;
     File file;
 
     PutHelper(OzoneRestClient client, String dir) {
+      this(client, dir, OzoneUtils.getRequestID().toLowerCase());
+    }
+
+    PutHelper(OzoneRestClient client, String dir, String key) {
       this.client = client;
       this.dir = dir;
+      this.keyName = key;
     }
 
     public OzoneVolume getVol() {
@@ -163,8 +190,8 @@ public class TestKeys {
       String bucketName = OzoneUtils.getRequestID().toLowerCase();
       bucket = vol.createBucket(bucketName, acls, StorageType.DEFAULT);
 
-      String keyName = OzoneUtils.getRequestID().toLowerCase();
-      file = createRandomDataFile(dir, keyName, 1024);
+      String fileName = OzoneUtils.getRequestID().toLowerCase();
+      file = createRandomDataFile(dir, fileName, 1024);
 
       bucket.putKey(keyName, file);
       return keyName;
@@ -174,7 +201,12 @@ public class TestKeys {
 
   @Test
   public void testPutKey() throws OzoneException {
+    // Test non-delimited keys
     runTestPutKey(new PutHelper(ozoneRestClient, path));
+    // Test key delimited by a random delimiter
+    String delimiter = RandomStringUtils.randomAscii(1);
+    runTestPutKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   static void runTestPutKey(PutHelper helper) throws OzoneException {
@@ -229,6 +261,10 @@ public class TestKeys {
       throws OzoneException, IOException, URISyntaxException {
     runTestPutAndGetKeyWithDnRestart(
         new PutHelper(ozoneRestClient, path), ozoneCluster);
+    String delimiter = RandomStringUtils.randomAscii(1);
+    runTestPutAndGetKeyWithDnRestart(
+        new PutHelper(ozoneRestClient, path,
+            getMultiPartKey(delimiter)), ozoneCluster);
   }
 
   static void runTestPutAndGetKeyWithDnRestart(
@@ -263,6 +299,9 @@ public class TestKeys {
   @Test
   public void testPutAndGetKey() throws OzoneException, IOException {
     runTestPutAndGetKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAscii(1);
+    runTestPutAndGetKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   static void runTestPutAndGetKey(PutHelper helper)
@@ -326,6 +365,9 @@ public class TestKeys {
   @Test
   public void testPutAndDeleteKey() throws OzoneException, IOException {
     runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAscii(1);
+    runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   static void runTestPutAndDeleteKey(PutHelper helper)
@@ -347,6 +389,9 @@ public class TestKeys {
   @Test
   public void testPutAndListKey() throws OzoneException, IOException {
     runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAscii(1);
+    runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   static void runTestPutAndListKey(PutHelper helper)
@@ -378,9 +423,9 @@ public class TestKeys {
     Assert.assertEquals(keyList2.size(), 1);
 
     // test startKey parameter of list keys
-    keyList1 = helper.getBucket().listKeys("100", "list-key4", null);
+    keyList1 = helper.getBucket().listKeys("100", "list-key4", "list-key");
     keyList2 = client.listKeys(helper.getVol().getVolumeName(),
-        helper.getBucket().getBucketName(), "100", "list-key4", null);
+        helper.getBucket().getBucketName(), "100", "list-key4", "list-key");
     Assert.assertEquals(keyList1.size(), 5);
     Assert.assertEquals(keyList2.size(), 5);
 
@@ -416,6 +461,9 @@ public class TestKeys {
   @Test
   public void testGetKeyInfo() throws OzoneException, IOException {
     runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAscii(1);
+    runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   static void runTestGetKeyInfo(PutHelper helper) throws OzoneException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c651868c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
index b0fea90..1d33644 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.ozone.web.client;
 
+import org.apache.commons.lang.RandomStringUtils;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.RatisTestHelper;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@@ -57,6 +58,9 @@ public class TestKeysRatis {
   @Test
   public void testPutKey() throws OzoneException {
     runTestPutKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAlphanumeric(1);
+    runTestPutKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   @Test
@@ -64,25 +68,41 @@ public class TestKeysRatis {
       throws OzoneException, IOException, URISyntaxException {
     runTestPutAndGetKeyWithDnRestart(
         new PutHelper(ozoneRestClient, path), suite.getCluster());
+    String delimiter = RandomStringUtils.randomAlphanumeric(1);
+    runTestPutAndGetKeyWithDnRestart(
+        new PutHelper(ozoneRestClient, path, getMultiPartKey(delimiter)),
+        suite.getCluster());
   }
 
   @Test
   public void testPutAndGetKey() throws OzoneException, IOException {
     runTestPutAndGetKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAlphanumeric(1);
+    runTestPutAndGetKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   @Test
   public void testPutAndDeleteKey() throws OzoneException, IOException {
     runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAlphanumeric(1);
+    runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   @Test
   public void testPutAndListKey() throws OzoneException, IOException {
     runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAlphanumeric(1);
+    runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 
   @Test
   public void testGetKeyInfo() throws OzoneException, IOException {
     runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
+    String delimiter = RandomStringUtils.randomAlphanumeric(1);
+    runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,
+        getMultiPartKey(delimiter)));
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message