hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [1/2] hadoop git commit: HDFS-10268. Ozone: end-to-end integration for create/get volumes, buckets and keys. Contributed by Chris Nauroth.
Date Thu, 07 Apr 2016 21:41:19 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 b3044db40 -> fedb22d9b


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fedb22d9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 218058c..da6d9c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -23,6 +23,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_STORAGE_RPC_ADDRESS_KE
 import java.io.Closeable;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.util.Random;
+
+import com.google.common.base.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -37,6 +41,8 @@ import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolPB;
 import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.ozone.storage.StorageContainerManager;
+import org.apache.hadoop.ozone.web.client.OzoneClient;
+import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
 
@@ -53,6 +59,8 @@ public class MiniOzoneCluster extends MiniDFSCluster implements Closeable
{
   private static final Logger LOG =
       LoggerFactory.getLogger(MiniOzoneCluster.class);
 
+  private static final String USER_AUTH = "hdfs";
+
   private final OzoneConfiguration conf;
   private final StorageContainerManager scm;
 
@@ -126,24 +134,26 @@ public class MiniOzoneCluster extends MiniDFSCluster implements Closeable
{
   }
 
   /**
-   * Waits for the Ozone cluster to be ready for processing requests.
+   * Creates an {@link OzoneClient} connected to this cluster's REST service.
+   * Callers take ownership of the client and must close it when done.
+   *
+   * @return OzoneClient connected to this cluster's REST service
+   * @throws OzoneException if Ozone encounters an error creating the client
    */
-  public void waitOzoneReady() {
-    long begin = Time.monotonicNow();
-    while (scm.getDatanodeReport(DatanodeReportType.LIVE).length <
-        numDataNodes) {
-      if (Time.monotonicNow() - begin > 20000) {
-        throw new IllegalStateException(
-            "Timed out waiting for Ozone cluster to become ready.");
-      }
-      LOG.info("Waiting for Ozone cluster to become ready");
-      try {
-        Thread.sleep(100);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        throw new IllegalStateException(
-            "Interrupted while waiting for Ozone cluster to become ready.");
-      }
+  public OzoneClient createOzoneClient() throws OzoneException {
+    Preconditions.checkState(!getDataNodes().isEmpty(),
+        "Cannot create OzoneClient if the cluster has no DataNodes.");
+    // An Ozone request may originate at any DataNode, so pick one at random.
+    int dnIndex = new Random().nextInt(getDataNodes().size());
+    String uri = String.format("http://127.0.0.1:%d",
+        getDataNodes().get(dnIndex).getInfoPort());
+    LOG.info("Creating Ozone client to DataNode {} with URI {} and user {}",
+        dnIndex, uri, USER_AUTH);
+    try {
+      return new OzoneClient(uri, USER_AUTH);
+    } catch (URISyntaxException e) {
+      // We control the REST service URI, so it should never be invalid.
+      throw new IllegalStateException("Unexpected URISyntaxException", e);
     }
   }
 
@@ -155,14 +165,39 @@ public class MiniOzoneCluster extends MiniDFSCluster implements Closeable
{
    * @return RPC proxy for accessing container location information
    * @throws IOException if there is an I/O error
    */
-  protected StorageContainerLocationProtocolClientSideTranslatorPB
+  public StorageContainerLocationProtocolClientSideTranslatorPB
       createStorageContainerLocationClient() throws IOException {
     long version = RPC.getProtocolVersion(
         StorageContainerLocationProtocolPB.class);
     InetSocketAddress address = scm.getStorageContainerLocationRpcAddress();
+    LOG.info(
+        "Creating StorageContainerLocationProtocol RPC client with address {}",
+        address);
     return new StorageContainerLocationProtocolClientSideTranslatorPB(
         RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
         address, UserGroupInformation.getCurrentUser(), conf,
         NetUtils.getDefaultSocketFactory(conf), Client.getTimeout(conf)));
   }
+
+  /**
+   * Waits for the Ozone cluster to be ready for processing requests.
+   */
+  public void waitOzoneReady() {
+    long begin = Time.monotonicNow();
+    while (scm.getDatanodeReport(DatanodeReportType.LIVE).length <
+        numDataNodes) {
+      if (Time.monotonicNow() - begin > 20000) {
+        throw new IllegalStateException(
+            "Timed out waiting for Ozone cluster to become ready.");
+      }
+      LOG.info("Waiting for Ozone cluster to become ready");
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new IllegalStateException(
+            "Interrupted while waiting for Ozone cluster to become ready.");
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fedb22d9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
new file mode 100644
index 0000000..315f59d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.web;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.ozone.OzoneConsts.CHUNK_SIZE;
+import static org.junit.Assert.*;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConfiguration;
+import org.apache.hadoop.ozone.web.client.OzoneBucket;
+import org.apache.hadoop.ozone.web.client.OzoneClient;
+import org.apache.hadoop.ozone.web.client.OzoneVolume;
+import org.apache.hadoop.ozone.web.request.OzoneQuota;
+
+/**
+ * End-to-end testing of Ozone REST operations.
+ */
+public class TestOzoneRestWithMiniCluster {
+
+  private static MiniOzoneCluster cluster;
+  private static OzoneConfiguration conf;
+  private static int idSuffix;
+  private static OzoneClient ozoneClient;
+
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
+
+  @BeforeClass
+  public static void init() throws Exception {
+    conf = new OzoneConfiguration();
+    conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true);
+    conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "distributed");
+    conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_TRACE_ENABLED_KEY, true);
+    cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3).build();
+    cluster.waitOzoneReady();
+    ozoneClient = cluster.createOzoneClient();
+  }
+
+  @AfterClass
+  public static void shutdown() throws InterruptedException {
+    IOUtils.cleanup(null, ozoneClient, cluster);
+  }
+
+  @Test
+  public void testCreateAndGetVolume() throws Exception {
+    String volumeName = nextId("volume");
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    volume = ozoneClient.getVolume(volumeName);
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+  }
+
+  @Test
+  public void testCreateAndGetBucket() throws Exception {
+    String volumeName = nextId("volume");
+    String bucketName = nextId("bucket");
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    OzoneBucket bucket = volume.createBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+    bucket = volume.getBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+  }
+
+  @Test
+  public void testPutAndGetKey() throws Exception {
+    String volumeName = nextId("volume");
+    String bucketName = nextId("bucket");
+    String keyName = nextId("key");
+    String keyData = nextId("data");
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    OzoneBucket bucket = volume.createBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+  }
+
+  @Test
+  public void testPutAndGetEmptyKey() throws Exception {
+    String volumeName = nextId("volume");
+    String bucketName = nextId("bucket");
+    String keyName = nextId("key");
+    String keyData = "";
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    OzoneBucket bucket = volume.createBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+  }
+
+  @Test
+  public void testPutAndGetMultiChunkKey() throws Exception {
+    String volumeName = nextId("volume");
+    String bucketName = nextId("bucket");
+    String keyName = nextId("key");
+    int keyDataLen = 3 * CHUNK_SIZE;
+    String keyData = buildKeyData(keyDataLen);
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    OzoneBucket bucket = volume.createBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+  }
+
+  @Test
+  public void testPutAndGetMultiChunkKeyLastChunkPartial() throws Exception {
+    String volumeName = nextId("volume");
+    String bucketName = nextId("bucket");
+    String keyName = nextId("key");
+    int keyDataLen = (int)(2.5 * CHUNK_SIZE);
+    String keyData = buildKeyData(keyDataLen);
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    OzoneBucket bucket = volume.createBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+  }
+
+  @Test
+  public void testReplaceKey() throws Exception {
+    String volumeName = nextId("volume");
+    String bucketName = nextId("bucket");
+    String keyName = nextId("key");
+    int keyDataLen = (int)(2.5 * CHUNK_SIZE);
+    String keyData = buildKeyData(keyDataLen);
+    OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo", "100TB");
+    assertNotNull(volume);
+    assertEquals(volumeName, volume.getVolumeName());
+    assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
+    assertEquals("bilbo", volume.getOwnerName());
+    assertNotNull(volume.getQuota());
+    assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
+        volume.getQuota().sizeInBytes());
+    OzoneBucket bucket = volume.createBucket(bucketName);
+    assertNotNull(bucket);
+    assertEquals(bucketName, bucket.getBucketName());
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+
+    // Replace key with data consisting of fewer chunks.
+    keyDataLen = (int)(1.5 * CHUNK_SIZE);
+    keyData = buildKeyData(keyDataLen);
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+
+    // Replace key with data consisting of more chunks.
+    keyDataLen = (int)(3.5 * CHUNK_SIZE);
+    keyData = buildKeyData(keyDataLen);
+    bucket.putKey(keyName, keyData);
+    assertEquals(keyData, bucket.getKey(keyName));
+  }
+
+  /**
+   * Creates sample key data of the specified length.  The data is a string of
+   * printable ASCII characters.  This makes it easy to debug through visual
+   * inspection of the chunk files if a test fails.
+   *
+   * @param keyDataLen desired length of key data
+   * @return string of printable ASCII characters of the specified length
+   */
+  private static String buildKeyData(int keyDataLen) {
+    return new String(dataset(keyDataLen, 33, 93), UTF_8);
+  }
+
+  /**
+   * Generates identifiers unique enough for use in tests, so that individual
+   * tests don't collide on each others' data in the shared mini-cluster.
+   *
+   * @param idPrefix prefix to put in front of ID
+   * @return unique ID generated by appending a suffix to the given prefix
+   */
+  private static String nextId(String idPrefix) {
+    return idPrefix + ++idSuffix;
+  }
+}


Mime
View raw message