hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [1/7] hadoop git commit: HDDS-1. Remove SCM Block DB. Contributed by Xiaoyu Yao.
Date Mon, 07 May 2018 21:59:58 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk a3a1552c3 -> 3a43ac285


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
index 04473d1..888b72e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
@@ -17,11 +17,13 @@
  */
 package org.apache.hadoop.ozone.scm;
 
+import com.google.common.primitives.Longs;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -43,6 +45,8 @@ import org.junit.rules.Timeout;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
 
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
@@ -122,28 +126,29 @@ public class TestSCMCli {
 
   @Test
   public void testCreateContainer() throws Exception {
-    String containerName =  "containerTestCreate";
+    long containerID = ContainerTestHelper.getTestContainerID();
     try {
-      scm.getClientProtocolServer().getContainer(containerName);
+      scm.getClientProtocolServer().getContainer(containerID);
       fail("should not be able to get the container");
     } catch (IOException ioe) {
       assertTrue(ioe.getMessage().contains(
-          "Specified key does not exist. key : " + containerName));
+          "Specified key does not exist. key : " + containerID));
     }
-    String[] args = {"-container", "-create", "-c", containerName};
+    String[] args = {"-container", "-create", "-c",
+        Long.toString(containerID)};
     assertEquals(ResultCode.SUCCESS, cli.run(args));
-    Pipeline container = scm.getClientProtocolServer()
-        .getContainer(containerName);
+    ContainerInfo container = scm.getClientProtocolServer()
+        .getContainer(containerID);
     assertNotNull(container);
-    assertEquals(containerName, container.getContainerName());
+    assertEquals(containerID, container.containerID());
   }
 
-  private boolean containerExist(String containerName) {
+  private boolean containerExist(long containerID) {
     try {
-      Pipeline scmPipeline = scm.getClientProtocolServer()
-          .getContainer(containerName);
-      return scmPipeline != null
-          && containerName.equals(scmPipeline.getContainerName());
+      ContainerInfo container = scm.getClientProtocolServer()
+          .getContainer(containerID);
+      return container != null
+          && containerID == container.getContainerID();
     } catch (IOException e) {
       return false;
     }
@@ -162,29 +167,31 @@ public class TestSCMCli {
     // 1. Test to delete a non-empty container.
     // ****************************************
     // Create an non-empty container
-    containerName = "non-empty-container";
-    pipeline = containerOperationClient
+    ContainerInfo container = containerOperationClient
         .createContainer(xceiverClientManager.getType(),
-            HddsProtos.ReplicationFactor.ONE, containerName, containerOwner);
+            HddsProtos.ReplicationFactor.ONE, containerOwner);
 
     ContainerData cdata = ContainerData
-        .getFromProtBuf(containerOperationClient.readContainer(pipeline), conf);
-    KeyUtils.getDB(cdata, conf).put(containerName.getBytes(),
+        .getFromProtBuf(containerOperationClient.readContainer(
+            container.getContainerID(), container.getPipeline()), conf);
+    KeyUtils.getDB(cdata, conf).put(Longs.toByteArray(container.getContainerID()),
         "someKey".getBytes());
-    Assert.assertTrue(containerExist(containerName));
+    Assert.assertTrue(containerExist(container.getContainerID()));
 
     // Gracefully delete a container should fail because it is open.
-    delCmd = new String[] {"-container", "-delete", "-c", containerName};
+    delCmd = new String[] {"-container", "-delete", "-c",
+        Long.toString(container.getContainerID())};
     testErr = new ByteArrayOutputStream();
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     exitCode = runCommandAndGetOutput(delCmd, out, testErr);
     assertEquals(EXECUTION_ERROR, exitCode);
     assertTrue(testErr.toString()
         .contains("Deleting an open container is not allowed."));
-    Assert.assertTrue(containerExist(containerName));
+    Assert.assertTrue(containerExist(container.getContainerID()));
 
     // Close the container
-    containerOperationClient.closeContainer(pipeline);
+    containerOperationClient.closeContainer(
+        container.getContainerID(), container.getPipeline());
 
     // Gracefully delete a container should fail because it is not empty.
     testErr = new ByteArrayOutputStream();
@@ -192,42 +199,46 @@ public class TestSCMCli {
     assertEquals(EXECUTION_ERROR, exitCode2);
     assertTrue(testErr.toString()
         .contains("Container cannot be deleted because it is not empty."));
-    Assert.assertTrue(containerExist(containerName));
+    Assert.assertTrue(containerExist(container.getContainerID()));
 
     // Try force delete again.
-    delCmd = new String[] {"-container", "-delete", "-c", containerName, "-f"};
+    delCmd = new String[] {"-container", "-delete", "-c",
+        Long.toString(container.getContainerID()), "-f"};
     exitCode = runCommandAndGetOutput(delCmd, out, null);
     assertEquals("Expected success, found:", ResultCode.SUCCESS, exitCode);
-    assertFalse(containerExist(containerName));
+    assertFalse(containerExist(container.getContainerID()));
 
     // ****************************************
     // 2. Test to delete an empty container.
     // ****************************************
     // Create an empty container
-    containerName = "empty-container";
-    pipeline = containerOperationClient
+    ContainerInfo emptyContainer = containerOperationClient
         .createContainer(xceiverClientManager.getType(),
-            HddsProtos.ReplicationFactor.ONE, containerName, containerOwner);
-    containerOperationClient.closeContainer(pipeline);
-    Assert.assertTrue(containerExist(containerName));
+            HddsProtos.ReplicationFactor.ONE, containerOwner);
+    containerOperationClient.closeContainer(container.getContainerID(),
+        container.getPipeline());
+    Assert.assertTrue(containerExist(container.getContainerID()));
 
     // Successfully delete an empty container.
-    delCmd = new String[] {"-container", "-delete", "-c", containerName};
+    delCmd = new String[] {"-container", "-delete", "-c",
+        Long.toString(emptyContainer.getContainerID())};
     exitCode = runCommandAndGetOutput(delCmd, out, null);
     assertEquals(ResultCode.SUCCESS, exitCode);
-    assertFalse(containerExist(containerName));
+    assertFalse(containerExist(emptyContainer.getContainerID()));
 
     // After the container is deleted,
-    // a same name container can now be recreated.
-    containerOperationClient.createContainer(xceiverClientManager.getType(),
-        HddsProtos.ReplicationFactor.ONE, containerName, containerOwner);
-    Assert.assertTrue(containerExist(containerName));
+    // another container can now be recreated.
+    ContainerInfo newContainer = containerOperationClient.
+        createContainer(xceiverClientManager.getType(),
+            HddsProtos.ReplicationFactor.ONE, containerOwner);
+    Assert.assertTrue(containerExist(newContainer.getContainerID()));
 
     // ****************************************
     // 3. Test to delete a non-exist container.
     // ****************************************
-    containerName = "non-exist-container";
-    delCmd = new String[] {"-container", "-delete", "-c", containerName};
+    long nonExistContainerID =  ContainerTestHelper.getTestContainerID();
+    delCmd = new String[] {"-container", "-delete", "-c",
+        Long.toString(nonExistContainerID)};
     testErr = new ByteArrayOutputStream();
     exitCode = runCommandAndGetOutput(delCmd, out, testErr);
     assertEquals(EXECUTION_ERROR, exitCode);
@@ -267,12 +278,13 @@ public class TestSCMCli {
         EXECUTION_ERROR, exitCode);
 
     // Create an empty container.
-    cname = "ContainerTestInfo1";
-    Pipeline pipeline = containerOperationClient
+    ContainerInfo container = containerOperationClient
         .createContainer(xceiverClientManager.getType(),
-            HddsProtos.ReplicationFactor.ONE, cname, containerOwner);
+            HddsProtos.ReplicationFactor.ONE, containerOwner);
     ContainerData data = ContainerData
-        .getFromProtBuf(containerOperationClient.readContainer(pipeline), conf);
+        .getFromProtBuf(containerOperationClient.
+            readContainer(container.getContainerID(),
+                container.getPipeline()), conf);
 
     info = new String[]{"-container", "-info", "-c", cname};
     ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -289,12 +301,12 @@ public class TestSCMCli {
     out.reset();
 
     // Create an non-empty container
-    cname = "ContainerTestInfo2";
-    pipeline = containerOperationClient
+    container = containerOperationClient
         .createContainer(xceiverClientManager.getType(),
-            HddsProtos.ReplicationFactor.ONE, cname, containerOwner);
+            HddsProtos.ReplicationFactor.ONE, containerOwner);
     data = ContainerData
-        .getFromProtBuf(containerOperationClient.readContainer(pipeline), conf);
+        .getFromProtBuf(containerOperationClient.readContainer(
+            container.getContainerID(), container.getPipeline()), conf);
     KeyUtils.getDB(data, conf).put(cname.getBytes(), "someKey".getBytes());
 
     info = new String[]{"-container", "-info", "-c", cname};
@@ -311,13 +323,15 @@ public class TestSCMCli {
 
 
     // Close last container and test info again.
-    containerOperationClient.closeContainer(pipeline);
+    containerOperationClient.closeContainer(
+        container.getContainerID(), container.getPipeline());
 
     info = new String[] {"-container", "-info", "-c", cname};
     exitCode = runCommandAndGetOutput(info, out, null);
     assertEquals(ResultCode.SUCCESS, exitCode);
     data = ContainerData
-        .getFromProtBuf(containerOperationClient.readContainer(pipeline), conf);
+        .getFromProtBuf(containerOperationClient.readContainer(
+            container.getContainerID(), container.getPipeline()), conf);
 
     openStatus = data.isOpen() ? "OPEN" : "CLOSED";
     expected = String.format(formatStrWithHash, cname, openStatus,
@@ -347,11 +361,12 @@ public class TestSCMCli {
   @Test
   public void testListContainerCommand() throws Exception {
     // Create 20 containers for testing.
-    String prefix = "ContainerForTesting";
+    List<ContainerInfo> containers = new ArrayList<>();
     for (int index = 0; index < 20; index++) {
-      String containerName = String.format("%s%02d", prefix, index);
-      containerOperationClient.createContainer(xceiverClientManager.getType(),
-          HddsProtos.ReplicationFactor.ONE, containerName, containerOwner);
+      ContainerInfo container = containerOperationClient.createContainer(
+          xceiverClientManager.getType(), HddsProtos.ReplicationFactor.ONE,
+          containerOwner);
+      containers.add(container);
     }
 
     ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -367,9 +382,11 @@ public class TestSCMCli {
     out.reset();
     err.reset();
 
+    long startContainerID = containers.get(0).getContainerID();
+    String startContainerIDStr = Long.toString(startContainerID);
     // Test with -start and -count, the value of -count is negative.
     args = new String[] {"-container", "-list",
-        "-start", prefix + 0, "-count", "-1"};
+        "-start", startContainerIDStr, "-count", "-1"};
     exitCode = runCommandAndGetOutput(args, out, err);
     assertEquals(EXECUTION_ERROR, exitCode);
     assertTrue(err.toString()
@@ -378,67 +395,23 @@ public class TestSCMCli {
     out.reset();
     err.reset();
 
-    String startName = String.format("%s%02d", prefix, 0);
-
     // Test with -start and -count.
     args = new String[] {"-container", "-list", "-start",
-        startName, "-count", "10"};
-    exitCode = runCommandAndGetOutput(args, out, err);
-    assertEquals(ResultCode.SUCCESS, exitCode);
-    for (int index = 0; index < 10; index++) {
-      String containerName = String.format("%s%02d", prefix, index);
-      assertTrue(out.toString().contains(containerName));
-    }
-
-    out.reset();
-    err.reset();
-
-    // Test with -start, -prefix and -count.
-    startName = String.format("%s%02d", prefix, 0);
-    String prefixName = String.format("%s0", prefix);
-    args = new String[] {"-container", "-list", "-start",
-        startName, "-prefix", prefixName, "-count", "20"};
-    exitCode = runCommandAndGetOutput(args, out, err);
-    assertEquals(ResultCode.SUCCESS, exitCode);
-    for (int index = 0; index < 10; index++) {
-      String containerName = String.format("%s%02d", prefix, index);
-      assertTrue(out.toString().contains(containerName));
-    }
-
-    out.reset();
-    err.reset();
-
-    startName = String.format("%s%02d", prefix, 0);
-    prefixName = String.format("%s0", prefix);
-    args = new String[] {"-container", "-list", "-start",
-        startName, "-prefix", prefixName, "-count", "4"};
+        startContainerIDStr, "-count", "10"};
     exitCode = runCommandAndGetOutput(args, out, err);
     assertEquals(ResultCode.SUCCESS, exitCode);
-    for (int index = 0; index < 4; index++) {
-      String containerName = String.format("%s%02d", prefix, index);
-      assertTrue(out.toString().contains(containerName));
+    for (int index = 1; index < 10; index++) {
+      String containerID = Long.toString(
+          containers.get(index).getContainerID());
+      assertTrue(out.toString().contains(containerID));
     }
 
     out.reset();
     err.reset();
 
-    prefixName = String.format("%s0", prefix);
-    args = new String[] {"-container", "-list",
-        "-prefix", prefixName, "-count", "6"};
-    exitCode = runCommandAndGetOutput(args, out, err);
-    assertEquals(ResultCode.SUCCESS, exitCode);
-    for (int index = 0; index < 6; index++) {
-      String containerName = String.format("%s%02d", prefix, index);
-      assertTrue(out.toString().contains(containerName));
-    }
-
-    out.reset();
-    err.reset();
-
-    // Test with -start and -prefix, while -count doesn't exist.
-    prefixName = String.format("%s%02d", prefix, 20);
+    // Test with -start, while -count doesn't exist.
     args = new String[] {"-container", "-list", "-start",
-        startName, "-prefix", prefixName, "-count", "10"};
+        startContainerIDStr};
     exitCode = runCommandAndGetOutput(args, out, err);
     assertEquals(ResultCode.SUCCESS, exitCode);
     assertTrue(out.toString().isEmpty());
@@ -446,21 +419,23 @@ public class TestSCMCli {
 
   @Test
   public void testCloseContainer() throws Exception {
-    String containerName =  "containerTestClose";
-    String[] args = {"-container", "-create", "-c", containerName};
+    long containerID = ContainerTestHelper.getTestContainerID();
+    String[] args = {"-container", "-create", "-c",
+        Long.toString(containerID)};
     assertEquals(ResultCode.SUCCESS, cli.run(args));
-    Pipeline container = scm.getClientProtocolServer()
-        .getContainer(containerName);
+    ContainerInfo container = scm.getClientProtocolServer()
+        .getContainer(containerID);
     assertNotNull(container);
-    assertEquals(containerName, container.getContainerName());
+    assertEquals(containerID, container.getContainerID());
 
-    ContainerInfo containerInfo = scm.getContainerInfo(containerName);
+    ContainerInfo containerInfo = scm.getContainerInfo(containerID);
     assertEquals(OPEN, containerInfo.getState());
 
-    String[] args1 = {"-container", "-close", "-c", containerName};
+    String[] args1 = {"-container", "-close", "-c",
+        Long.toString(containerID)};
     assertEquals(ResultCode.SUCCESS, cli.run(args1));
 
-    containerInfo = scm.getContainerInfo(containerName);
+    containerInfo = scm.getContainerInfo(containerID);
     assertEquals(CLOSED, containerInfo.getState());
 
     // closing this container again will trigger an error.
@@ -502,9 +477,7 @@ public class TestSCMCli {
     String[] args2 = {"-container", "-create", "-help"};
     assertEquals(ResultCode.SUCCESS, cli.run(args2));
     String expected2 =
-        "usage: hdfs scm -container -create <option>\n" +
-        "where <option> is\n" +
-        " -c <arg>   Specify container name\n";
+        "usage: hdfs scm -container -create\n";
     assertEquals(expected2, testContent.toString());
     testContent.reset();
 
@@ -513,7 +486,7 @@ public class TestSCMCli {
     String expected3 =
         "usage: hdfs scm -container -delete <option>\n" +
         "where <option> is\n" +
-        " -c <arg>   Specify container name\n" +
+        " -c <arg>   Specify container id\n" +
         " -f         forcibly delete a container\n";
     assertEquals(expected3, testContent.toString());
     testContent.reset();
@@ -523,7 +496,7 @@ public class TestSCMCli {
     String expected4 =
         "usage: hdfs scm -container -info <option>\n" +
         "where <option> is\n" +
-        " -c <arg>   Specify container name\n";
+        " -c <arg>   Specify container id\n";
     assertEquals(expected4, testContent.toString());
     testContent.reset();
 
@@ -532,9 +505,8 @@ public class TestSCMCli {
     String expected5 =
         "usage: hdfs scm -container -list <option>\n" +
             "where <option> can be the following\n" +
-            " -count <arg>    Specify count number, required\n" +
-            " -prefix <arg>   Specify prefix container name\n" +
-            " -start <arg>    Specify start container name\n";
+            " -start <arg>    Specify start container id, required\n" +
+            " -count <arg>   Specify count number name\n";
     assertEquals(expected5, testContent.toString());
     testContent.reset();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
index 332e679..1d19bb3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
@@ -22,9 +22,8 @@ import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
 import static org.junit.Assert.assertEquals;
 
-import java.util.UUID;
-
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.scm.TestUtils;
@@ -225,7 +224,7 @@ public class TestSCMMetrics {
 
     for (int i = 0; i < numReport; i++) {
       ContainerReport report = new ContainerReport(
-          UUID.randomUUID().toString(), DigestUtils.sha256Hex("Simulated"));
+          RandomUtils.nextLong(), DigestUtils.sha256Hex("Simulated"));
       report.setSize(stat.getSize().get());
       report.setBytesUsed(stat.getUsed().get());
       report.setReadCount(stat.getReadCount().get());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
index 85403a2..07ad6ef 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
@@ -19,13 +19,13 @@ package org.apache.hadoop.ozone.scm;
 
 import com.google.common.cache.Cache;
 import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
 import org.apache.hadoop.hdds.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
@@ -78,29 +78,24 @@ public class TestXceiverClientManager {
     OzoneConfiguration conf = new OzoneConfiguration();
     XceiverClientManager clientManager = new XceiverClientManager(conf);
 
-    String containerName1 = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline1 = storageContainerLocationClient
+    ContainerInfo container1 = storageContainerLocationClient
         .allocateContainer(clientManager.getType(), clientManager.getFactor(),
-            containerName1, containerOwner);
-    XceiverClientSpi client1 = clientManager.acquireClient(pipeline1);
+            containerOwner);
+    XceiverClientSpi client1 = clientManager.acquireClient(container1.getPipeline(),
+        container1.getContainerID());
     Assert.assertEquals(1, client1.getRefcount());
-    Assert.assertEquals(containerName1,
-        client1.getPipeline().getContainerName());
 
-    String containerName2 = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline2 = storageContainerLocationClient
+    ContainerInfo container2 = storageContainerLocationClient
         .allocateContainer(clientManager.getType(), clientManager.getFactor(),
-            containerName2, containerOwner);
-    XceiverClientSpi client2 = clientManager.acquireClient(pipeline2);
+            containerOwner);
+    XceiverClientSpi client2 = clientManager.acquireClient(container2.getPipeline(),
+        container2.getContainerID());
     Assert.assertEquals(1, client2.getRefcount());
-    Assert.assertEquals(containerName2,
-        client2.getPipeline().getContainerName());
 
-    XceiverClientSpi client3 = clientManager.acquireClient(pipeline1);
+    XceiverClientSpi client3 = clientManager.acquireClient(container1.getPipeline(),
+        container1.getContainerID());
     Assert.assertEquals(2, client3.getRefcount());
     Assert.assertEquals(2, client1.getRefcount());
-    Assert.assertEquals(containerName1,
-        client3.getPipeline().getContainerName());
     Assert.assertEquals(client1, client3);
     clientManager.releaseClient(client1);
     clientManager.releaseClient(client2);
@@ -112,43 +107,43 @@ public class TestXceiverClientManager {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(SCM_CONTAINER_CLIENT_MAX_SIZE_KEY, 1);
     XceiverClientManager clientManager = new XceiverClientManager(conf);
-    Cache<String, XceiverClientSpi> cache =
+    Cache<Long, XceiverClientSpi> cache =
         clientManager.getClientCache();
 
-    String containerName1 = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline1 =
+    ContainerInfo container1 =
         storageContainerLocationClient.allocateContainer(
             clientManager.getType(), HddsProtos.ReplicationFactor.ONE,
-            containerName1, containerOwner);
-    XceiverClientSpi client1 = clientManager.acquireClient(pipeline1);
+            containerOwner);
+    XceiverClientSpi client1 = clientManager.acquireClient(container1.getPipeline(),
+        container1.getContainerID());
     Assert.assertEquals(1, client1.getRefcount());
-    Assert.assertEquals(containerName1,
-        client1.getPipeline().getContainerName());
+    Assert.assertEquals(container1.getPipeline(),
+        client1.getPipeline());
 
-    String containerName2 = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline2 =
+    ContainerInfo container2 =
         storageContainerLocationClient.allocateContainer(
             clientManager.getType(),
-            HddsProtos.ReplicationFactor.ONE, containerName2, containerOwner);
-    XceiverClientSpi client2 = clientManager.acquireClient(pipeline2);
+            HddsProtos.ReplicationFactor.ONE, containerOwner);
+    XceiverClientSpi client2 = clientManager.acquireClient(container2.getPipeline(),
+        container2.getContainerID());
     Assert.assertEquals(1, client2.getRefcount());
-    Assert.assertEquals(containerName2,
-        client2.getPipeline().getContainerName());
     Assert.assertNotEquals(client1, client2);
 
     // least recent container (i.e containerName1) is evicted
-    XceiverClientSpi nonExistent1 = cache.getIfPresent(containerName1);
+    XceiverClientSpi nonExistent1 = cache.getIfPresent(container1.getContainerID());
     Assert.assertEquals(null, nonExistent1);
     // However container call should succeed because of refcount on the client.
     String traceID1 = "trace" + RandomStringUtils.randomNumeric(4);
-    ContainerProtocolCalls.createContainer(client1,  traceID1);
+    ContainerProtocolCalls.createContainer(client1,
+        container1.getContainerID(),  traceID1);
 
     // After releasing the client, this connection should be closed
     // and any container operations should fail
     clientManager.releaseClient(client1);
     exception.expect(IOException.class);
     exception.expectMessage("This channel is not connected.");
-    ContainerProtocolCalls.createContainer(client1,  traceID1);
+    ContainerProtocolCalls.createContainer(client1,
+        container1.getContainerID(), traceID1);
     clientManager.releaseClient(client2);
   }
 
@@ -157,42 +152,39 @@ public class TestXceiverClientManager {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(SCM_CONTAINER_CLIENT_MAX_SIZE_KEY, 1);
     XceiverClientManager clientManager = new XceiverClientManager(conf);
-    Cache<String, XceiverClientSpi> cache =
+    Cache<Long, XceiverClientSpi> cache =
         clientManager.getClientCache();
 
-    String containerName1 = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline1 =
+    ContainerInfo container1 =
         storageContainerLocationClient.allocateContainer(
             clientManager.getType(),
-            clientManager.getFactor(), containerName1, containerOwner);
-    XceiverClientSpi client1 = clientManager.acquireClient(pipeline1);
+            clientManager.getFactor(), containerOwner);
+    XceiverClientSpi client1 = clientManager.acquireClient(container1.getPipeline(),
+        container1.getContainerID());
     Assert.assertEquals(1, client1.getRefcount());
-    Assert.assertEquals(containerName1,
-        client1.getPipeline().getContainerName());
 
     clientManager.releaseClient(client1);
     Assert.assertEquals(0, client1.getRefcount());
 
-    String containerName2 = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline2 = storageContainerLocationClient
+    ContainerInfo container2 = storageContainerLocationClient
         .allocateContainer(clientManager.getType(), clientManager.getFactor(),
-            containerName2, containerOwner);
-    XceiverClientSpi client2 = clientManager.acquireClient(pipeline2);
+            containerOwner);
+    XceiverClientSpi client2 = clientManager.acquireClient(container2.getPipeline(),
+        container2.getContainerID());
     Assert.assertEquals(1, client2.getRefcount());
-    Assert.assertEquals(containerName2,
-        client2.getPipeline().getContainerName());
     Assert.assertNotEquals(client1, client2);
 
 
     // now client 1 should be evicted
-    XceiverClientSpi nonExistent = cache.getIfPresent(containerName1);
+    XceiverClientSpi nonExistent = cache.getIfPresent(container1.getContainerID());
     Assert.assertEquals(null, nonExistent);
 
     // Any container operation should now fail
     String traceID2 = "trace" + RandomStringUtils.randomNumeric(4);
     exception.expect(IOException.class);
     exception.expectMessage("This channel is not connected.");
-    ContainerProtocolCalls.createContainer(client1, traceID2);
+    ContainerProtocolCalls.createContainer(client1,
+        container1.getContainerID(), traceID2);
     clientManager.releaseClient(client2);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java
index 1403f89..99742c2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java
@@ -27,9 +27,11 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 
 import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.protocol.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -41,6 +43,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
 import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -78,14 +81,15 @@ public class TestXceiverClientMetrics {
     OzoneConfiguration conf = new OzoneConfiguration();
     XceiverClientManager clientManager = new XceiverClientManager(conf);
 
-    String containerName = "container" + RandomStringUtils.randomNumeric(10);
-    Pipeline pipeline = storageContainerLocationClient
+    ContainerInfo container = storageContainerLocationClient
         .allocateContainer(clientManager.getType(), clientManager.getFactor(),
-            containerName, containerOwner);
-    XceiverClientSpi client = clientManager.acquireClient(pipeline);
+            containerOwner);
+    XceiverClientSpi client = clientManager.acquireClient(
+        container.getPipeline(), container.getContainerID());
 
     ContainerCommandRequestProto request = ContainerTestHelper
-        .getCreateContainerRequest(containerName, pipeline);
+        .getCreateContainerRequest(container.getContainerID(),
+            container.getPipeline());
     client.sendCommand(request);
 
     MetricsRecordBuilder containerMetrics = getMetrics(
@@ -109,11 +113,12 @@ public class TestXceiverClientMetrics {
         try {
           // use async interface for testing pending metrics
           for (int i = 0; i < numRequest; i++) {
-            String keyName = OzoneUtils.getRequestID();
+            BlockID blockID = ContainerTestHelper.
+                getTestBlockID(container.getContainerID());
             ContainerProtos.ContainerCommandRequestProto smallFileRequest;
 
             smallFileRequest = ContainerTestHelper.getWriteSmallFileRequest(
-                client.getPipeline(), containerName, keyName, 1024);
+                client.getPipeline(), blockID, 1024);
             CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
                 response = client.sendCommandAsync(smallFileRequest);
             computeResults.add(response);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index ae30fb3..b621a08 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -623,12 +623,11 @@ public class TestKeys {
         List<KsmKeyLocationInfo> locations =
             keyInfo.getLatestVersionLocations().getLocationList();
         for (KsmKeyLocationInfo location : locations) {
-          String containerName = location.getContainerName();
-          KeyData keyData = new KeyData(containerName, location.getBlockID());
+          KeyData keyData = new KeyData(location.getBlockID());
           KeyData blockInfo = cm.getContainerManager()
               .getKeyManager().getKey(keyData);
           ContainerData containerData = cm.getContainerManager()
-              .readContainer(containerName);
+              .readContainer(keyData.getContainerID());
           File dataDir = ContainerUtils
               .getDataDirectory(containerData).toFile();
           for (ContainerProtos.ChunkInfo chunkInfo : blockInfo.getChunks()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
index fa0eaa2..13cc40b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
@@ -21,11 +21,11 @@ import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
 import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
 import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
 import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
 import org.apache.hadoop.ozone.common.BlockGroup;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -480,8 +480,8 @@ public class KSMMetadataManagerImpl implements KSMMetadataManager {
       if (latest == null) {
         return Collections.emptyList();
       }
-      List<String> item = latest.getLocationList().stream()
-          .map(KsmKeyLocationInfo::getBlockID)
+      List<BlockID> item = latest.getLocationList().stream()
+          .map(b->new BlockID(b.getContainerID(), b.getLocalID()))
           .collect(Collectors.toList());
       BlockGroup keyBlocks = BlockGroup.newBuilder()
           .setKeyName(DFSUtil.bytes2String(entry.getKey()))
@@ -510,9 +510,9 @@ public class KSMMetadataManagerImpl implements KSMMetadataManager {
         continue;
       }
       // Get block keys as a list.
-      List<String> item = info.getLatestVersionLocations()
+      List<BlockID> item = info.getLatestVersionLocations()
           .getBlocksLatestVersionOnly().stream()
-          .map(KsmKeyLocationInfo::getBlockID)
+          .map(b->new BlockID(b.getContainerID(), b.getLocalID()))
           .collect(Collectors.toList());
       BlockGroup keyBlocks = BlockGroup.newBuilder()
           .setKeyName(DFSUtil.bytes2String(entry.getKey()))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
index 14fb69c..e51ab28 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
 import org.apache.hadoop.ozone.common.BlockGroup;
 import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.utils.BackgroundService;
 import org.apache.hadoop.utils.BackgroundTask;
@@ -117,7 +118,7 @@ public class KeyDeletingService extends BackgroundService {
               LOG.warn("Key {} deletion failed because some of the blocks"
                   + " were failed to delete, failed blocks: {}",
                   result.getObjectKey(),
-                  String.join(",", result.getFailedBlocks()));
+                  StringUtils.join(",", result.getFailedBlocks()));
             }
           }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
index 70ba178..8ee7d25 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
@@ -203,8 +203,7 @@ public class KeyManagerImpl implements KeyManager {
       KsmKeyInfo keyInfo =
           KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
       KsmKeyLocationInfo info = new KsmKeyLocationInfo.Builder()
-          .setContainerName(allocatedBlock.getPipeline().getContainerName())
-          .setBlockID(allocatedBlock.getKey())
+          .setBlockID(allocatedBlock.getBlockID())
           .setShouldCreateContainer(allocatedBlock.getCreateContainer())
           .setLength(scmBlockSize)
           .setOffset(0)
@@ -256,8 +255,7 @@ public class KeyManagerImpl implements KeyManager {
         AllocatedBlock allocatedBlock =
             scmBlockClient.allocateBlock(allocateSize, type, factor, ksmId);
         KsmKeyLocationInfo subKeyInfo = new KsmKeyLocationInfo.Builder()
-            .setContainerName(allocatedBlock.getPipeline().getContainerName())
-            .setBlockID(allocatedBlock.getKey())
+            .setBlockID(allocatedBlock.getBlockID())
             .setShouldCreateContainer(allocatedBlock.getCreateContainer())
             .setLength(allocateSize)
             .setOffset(0)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
index 76312e7..120eb06 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
@@ -715,7 +715,7 @@ public final class KeySpaceManager extends ServiceRuntimeInfoImpl
   }
 
   @Override
-  public KsmKeyLocationInfo  allocateBlock(KsmKeyArgs args, int clientID)
+  public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
       throws IOException {
     try {
       metrics.incNumBlockAllocateCalls();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
index 7a2d7cc..8e2540a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.ksm;
 import org.apache.hadoop.ozone.common.BlockGroup;
 import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
 import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.utils.BackgroundService;
 import org.apache.hadoop.utils.BackgroundTask;
 import org.apache.hadoop.utils.BackgroundTaskQueue;
@@ -97,7 +98,7 @@ public class OpenKeyCleanupService extends BackgroundService {
               LOG.warn("Deleting open Key {} failed because some of the blocks"
                       + " were failed to delete, failed blocks: {}",
                   result.getObjectKey(),
-                  String.join(",", result.getFailedBlocks()));
+                  StringUtils.join(",", result.getFailedBlocks()));
             }
           }
           LOG.info("Found {} expired open key entries, successfully " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
index 70d80d5..e3f6cc9 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
@@ -60,7 +60,7 @@ public class BenchMarkContainerStateMap {
     for (int x = 1; x < 1000; x++) {
       try {
         ContainerInfo containerInfo = new ContainerInfo.Builder()
-            .setContainerName(pipeline.getContainerName()).setState(CLOSED)
+            .setState(CLOSED)
             .setPipeline(pipeline)
             // This is bytes allocated for blocks inside container, not the
             // container size
@@ -76,7 +76,7 @@ public class BenchMarkContainerStateMap {
     for (int y = currentCount; y < 2000; y++) {
       try {
         ContainerInfo containerInfo = new ContainerInfo.Builder()
-            .setContainerName(pipeline.getContainerName()).setState(OPEN)
+            .setState(OPEN)
             .setPipeline(pipeline)
             // This is bytes allocated for blocks inside container, not the
             // container size
@@ -91,7 +91,7 @@ public class BenchMarkContainerStateMap {
     }
     try {
       ContainerInfo containerInfo = new ContainerInfo.Builder()
-          .setContainerName(pipeline.getContainerName()).setState(OPEN)
+          .setState(OPEN)
           .setPipeline(pipeline)
           // This is bytes allocated for blocks inside container, not the
           // container size
@@ -142,7 +142,7 @@ public class BenchMarkContainerStateMap {
     for (; i.hasNext();) {
       pipelineChannel.addMember(i.next());
     }
-    return new Pipeline(containerName, pipelineChannel);
+    return new Pipeline(pipelineChannel);
   }
 
   @Benchmark
@@ -151,7 +151,7 @@ public class BenchMarkContainerStateMap {
     Pipeline pipeline = createSingleNodePipeline(UUID.randomUUID().toString());
     int cid = state.containerID.incrementAndGet();
     ContainerInfo containerInfo = new ContainerInfo.Builder()
-        .setContainerName(pipeline.getContainerName()).setState(CLOSED)
+        .setState(CLOSED)
         .setPipeline(pipeline)
         // This is bytes allocated for blocks inside container, not the
         // container size

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 468fee5..b73f108 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -21,7 +21,9 @@ import com.google.protobuf.ByteString;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
@@ -32,6 +34,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
 
 import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
 import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
+import org.apache.hadoop.util.Time;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Scope;
@@ -41,6 +44,7 @@ import org.openjdk.jmh.annotations.TearDown;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
@@ -75,6 +79,14 @@ public class BenchMarkDatanodeDispatcher {
   private AtomicInteger keyCount;
   private AtomicInteger chunkCount;
 
+  final int initContainers = 100;
+  final int initKeys = 50;
+  final int initChunks = 100;
+
+  List<Long> containers;
+  List<Long> keys;
+  List<String> chunks;
+
   @Setup(Level.Trial)
   public void initialize() throws IOException {
     datanodeUuid = UUID.randomUUID().toString();
@@ -110,20 +122,39 @@ public class BenchMarkDatanodeDispatcher {
     keyCount = new AtomicInteger();
     chunkCount = new AtomicInteger();
 
+    containers = new ArrayList<>();
+    keys = new ArrayList<>();
+    chunks = new ArrayList<>();
+
     // Create containers
-    for (int x = 0; x < 100; x++) {
-      String containerName = "container-" + containerCount.getAndIncrement();
-      dispatcher.dispatch(getCreateContainerCommand(containerName));
+    for (int x = 0; x < initContainers; x++) {
+      long containerID = Time.getUtcTime() + x;
+      ContainerCommandRequestProto req = getCreateContainerCommand(containerID);
+      dispatcher.dispatch(req);
+      containers.add(containerID);
+      containerCount.getAndIncrement();
     }
+
+    for (int x = 0; x < initKeys; x++) {
+      keys.add(Time.getUtcTime()+x);
+    }
+
+    for (int x = 0; x < initChunks; x++) {
+      chunks.add("chunk-" + x);
+    }
+
     // Add chunk and keys to the containers
-    for (int x = 0; x < 50; x++) {
-      String chunkName = "chunk-" + chunkCount.getAndIncrement();
-      String keyName = "key-" + keyCount.getAndIncrement();
-      for (int y = 0; y < 100; y++) {
-        String containerName = "container-" + y;
-        dispatcher.dispatch(getWriteChunkCommand(containerName, chunkName));
+    for (int x = 0; x < initKeys; x++) {
+      String chunkName = chunks.get(x);
+      chunkCount.getAndIncrement();
+      long key = keys.get(x);
+      keyCount.getAndIncrement();
+      for (int y = 0; y < initContainers; y++) {
+        long containerID = containers.get(y);
+        BlockID  blockID = new BlockID(containerID, key);
         dispatcher
-            .dispatch(getPutKeyCommand(containerName, chunkName, keyName));
+            .dispatch(getPutKeyCommand(blockID, chunkName));
+        dispatcher.dispatch(getWriteChunkCommand(blockID, chunkName));
       }
     }
   }
@@ -134,147 +165,166 @@ public class BenchMarkDatanodeDispatcher {
     FileUtils.deleteDirectory(new File(baseDir));
   }
 
-  private ContainerCommandRequestProto getCreateContainerCommand(
-      String containerName) {
+  private ContainerCommandRequestProto getCreateContainerCommand(long containerID) {
     CreateContainerRequestProto.Builder createRequest =
         CreateContainerRequestProto.newBuilder();
     createRequest.setPipeline(
-        new Pipeline(containerName, pipelineChannel).getProtobufMessage());
+        new Pipeline(pipelineChannel).getProtobufMessage());
     createRequest.setContainerData(
-        ContainerData.newBuilder().setName(containerName).build());
+        ContainerData.newBuilder().setContainerID(
+            containerID).build());
 
     ContainerCommandRequestProto.Builder request =
         ContainerCommandRequestProto.newBuilder();
     request.setCmdType(ContainerProtos.Type.CreateContainer);
     request.setCreateContainer(createRequest);
     request.setDatanodeUuid(datanodeUuid);
-    request.setTraceID(containerName + "-trace");
+    request.setTraceID(containerID + "-trace");
     return request.build();
   }
 
   private ContainerCommandRequestProto getWriteChunkCommand(
-      String containerName, String key) {
-
+      BlockID blockID, String chunkName) {
     WriteChunkRequestProto.Builder writeChunkRequest = WriteChunkRequestProto
         .newBuilder()
-        .setPipeline(
-            new Pipeline(containerName, pipelineChannel).getProtobufMessage())
-        .setKeyName(key)
-        .setChunkData(getChunkInfo(containerName, key))
+        .setBlockID(blockID.getProtobuf())
+        .setChunkData(getChunkInfo(blockID, chunkName))
         .setData(data);
 
     ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
         .newBuilder();
     request.setCmdType(ContainerProtos.Type.WriteChunk)
-        .setTraceID(containerName + "-" + key +"-trace")
+        .setTraceID(getBlockTraceID(blockID))
         .setDatanodeUuid(datanodeUuid)
         .setWriteChunk(writeChunkRequest);
     return request.build();
   }
 
   private ContainerCommandRequestProto getReadChunkCommand(
-      String containerName, String key) {
+      BlockID blockID, String chunkName) {
     ReadChunkRequestProto.Builder readChunkRequest = ReadChunkRequestProto
         .newBuilder()
-        .setPipeline(
-            new Pipeline(containerName, pipelineChannel).getProtobufMessage())
-        .setKeyName(key)
-        .setChunkData(getChunkInfo(containerName, key));
+        .setBlockID(blockID.getProtobuf())
+        .setChunkData(getChunkInfo(blockID, chunkName));
     ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
         .newBuilder();
     request.setCmdType(ContainerProtos.Type.ReadChunk)
-        .setTraceID(containerName + "-" + key +"-trace")
+        .setTraceID(getBlockTraceID(blockID))
         .setDatanodeUuid(datanodeUuid)
         .setReadChunk(readChunkRequest);
     return request.build();
   }
 
   private ContainerProtos.ChunkInfo getChunkInfo(
-      String containerName, String key) {
+      BlockID blockID, String chunkName) {
     ContainerProtos.ChunkInfo.Builder builder =
         ContainerProtos.ChunkInfo.newBuilder()
             .setChunkName(
-                DigestUtils.md5Hex(key) + "_stream_" + containerName + "_chunk_"
-                    + key)
+                DigestUtils.md5Hex(chunkName)
+                    + "_stream_" + blockID.getContainerID() + "_block_"
+                    + blockID.getLocalID())
             .setOffset(0).setLen(data.size());
     return builder.build();
   }
 
   private ContainerCommandRequestProto getPutKeyCommand(
-      String containerName, String chunkKey, String key) {
+      BlockID blockID, String chunkKey) {
     PutKeyRequestProto.Builder putKeyRequest = PutKeyRequestProto
         .newBuilder()
-        .setPipeline(
-            new Pipeline(containerName, pipelineChannel).getProtobufMessage())
-        .setKeyData(getKeyData(containerName, chunkKey, key));
+        .setKeyData(getKeyData(blockID, chunkKey));
     ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
         .newBuilder();
     request.setCmdType(ContainerProtos.Type.PutKey)
-        .setTraceID(containerName + "-" + key +"-trace")
+        .setTraceID(getBlockTraceID(blockID))
         .setDatanodeUuid(datanodeUuid)
         .setPutKey(putKeyRequest);
     return request.build();
   }
 
   private ContainerCommandRequestProto getGetKeyCommand(
-      String containerName, String chunkKey, String key) {
+      BlockID blockID, String chunkKey) {
     GetKeyRequestProto.Builder readKeyRequest = GetKeyRequestProto.newBuilder()
-        .setPipeline(
-            new Pipeline(containerName, pipelineChannel).getProtobufMessage())
-        .setKeyData(getKeyData(containerName, chunkKey, key));
+        .setKeyData(getKeyData(blockID, chunkKey));
     ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
         .newBuilder()
         .setCmdType(ContainerProtos.Type.GetKey)
-        .setTraceID(containerName + "-" + key +"-trace")
+        .setTraceID(getBlockTraceID(blockID))
         .setDatanodeUuid(datanodeUuid)
         .setGetKey(readKeyRequest);
     return request.build();
   }
 
   private ContainerProtos.KeyData getKeyData(
-      String containerName, String chunkKey, String key) {
+      BlockID blockID, String chunkKey) {
     ContainerProtos.KeyData.Builder builder =  ContainerProtos.KeyData
         .newBuilder()
-        .setContainerName(containerName)
-        .setName(key)
-        .addChunks(getChunkInfo(containerName, chunkKey));
+        .setBlockID(blockID.getProtobuf())
+        .addChunks(getChunkInfo(blockID, chunkKey));
     return builder.build();
   }
 
   @Benchmark
   public void createContainer(BenchMarkDatanodeDispatcher bmdd) {
-    bmdd.dispatcher.dispatch(getCreateContainerCommand(
-        "container-" + containerCount.getAndIncrement()));
+    long containerID = RandomUtils.nextLong();
+    ContainerCommandRequestProto req = getCreateContainerCommand(containerID);
+    bmdd.dispatcher.dispatch(req);
+    bmdd.containers.add(containerID);
+    bmdd.containerCount.getAndIncrement();
   }
 
 
   @Benchmark
   public void writeChunk(BenchMarkDatanodeDispatcher bmdd) {
-    String containerName = "container-" + random.nextInt(containerCount.get());
     bmdd.dispatcher.dispatch(getWriteChunkCommand(
-        containerName, "chunk-" + chunkCount.getAndIncrement()));
+        getRandomBlockID(), getNewChunkToWrite()));
   }
 
   @Benchmark
   public void readChunk(BenchMarkDatanodeDispatcher bmdd) {
-    String containerName = "container-" + random.nextInt(containerCount.get());
-    String chunkKey = "chunk-" + random.nextInt(chunkCount.get());
-    bmdd.dispatcher.dispatch(getReadChunkCommand(containerName, chunkKey));
+    BlockID blockID = getRandomBlockID();
+    String chunkKey = getRandomChunkToRead();
+    bmdd.dispatcher.dispatch(getReadChunkCommand(blockID, chunkKey));
   }
 
   @Benchmark
   public void putKey(BenchMarkDatanodeDispatcher bmdd) {
-    String containerName = "container-" + random.nextInt(containerCount.get());
-    String chunkKey = "chunk-" + random.nextInt(chunkCount.get());
-    bmdd.dispatcher.dispatch(getPutKeyCommand(
-        containerName, chunkKey, "key-" + keyCount.getAndIncrement()));
+    BlockID blockID = getRandomBlockID();
+    String chunkKey = getNewChunkToWrite();
+    bmdd.dispatcher.dispatch(getPutKeyCommand(blockID, chunkKey));
   }
 
   @Benchmark
   public void getKey(BenchMarkDatanodeDispatcher bmdd) {
-    String containerName = "container-" + random.nextInt(containerCount.get());
-    String chunkKey = "chunk-" + random.nextInt(chunkCount.get());
-    String key = "key-" + random.nextInt(keyCount.get());
-    bmdd.dispatcher.dispatch(getGetKeyCommand(containerName, chunkKey, key));
+    BlockID blockID = getRandomBlockID();
+    String chunkKey = getNewChunkToWrite();
+    bmdd.dispatcher.dispatch(getGetKeyCommand(blockID, chunkKey));
+  }
+
+  // Chunks writes from benchmark only reaches certain containers
+  // Use initChunks instead of updated counters to guarantee
+  // key/chunks are readable.
+
+  private BlockID getRandomBlockID() {
+    return new BlockID(getRandomContainerID(), getRandomKeyID());
+  }
+
+  private long getRandomContainerID() {
+    return containers.get(random.nextInt(initContainers));
+  }
+
+  private long getRandomKeyID() {
+    return keys.get(random.nextInt(initKeys));
+  }
+
+  private String getRandomChunkToRead() {
+    return chunks.get(random.nextInt(initChunks));
+  }
+
+  private String getNewChunkToWrite() {
+    return "chunk-" + chunkCount.getAndIncrement();
+  }
+
+  private String getBlockTraceID(BlockID blockID) {
+    return blockID.getContainerID() + "-" + blockID.getLocalID() +"-trace";
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
index 0890e4b..c4c6f9e 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
@@ -72,21 +72,21 @@ public class BenchMarkRocksDbStore {
         .toFile();
     opts.setCreateIfMissing(true);
     opts.setWriteBufferSize(
-        (long) StorageUnit.MB.toBytes(Long.valueOf(writeBufferSize)));
-    opts.setMaxWriteBufferNumber(Integer.valueOf(maxWriteBufferNumber));
-    opts.setMaxBackgroundFlushes(Integer.valueOf(maxBackgroundFlushes));
+        (long) StorageUnit.MB.toBytes(Long.parseLong(writeBufferSize)));
+    opts.setMaxWriteBufferNumber(Integer.parseInt(maxWriteBufferNumber));
+    opts.setMaxBackgroundFlushes(Integer.parseInt(maxBackgroundFlushes));
     BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
     tableConfig.setBlockSize(
-        (long) StorageUnit.KB.toBytes(Long.valueOf(blockSize)));
-    opts.setMaxOpenFiles(Integer.valueOf(maxOpenFiles));
+        (long) StorageUnit.KB.toBytes(Long.parseLong(blockSize)));
+    opts.setMaxOpenFiles(Integer.parseInt(maxOpenFiles));
     opts.setMaxBytesForLevelBase(
-        (long) StorageUnit.MB.toBytes(Long.valueOf(maxBytesForLevelBase)));
+        (long) StorageUnit.MB.toBytes(Long.parseLong(maxBytesForLevelBase)));
     opts.setCompactionStyle(CompactionStyle.UNIVERSAL);
     opts.setLevel0FileNumCompactionTrigger(10);
     opts.setLevel0SlowdownWritesTrigger(20);
     opts.setLevel0StopWritesTrigger(40);
     opts.setTargetFileSizeBase(
-        (long) StorageUnit.MB.toBytes(Long.valueOf(maxBytesForLevelBase)) / 10);
+        (long) StorageUnit.MB.toBytes(Long.parseLong(maxBytesForLevelBase)) / 10);
     opts.setMaxBackgroundCompactions(8);
     opts.setUseFsync(false);
     opts.setBytesPerSync(8388608);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message