hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [1/2] hadoop git commit: HDFS-11564. Ozone: SCM: Add Comparable Metric Support. Contributed by Anu Engineer.
Date Mon, 03 Apr 2017 20:40:44 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 558b478ff -> 68eab679e


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
new file mode 100644
index 0000000..d798c61
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.placement;
+
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.scm.container.MockNodeManager;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom;
+import org.apache.hadoop.ozone.scm.exceptions.SCMException;
+import org.apache.hadoop.ozone.scm.node.NodeManager;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Asserts that allocation strategy works as expected.
+ */
+public class TestContainerPlacement {
+
+  private DescriptiveStatistics computeStatistics(NodeManager nodeManager) {
+    DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics();
+    for (DatanodeID id : nodeManager.getNodes(NodeManager.NODESTATE.HEALTHY)) {
+      float weightedValue =
+          nodeManager.getNodeStat(id).get().getScmUsed().get() / (float)
+              nodeManager.getNodeStat(id).get().getCapacity().get();
+      descriptiveStatistics.addValue(weightedValue);
+    }
+    return descriptiveStatistics;
+  }
+
+  /**
+   * This test simulates lots of Cluster I/O and updates the metadata in SCM.
+   * We simulate adding and removing containers from the cluster. It asserts
+   * that our placement algorithm has taken the capacity of nodes into
+   * consideration by asserting that standard deviation of used space on these
+   * has improved.
+   */
+  @Test
+  public void testCapacityPlacementYieldsBetterDataDistribution() throws
+      SCMException {
+    final int opsCount = 200 * 1000;
+    final int nodesRequired = 3;
+    Random random = new Random();
+
+    // The nature of init code in MockNodeManager yields similar clusters.
+    MockNodeManager nodeManagerCapacity = new MockNodeManager(true, 100);
+    MockNodeManager nodeManagerRandom = new MockNodeManager(true, 100);
+    DescriptiveStatistics beforeCapacity =
+        computeStatistics(nodeManagerCapacity);
+    DescriptiveStatistics beforeRandom = computeStatistics(nodeManagerRandom);
+
+    //Assert that our initial layout of clusters are similar.
+    assertEquals(beforeCapacity.getStandardDeviation(), beforeRandom
+        .getStandardDeviation(), 0.001);
+
+    SCMContainerPlacementCapacity capacityPlacer = new
+        SCMContainerPlacementCapacity(nodeManagerCapacity, new Configuration());
+    SCMContainerPlacementRandom randomPlacer = new
+        SCMContainerPlacementRandom(nodeManagerRandom, new Configuration());
+
+    for (int x = 0; x < opsCount; x++) {
+      long containerSize = random.nextInt(100) * OzoneConsts.GB;
+      List<DatanodeID> nodesCapacity =
+          capacityPlacer.chooseDatanodes(nodesRequired, containerSize);
+      assertEquals(nodesRequired, nodesCapacity.size());
+
+      List<DatanodeID> nodesRandom = randomPlacer.chooseDatanodes(nodesRequired,
+          containerSize);
+
+      // One fifth of all calls are delete
+      if (x % 5 == 0) {
+        deleteContainer(nodeManagerCapacity, nodesCapacity, containerSize);
+        deleteContainer(nodeManagerRandom, nodesRandom, containerSize);
+      } else {
+        createContainer(nodeManagerCapacity, nodesCapacity, containerSize);
+        createContainer(nodeManagerRandom, nodesRandom, containerSize);
+      }
+    }
+    DescriptiveStatistics postCapacity = computeStatistics(nodeManagerCapacity);
+    DescriptiveStatistics postRandom = computeStatistics(nodeManagerRandom);
+
+    // This is a very bold claim, and needs large number of I/O operations.
+    // The claim in this assertion is that we improved the data distribution
+    // of this cluster in relation to the start state of the cluster.
+    Assert.assertTrue(beforeCapacity.getStandardDeviation() >
+        postCapacity.getStandardDeviation());
+
+    // This asserts that Capacity placement yields a better placement
+    // algorithm than random placement, since both cluster started at an
+    // identical state.
+
+    Assert.assertTrue(postRandom.getStandardDeviation() >
+        postCapacity.getStandardDeviation());
+  }
+
+  private void deleteContainer(MockNodeManager nodeManager,
+      List<DatanodeID> nodes, long containerSize) {
+    for (DatanodeID id : nodes) {
+      nodeManager.delContainer(id, containerSize);
+    }
+  }
+
+  private void createContainer(MockNodeManager nodeManager,
+      List<DatanodeID> nodes, long containerSize) {
+    for (DatanodeID id : nodes) {
+      nodeManager.addContainer(id, containerSize);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java
new file mode 100644
index 0000000..ccc23c3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/placement/TestDatanodeMetrics.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.placement;
+
+import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests that test Metrics that support placement.
+ */
+public class TestDatanodeMetrics {
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
+  @Test
+  public void testSCMNodeMetric() {
+    SCMNodeStat stat = new SCMNodeStat(100L, 10L, 90L);
+    assertEquals((long) stat.getCapacity().get(), 100L);
+    assertEquals((long) stat.getScmUsed().get(), 10L);
+    assertEquals((long) stat.getRemaining().get(), 90L);
+    SCMNodeMetric metric = new SCMNodeMetric(stat);
+
+    SCMNodeStat newStat = new SCMNodeStat(100L, 10L, 90L);
+    assertEquals((long) stat.getCapacity().get(), 100L);
+    assertEquals((long) stat.getScmUsed().get(), 10L);
+    assertEquals((long) stat.getRemaining().get(), 90L);
+
+    SCMNodeMetric newMetric = new SCMNodeMetric(newStat);
+    assertTrue(metric.isEqual(newMetric.get()));
+
+    newMetric.add(stat);
+    assertTrue(newMetric.isGreater(metric.get()));
+
+    SCMNodeMetric zeroMetric = new SCMNodeMetric(new SCMNodeStat());
+    // Assert we can handle zero capacity.
+    assertTrue(metric.isGreater(zeroMetric.get()));
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index 8a8ea68..a30707c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@ -22,8 +22,8 @@ import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.scm.container.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.container.SCMContainerPlacementCapacity;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
 import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java
index 14c36fc..638de9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/MockNodeManager.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.scm.container;
 
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
+import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
 import org.apache.hadoop.ozone.protocol.VersionResponse;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -25,9 +26,9 @@ import org.apache.hadoop.ozone.protocol.proto
     .StorageContainerDatanodeProtocolProtos;
 import org.apache.hadoop.ozone.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-
+import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.SCMNodeStat;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -39,21 +40,55 @@ import java.util.Map;
  * Test Helper for testing container Mapping.
  */
 public class MockNodeManager implements NodeManager {
-  private final List<DatanodeID> healthyNodes;
   private static final int HEALTHY_NODE_COUNT = 10;
+  private final static NodeData[] NODES = {
+      new NodeData(10L * OzoneConsts.TB,  OzoneConsts.GB),
+      new NodeData(64L * OzoneConsts.TB, 100 * OzoneConsts.GB),
+      new NodeData(128L * OzoneConsts.TB, 256 * OzoneConsts.GB),
+      new NodeData(40L * OzoneConsts.TB, OzoneConsts.TB),
+      new NodeData(256L * OzoneConsts.TB, 200 * OzoneConsts.TB),
+      new NodeData(20L * OzoneConsts.TB, 10 * OzoneConsts.GB),
+      new NodeData(32L * OzoneConsts.TB, 16 * OzoneConsts.TB),
+      new NodeData(OzoneConsts.TB, 900 * OzoneConsts.GB),
+  };
+  private final List<DatanodeID> healthyNodes;
+  private final Map<String, SCMNodeStat> nodeMetricMap;
+  private final SCMNodeStat aggregateStat;
   private boolean chillmode;
 
-  public MockNodeManager() {
+  public MockNodeManager(boolean initializeFakeNodes, int nodeCount) {
     this.healthyNodes = new LinkedList<>();
-    for (int x = 0; x < 10; x++) {
-      healthyNodes.add(SCMTestUtils.getDatanodeID());
+    this.nodeMetricMap = new HashMap<>();
+    aggregateStat = new SCMNodeStat();
+    if (initializeFakeNodes) {
+      for (int x = 0; x < nodeCount; x++) {
+        DatanodeID id = SCMTestUtils.getDatanodeID();
+        healthyNodes.add(id);
+        populateNodeMetric(id, x);
+      }
     }
     chillmode = false;
   }
 
   /**
+   * Invoked from ctor to create some node Metrics.
+   *
+   * @param datanodeID - Datanode ID
+   */
+  private void populateNodeMetric(DatanodeID datanodeID, int x) {
+    SCMNodeStat newStat = new SCMNodeStat();
+    long remaining =
+        NODES[x % NODES.length].capacity - NODES[x % NODES.length].used;
+    newStat.set(
+        (NODES[x % NODES.length].capacity),
+        (NODES[x % NODES.length].used), remaining);
+    this.nodeMetricMap.put(datanodeID.toString(), newStat);
+    aggregateStat.add(newStat);
+  }
+
+  /**
    * Sets the chill mode value.
-   * @param chillmode  boolean
+   * @param chillmode boolean
    */
   public void setChillmode(boolean chillmode) {
     this.chillmode = chillmode;
@@ -184,7 +219,7 @@ public class MockNodeManager implements NodeManager {
    */
   @Override
   public SCMNodeStat getStats() {
-    return null;
+    return aggregateStat;
   }
 
   /**
@@ -193,7 +228,7 @@ public class MockNodeManager implements NodeManager {
    */
   @Override
   public Map<String, SCMNodeStat> getNodeStats() {
-    return null;
+    return nodeMetricMap;
   }
 
   /**
@@ -202,8 +237,8 @@ public class MockNodeManager implements NodeManager {
    * @return node stat if it is live/stale, null if it is dead or does't exist.
    */
   @Override
-  public SCMNodeStat getNodeStat(DatanodeID datanodeID) {
-    return null;
+  public SCMNodeMetric getNodeStat(DatanodeID datanodeID) {
+    return new SCMNodeMetric(nodeMetricMap.get(datanodeID.toString()));
   }
 
   /**
@@ -283,15 +318,103 @@ public class MockNodeManager implements NodeManager {
   @Override
   public List<SCMCommand> sendHeartbeat(DatanodeID datanodeID,
       SCMNodeReport nodeReport) {
+    if ((datanodeID != null) && (nodeReport != null) && (nodeReport
+        .getStorageReportCount() > 0)) {
+      SCMNodeStat stat = this.nodeMetricMap.get(datanodeID.toString());
+
+      long totalCapacity = 0L;
+      long totalRemaining = 0L;
+      long totalScmUsed = 0L;
+      List<StorageContainerDatanodeProtocolProtos.SCMStorageReport>
+          storageReports = nodeReport.getStorageReportList();
+      for (StorageContainerDatanodeProtocolProtos.SCMStorageReport report :
+          storageReports) {
+        totalCapacity += report.getCapacity();
+        totalRemaining +=report.getRemaining();
+        totalScmUsed += report.getScmUsed();
+      }
+      aggregateStat.subtract(stat);
+      stat.set(totalCapacity, totalScmUsed, totalRemaining);
+      aggregateStat.add(stat);
+      nodeMetricMap.put(datanodeID.toString(), stat);
+
+    }
     return null;
   }
 
   @Override
   public Map<String, Integer> getNodeCount() {
     Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
-    for(NodeManager.NODESTATE state : NodeManager.NODESTATE.values()) {
+    for (NodeManager.NODESTATE state : NodeManager.NODESTATE.values()) {
       nodeCountMap.put(state.toString(), getNodeCount(state));
     }
     return nodeCountMap;
   }
+
+  /**
+   * Makes it easy to add a container.
+   *
+   * @param datanodeID datanode ID
+   * @param size number of bytes.
+   */
+  public void addContainer(DatanodeID datanodeID, long size) {
+    SCMNodeStat stat = this.nodeMetricMap.get(datanodeID.toString());
+    if (stat != null) {
+      aggregateStat.subtract(stat);
+      stat.getCapacity().add(size);
+      aggregateStat.add(stat);
+      nodeMetricMap.put(datanodeID.toString(), stat);
+    }
+  }
+
+  /**
+   * Makes it easy to simulate a delete of a container.
+   *
+   * @param datanodeID datanode ID
+   * @param size number of bytes.
+   */
+  public void delContainer(DatanodeID datanodeID, long size) {
+    SCMNodeStat stat = this.nodeMetricMap.get(datanodeID.toString());
+    if (stat != null) {
+      aggregateStat.subtract(stat);
+      stat.getCapacity().subtract(size);
+      aggregateStat.add(stat);
+      nodeMetricMap.put(datanodeID.toString(), stat);
+    }
+  }
+
+  /**
+   * A class to declare some values for the nodes so that our tests
+   * won't fail.
+   */
+  private static class NodeData {
+    private long capacity, used;
+
+    /**
+     * Constructs a nodeDefinition.
+     *
+     * @param capacity capacity.
+     * @param used used.
+     */
+    NodeData(long capacity, long used) {
+      this.capacity = capacity;
+      this.used = used;
+    }
+
+    public long getCapacity() {
+      return capacity;
+    }
+
+    public void setCapacity(long capacity) {
+      this.capacity = capacity;
+    }
+
+    public long getUsed() {
+      return used;
+    }
+
+    public void setUsed(long used) {
+      this.used = used;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
index d83053a..0b9b76f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/container/TestContainerMapping.java
@@ -59,15 +59,17 @@ public class TestContainerMapping {
     testDir = Paths.get(path).toFile();
     boolean folderExisted = testDir.exists() || testDir.mkdirs();
     if (!folderExisted) {
-      throw new IOException("Unable to create test diectory path");
+      throw new IOException("Unable to create test directory path");
     }
-    nodeManager = new MockNodeManager();
+    nodeManager = new MockNodeManager(true, 10);
     mapping = new ContainerMapping(conf, nodeManager, 128);
   }
 
   @AfterClass
   public static void cleanup() throws IOException {
-    mapping.close();
+    if(mapping != null) {
+      mapping.close();
+    }
     FileUtil.fullyDelete(testDir);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java
index 02f1486..d28866e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -26,10 +26,11 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto
+    .StorageContainerDatanodeProtocolProtos;
 import org.apache.hadoop.ozone.scm.container.ContainerMapping;
-import org.apache.hadoop.ozone.scm.container.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.container.SCMContainerPlacementCapacity;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
 import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.scm.client.ScmClient;
 import org.apache.hadoop.scm.container.common.helpers.Pipeline;
@@ -93,6 +94,7 @@ public class TestContainerPlacement {
     return new ContainerMapping(config, scmNodeManager, cacheSize);
 
   }
+
   /**
    * Test capacity based container placement policy with node reports.
    *
@@ -122,7 +124,7 @@ public class TestContainerPlacement {
     List<DatanodeID> datanodes =
         SCMTestUtils.getRegisteredDatanodeIDs(nodeManager, nodeCount);
     try {
-      for (DatanodeID datanodeID: datanodes) {
+      for (DatanodeID datanodeID : datanodes) {
         StorageContainerDatanodeProtocolProtos.SCMNodeReport.Builder nrb =
             StorageContainerDatanodeProtocolProtos.SCMNodeReport.newBuilder();
         StorageContainerDatanodeProtocolProtos.SCMStorageReport.Builder srb =
@@ -139,11 +141,11 @@ public class TestContainerPlacement {
           100, 4 * 1000);
       assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY));
       assertEquals(capacity * nodeCount,
-          nodeManager.getStats().getCapacity());
+          (long) nodeManager.getStats().getCapacity().get());
       assertEquals(used * nodeCount,
-          nodeManager.getStats().getScmUsed());
+          (long) nodeManager.getStats().getScmUsed().get());
       assertEquals(remaining * nodeCount,
-          nodeManager.getStats().getRemaining());
+          (long) nodeManager.getStats().getRemaining().get());
 
       assertTrue(nodeManager.isOutOfNodeChillMode());
 
@@ -155,7 +157,7 @@ public class TestContainerPlacement {
       final long newUsed = 7L * OzoneConsts.GB;
       final long newRemaining = capacity - newUsed;
 
-      for (DatanodeID datanodeID: datanodes) {
+      for (DatanodeID datanodeID : datanodes) {
         StorageContainerDatanodeProtocolProtos.SCMNodeReport.Builder nrb =
             StorageContainerDatanodeProtocolProtos.SCMNodeReport.newBuilder();
         StorageContainerDatanodeProtocolProtos.SCMStorageReport.Builder srb =
@@ -168,14 +170,14 @@ public class TestContainerPlacement {
             nrb.addStorageReport(srb).build());
       }
 
-      GenericTestUtils.waitFor(() -> nodeManager.getStats().getRemaining() ==
-              nodeCount * newRemaining,
+      GenericTestUtils.waitFor(() -> nodeManager.getStats().getRemaining()
+              .get() == nodeCount * newRemaining,
           100, 4 * 1000);
 
       thrown.expect(IOException.class);
       thrown.expectMessage(
-          startsWith("No healthy node found with enough remaining capacity to" +
-              " allocate container."));
+          startsWith("Unable to find enough nodes that meet the space " +
+              "requirement in healthy node set."));
       String container2 = UUID.randomUUID().toString();
       containerManager.allocateContainer(container2,
           ScmClient.ReplicationFactor.THREE);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
index da567cb..25ef7cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
 import org.apache.hadoop.ozone.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
+import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.PathUtils;
@@ -45,14 +46,14 @@ import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeoutException;
 
+import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD;
+import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY;
+import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE;
 import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_MS;
 import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS;
 import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS;
 import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS;
 import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_MS;
-import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY;
-import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE;
-import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.StringStartsWith.startsWith;
@@ -158,7 +159,7 @@ public class TestNodeManager {
       GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(),
           100, 4 * 1000);
       assertFalse("No heartbeats, Node manager should have been in" +
-              " chill mode.", nodeManager.isOutOfNodeChillMode());
+          " chill mode.", nodeManager.isOutOfNodeChillMode());
     }
   }
 
@@ -208,7 +209,7 @@ public class TestNodeManager {
       GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(),
           100, 4 * 1000);
       assertFalse("Not enough nodes have send heartbeat to node" +
-              "manager.", nodeManager.isOutOfNodeChillMode());
+          "manager.", nodeManager.isOutOfNodeChillMode());
     }
   }
 
@@ -631,11 +632,12 @@ public class TestNodeManager {
    * @throws InterruptedException
    */
   private void heartbeatNodeSet(SCMNodeManager manager, List<DatanodeID> list,
-                                int sleepDuration) throws InterruptedException {
+      int sleepDuration) throws InterruptedException {
     while (!Thread.currentThread().isInterrupted()) {
       for (DatanodeID dn : list) {
         manager.sendHeartbeat(dn, null);
-      }      Thread.sleep(sleepDuration);
+      }
+      Thread.sleep(sleepDuration);
     }
   }
 
@@ -664,7 +666,7 @@ public class TestNodeManager {
    * @return true if we found the expected number.
    */
   private boolean findNodes(NodeManager nodeManager, int count,
-                            NodeManager.NODESTATE state) {
+      NodeManager.NODESTATE state) {
     return count == nodeManager.getNodeCount(state);
   }
 
@@ -690,7 +692,6 @@ public class TestNodeManager {
     conf.setInt(OZONE_SCM_MAX_HB_COUNT_TO_PROCESS, 7000);
 
 
-
     try (SCMNodeManager nodeManager = createNodeManager(conf)) {
       List<DatanodeID> healthyNodeList = createNodeSet(nodeManager,
           healthyCount, "Healthy");
@@ -902,7 +903,7 @@ public class TestNodeManager {
 
 
       // Assert that node manager force enter cannot be overridden by nodes HBs.
-      for(int x= 0; x < 20; x++) {
+      for (int x = 0; x < 20; x++) {
         DatanodeID datanode = SCMTestUtils.getDatanodeID(nodeManager);
         nodeManager.sendHeartbeat(datanode, null);
       }
@@ -952,12 +953,12 @@ public class TestNodeManager {
       GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatProcessed(),
           100, 4 * 1000);
       assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY));
-      assertEquals(capacity * nodeCount,
-          nodeManager.getStats().getCapacity());
-      assertEquals(used * nodeCount,
-          nodeManager.getStats().getScmUsed());
-      assertEquals(remaining * nodeCount,
-          nodeManager.getStats().getRemaining());
+      assertEquals(capacity * nodeCount, (long) nodeManager.getStats()
+          .getCapacity().get());
+      assertEquals(used * nodeCount, (long) nodeManager.getStats()
+          .getScmUsed().get());
+      assertEquals(remaining * nodeCount, (long) nodeManager.getStats()
+          .getRemaining().get());
     }
   }
 
@@ -998,31 +999,41 @@ public class TestNodeManager {
         Thread.sleep(100);
       }
 
-      final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount -1);
-      final long expectedRemaining = capacity -
-          usedPerHeartbeat * (heartbeatCount - 1);
+      final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1);
+      final long expectedRemaining = capacity - expectedScmUsed;
 
       GenericTestUtils.waitFor(
-          () -> nodeManager.getStats().getScmUsed() == expectedScmUsed, 100,
-          4 * 1000);
+          () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed,
+          100, 4 * 1000);
 
-      assertEquals(capacity, nodeManager.getStats().getCapacity());
-      assertEquals(expectedScmUsed, nodeManager.getStats().getScmUsed());
-      assertEquals(expectedRemaining, nodeManager.getStats().getRemaining());
+      long foundCapacity = nodeManager.getStats().getCapacity().get();
+      assertEquals(capacity, foundCapacity);
+
+      long foundScmUsed = nodeManager.getStats().getScmUsed().get();
+      assertEquals(expectedScmUsed, foundScmUsed);
+
+      long foundRemaining = nodeManager.getStats().getRemaining().get();
+      assertEquals(expectedRemaining, foundRemaining);
 
       // Test NodeManager#getNodeStats
       assertEquals(nodeCount, nodeManager.getNodeStats().size());
-      assertEquals(capacity, nodeManager.getNodeStat(datanodeID).getCapacity());
-      assertEquals(expectedScmUsed,
-          nodeManager.getNodeStat(datanodeID).getScmUsed());
-      assertEquals(expectedRemaining,
-          nodeManager.getNodeStat(datanodeID).getRemaining());
+      long nodeCapacity = nodeManager.getNodeStat(datanodeID).get()
+          .getCapacity().get();
+      assertEquals(capacity, nodeCapacity);
+
+      foundScmUsed = nodeManager.getNodeStat(datanodeID).get().getScmUsed()
+          .get();
+      assertEquals(expectedScmUsed, foundScmUsed);
+
+      foundRemaining = nodeManager.getNodeStat(datanodeID).get()
+          .getRemaining().get();
+      assertEquals(expectedRemaining, foundRemaining);
 
       // Compare the result from
       // NodeManager#getNodeStats and NodeManager#getNodeStat
       SCMNodeStat stat1 = nodeManager.getNodeStats().
           get(datanodeID.getDatanodeUuid());
-      SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeID);
+      SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeID).get();
       assertEquals(stat1, stat2);
 
       // Wait up to 4s so that the node becomes stale
@@ -1031,11 +1042,17 @@ public class TestNodeManager {
           () -> nodeManager.getNodeCount(NodeManager.NODESTATE.STALE) == 1, 100,
           4 * 1000);
       assertEquals(nodeCount, nodeManager.getNodeStats().size());
-      assertEquals(capacity, nodeManager.getNodeStat(datanodeID).getCapacity());
-      assertEquals(expectedScmUsed,
-          nodeManager.getNodeStat(datanodeID).getScmUsed());
-      assertEquals(expectedRemaining,
-          nodeManager.getNodeStat(datanodeID).getRemaining());
+
+      foundCapacity = nodeManager.getNodeStat(datanodeID).get()
+          .getCapacity().get();
+      assertEquals(capacity, foundCapacity);
+      foundScmUsed = nodeManager.getNodeStat(datanodeID).get()
+          .getScmUsed().get();
+      assertEquals(expectedScmUsed, foundScmUsed);
+
+      foundRemaining = nodeManager.getNodeStat(datanodeID).get().
+          getRemaining().get();
+      assertEquals(expectedRemaining, foundRemaining);
 
       // Wait up to 4 more seconds so the node becomes dead
       // Verify usage info should be updated.
@@ -1044,11 +1061,16 @@ public class TestNodeManager {
           4 * 1000);
 
       assertEquals(0, nodeManager.getNodeStats().size());
-      assertEquals(0, nodeManager.getStats().getCapacity());
-      assertEquals(0, nodeManager.getStats().getScmUsed());
-      assertEquals(0, nodeManager.getStats().getRemaining());
+      foundCapacity = nodeManager.getStats().getCapacity().get();
+      assertEquals(0, foundCapacity);
+
+      foundScmUsed = nodeManager.getStats().getScmUsed().get();
+      assertEquals(0, foundScmUsed);
+
+      foundRemaining = nodeManager.getStats().getRemaining().get();
+      assertEquals(0, foundRemaining);
 
-      // Send a new report to bring the dead node back to healty
+      // Send a new report to bring the dead node back to healthy
       SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
       SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
       srb.setStorageUuid(UUID.randomUUID().toString());
@@ -1063,14 +1085,18 @@ public class TestNodeManager {
           () -> nodeManager.getNodeCount(NodeManager.NODESTATE.HEALTHY) == 1,
           100, 5 * 1000);
       GenericTestUtils.waitFor(
-          () -> nodeManager.getStats().getScmUsed() == expectedScmUsed, 100,
-          4 * 1000);
+          () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed,
+          100, 4 * 1000);
       assertEquals(nodeCount, nodeManager.getNodeStats().size());
-      assertEquals(capacity, nodeManager.getNodeStat(datanodeID).getCapacity());
-      assertEquals(expectedScmUsed,
-          nodeManager.getNodeStat(datanodeID).getScmUsed());
-      assertEquals(expectedRemaining,
-          nodeManager.getNodeStat(datanodeID).getRemaining());
+      foundCapacity = nodeManager.getNodeStat(datanodeID).get()
+          .getCapacity().get();
+      assertEquals(capacity, foundCapacity);
+      foundScmUsed = nodeManager.getNodeStat(datanodeID).get().getScmUsed()
+          .get();
+      assertEquals(expectedScmUsed, foundScmUsed);
+      foundRemaining = nodeManager.getNodeStat(datanodeID).get()
+          .getRemaining().get();
+      assertEquals(expectedRemaining, foundRemaining);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68eab679/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java
index 3655544..2fd2c83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodePoolManager.java
@@ -24,8 +24,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConfiguration;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
-import org.apache.hadoop.ozone.scm.container.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.container.SCMContainerPlacementCapacity;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
 import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.Rule;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message