hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sammic...@apache.org
Subject [hadoop-ozone] 17/18: HDDS-2924. Fix Pipeline#nodeIdsHash collision issue. (#478)
Date Mon, 10 Feb 2020 02:23:35 GMT
This is an automated email from the ASF dual-hosted git repository.

sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git

commit dd2b7e8cec022ccfc1f4abf193dbb62cc5e883ec
Author: Xiaoyu Yao <xyao@apache.org>
AuthorDate: Mon Jan 27 12:45:58 2020 -0800

    HDDS-2924. Fix Pipeline#nodeIdsHash collision issue. (#478)
---
 .../apache/hadoop/hdds/scm/pipeline/Pipeline.java  | 38 ++++++++++----------
 .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 40 ++++++++++------------
 .../hdds/scm/pipeline/PipelineStateManager.java    |  7 ----
 .../hdds/scm/pipeline/RatisPipelineProvider.java   |  8 -----
 .../hdds/scm/pipeline/RatisPipelineUtils.java      | 16 ++-------
 .../scm/pipeline/MockRatisPipelineProvider.java    |  4 ---
 .../scm/pipeline/TestPipelinePlacementPolicy.java  |  7 ++--
 .../scm/pipeline/TestRatisPipelineProvider.java    | 36 ++-----------------
 .../hdds/scm/pipeline/TestSCMPipelineManager.java  | 17 ++++-----
 .../hdds/scm/cli/datanode/ListInfoSubcommand.java  |  4 +--
 .../compose/ozone-topology/docker-compose.yaml     | 28 +++++++++++++++
 11 files changed, 84 insertions(+), 121 deletions(-)

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 6849494..5a28e42 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -21,11 +21,13 @@ package org.apache.hadoop.hdds.scm.pipeline;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
@@ -59,8 +61,6 @@ public final class Pipeline {
   private UUID leaderId;
   // Timestamp for pipeline upon creation
   private Instant creationTimestamp;
-  // Only valid for Ratis THREE pipeline. No need persist.
-  private int nodeIdsHash;
 
   /**
    * The immutable properties of pipeline object is used in
@@ -76,7 +76,6 @@ public final class Pipeline {
     this.state = state;
     this.nodeStatus = nodeStatus;
     this.creationTimestamp = Instant.now();
-    this.nodeIdsHash = 0;
   }
 
   /**
@@ -133,14 +132,6 @@ public final class Pipeline {
     this.creationTimestamp = creationTimestamp;
   }
 
-  public int getNodeIdsHash() {
-    return nodeIdsHash;
-  }
-
-  void setNodeIdsHash(int nodeIdsHash) {
-    this.nodeIdsHash = nodeIdsHash;
-  }
-
   /**
    * Return the pipeline leader's UUID.
    *
@@ -167,6 +158,23 @@ public final class Pipeline {
   }
 
   /**
+   * Return an immutable set of nodes which form this pipeline.
+   * @return Set of DatanodeDetails
+   */
+  public Set<DatanodeDetails> getNodeSet() {
+    return Collections.unmodifiableSet(nodeStatus.keySet());
+  }
+
+  /**
+   * Check if the input pipeline share the same set of datanodes.
+   * @param pipeline
+   * @return true if the input pipeline shares the same set of datanodes.
+   */
+  public boolean sameDatanodes(Pipeline pipeline) {
+    return getNodeSet().equals(pipeline.getNodeSet());
+  }
+
+  /**
    * Returns the leader if found else defaults to closest node.
    *
    * @return {@link DatanodeDetails}
@@ -360,7 +368,6 @@ public final class Pipeline {
     private List<DatanodeDetails> nodesInOrder = null;
     private UUID leaderId = null;
     private Instant creationTimestamp = null;
-    private int nodeIdsHash = 0;
 
     public Builder() {}
 
@@ -373,7 +380,6 @@ public final class Pipeline {
       this.nodesInOrder = pipeline.nodesInOrder.get();
       this.leaderId = pipeline.getLeaderId();
       this.creationTimestamp = pipeline.getCreationTimestamp();
-      this.nodeIdsHash = 0;
     }
 
     public Builder setId(PipelineID id1) {
@@ -417,11 +423,6 @@ public final class Pipeline {
       return this;
     }
 
-    public Builder setNodeIdsHash(int nodeIdsHash1) {
-      this.nodeIdsHash = nodeIdsHash1;
-      return this;
-    }
-
     public Pipeline build() {
       Preconditions.checkNotNull(id);
       Preconditions.checkNotNull(type);
@@ -430,7 +431,6 @@ public final class Pipeline {
       Preconditions.checkNotNull(nodeStatus);
       Pipeline pipeline = new Pipeline(id, type, factor, state, nodeStatus);
       pipeline.setLeaderId(leaderId);
-      pipeline.setNodeIdsHash(nodeIdsHash);
       // overwrite with original creationTimestamp
       if (creationTimestamp != null) {
         pipeline.setCreationTimestamp(creationTimestamp);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index 4261a87..9d78063 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -145,12 +145,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy
{
     String msg;
 
     if (initialHealthyNodesCount < nodesRequired) {
-      LOG.warn("Not enough healthy nodes to allocate pipeline." +
-              nodesRequired + " datanodes required. Found: " +
-          initialHealthyNodesCount);
       msg = String.format("Pipeline creation failed due to no sufficient" +
               " healthy datanodes. Required %d. Found %d.",
           nodesRequired, initialHealthyNodesCount);
+      LOG.warn(msg);
       throw new SCMException(msg,
           SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
     }
@@ -229,42 +227,49 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy
{
     // First choose an anchor nodes randomly
     DatanodeDetails anchor = chooseNode(healthyNodes);
     if (anchor == null) {
-      LOG.warn("Unable to find healthy nodes." +
+      LOG.warn("Unable to find healthy node for anchor(first) node." +
               " Required nodes: {}, Found nodes: {}",
           nodesRequired, results.size());
       throw new SCMException("Unable to find required number of nodes.",
           SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("First node chosen: {}", anchor);
+    }
 
     results.add(anchor);
     exclude.add(anchor);
-    nodesRequired--;
 
     // Choose the second node on different racks from anchor.
     DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
         healthyNodes, exclude,
         nodeManager.getClusterNetworkTopologyMap(), anchor);
     if (nodeOnDifferentRack == null) {
-      LOG.warn("Pipeline Placement: Unable to find nodes on different racks " +
-              " that meet the criteria. Required nodes: {}, Found nodes: {}",
-          nodesRequired, results.size());
+      LOG.warn("Pipeline Placement: Unable to find 2nd node on different " +
+          "racks that meets the criteria. Required nodes: {}, Found nodes:" +
+          " {}", nodesRequired, results.size());
       throw new SCMException("Unable to find required number of nodes.",
           SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Second node chosen: {}", nodeOnDifferentRack);
+    }
 
     results.add(nodeOnDifferentRack);
     exclude.add(nodeOnDifferentRack);
-    nodesRequired--;
 
     // Then choose nodes close to anchor based on network topology
-    for (int x = 0; x < nodesRequired; x++) {
+    int nodesToFind = nodesRequired - results.size();
+    for (int x = 0; x < nodesToFind; x++) {
       // invoke the choose function defined in the derived classes.
       DatanodeDetails pick = chooseNodeFromNetworkTopology(
           nodeManager.getClusterNetworkTopologyMap(), anchor, exclude);
       if (pick != null) {
         results.add(pick);
-        // exclude the picked node for next time
         exclude.add(pick);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Remaining node chosen: {}", pick);
+        }
       }
     }
 
@@ -306,9 +311,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy
{
       datanodeDetails = firstNodeMetric.isGreater(secondNodeMetric.get())
           ? firstNodeDetails : secondNodeDetails;
     }
-    // the pick is decided and it should be removed from candidates.
     healthyNodes.remove(datanodeDetails);
-
     return datanodeDetails;
   }
 
@@ -331,12 +334,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy
{
     }
 
     for (DatanodeDetails node : healthyNodes) {
-      if (excludedNodes.contains(node)
-          || networkTopology.isSameParent(anchor, node)) {
+      if (excludedNodes.contains(node) ||
+          anchor.getNetworkLocation().equals(node.getNetworkLocation())) {
         continue;
       } else {
-        // the pick is decided and it should be removed from candidates.
-        healthyNodes.remove(node);
         return node;
       }
     }
@@ -374,15 +375,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy
{
     if (excludedNodes != null && excludedNodes.size() != 0) {
       excluded.addAll(excludedNodes);
     }
-    excluded.add(anchor);
 
     Node pick = networkTopology.chooseRandom(
         anchor.getNetworkLocation(), excluded);
     DatanodeDetails pickedNode = (DatanodeDetails) pick;
-    // exclude the picked node for next time
-    if (excludedNodes != null) {
-      excludedNodes.add(pickedNode);
-    }
     return pickedNode;
   }
 }
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
index 051202b..bb56a03 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
@@ -132,13 +132,6 @@ public class PipelineStateManager {
       pipeline = pipelineStateMap
           .updatePipelineState(pipelineId, PipelineState.OPEN);
     }
-    // Amend nodeIdsHash if needed.
-    if (pipeline.getType() == ReplicationType.RATIS &&
-        pipeline.getFactor() == ReplicationFactor.THREE &&
-        pipeline.getNodeIdsHash() == 0) {
-      pipeline.setNodeIdsHash(RatisPipelineUtils
-          .encodeNodeIdsOfFactorThreePipeline(pipeline.getNodes()));
-    }
     return pipeline;
   }
 
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 4865074..13c3b6a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -157,7 +157,6 @@ public class RatisPipelineProvider implements PipelineProvider {
     }
 
     List<DatanodeDetails> dns;
-    int nodeIdHash = 0;
 
     switch(factor) {
     case ONE:
@@ -166,7 +165,6 @@ public class RatisPipelineProvider implements PipelineProvider {
     case THREE:
       dns = placementPolicy.chooseDatanodes(null,
           null, factor.getNumber(), 0);
-      nodeIdHash = RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(dns);
       break;
     default:
       throw new IllegalStateException("Unknown factor: " + factor.name());
@@ -178,7 +176,6 @@ public class RatisPipelineProvider implements PipelineProvider {
         .setType(ReplicationType.RATIS)
         .setFactor(factor)
         .setNodes(dns)
-        .setNodeIdsHash(nodeIdHash)
         .build();
 
     // Send command to datanodes to create pipeline
@@ -199,17 +196,12 @@ public class RatisPipelineProvider implements PipelineProvider {
   @Override
   public Pipeline create(ReplicationFactor factor,
                          List<DatanodeDetails> nodes) {
-    int nodeIdHash = 0;
-    if (factor == ReplicationFactor.THREE) {
-      nodeIdHash = RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes);
-    }
     return Pipeline.newBuilder()
         .setId(PipelineID.randomId())
         .setState(PipelineState.ALLOCATED)
         .setType(ReplicationType.RATIS)
         .setFactor(factor)
         .setNodes(nodes)
-        .setNodeIdsHash(nodeIdHash)
         .build();
   }
 
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
index 7fe1cc1..552ae7d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -96,21 +96,12 @@ public final class RatisPipelineUtils {
     }
   }
 
-  static int encodeNodeIdsOfFactorThreePipeline(List<DatanodeDetails> nodes) {
-    if (nodes.size() != HddsProtos.ReplicationFactor.THREE.getNumber()) {
-      return 0;
-    }
-    return nodes.get(0).getUuid().hashCode() ^
-        nodes.get(1).getUuid().hashCode() ^
-        nodes.get(2).getUuid().hashCode();
-  }
-
   /**
    * Return the list of pipelines who share the same set of datanodes
    * with the input pipeline.
    * @param stateManager PipelineStateManager
    * @param pipeline input pipeline
-   * @return first matched pipeline
+   * @return list of matched pipeline
    */
   static List<Pipeline> checkPipelineContainSameDatanodes(
       PipelineStateManager stateManager, Pipeline pipeline) {
@@ -118,9 +109,8 @@ public final class RatisPipelineUtils {
         HddsProtos.ReplicationType.RATIS,
         HddsProtos.ReplicationFactor.THREE)
         .stream().filter(p -> !p.getId().equals(pipeline.getId()) &&
-            (// For all OPEN or ALLOCATED pipelines
-                p.getPipelineState() != Pipeline.PipelineState.CLOSED &&
-                p.getNodeIdsHash() == pipeline.getNodeIdsHash()))
+            (p.getPipelineState() != Pipeline.PipelineState.CLOSED &&
+                p.sameDatanodes(pipeline)))
         .collect(Collectors.toList());
   }
 }
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
index 3eb146a..ff52470 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
@@ -73,8 +73,6 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
           .setType(initialPipeline.getType())
           .setFactor(factor)
           .setNodes(initialPipeline.getNodes())
-          .setNodeIdsHash(RatisPipelineUtils
-              .encodeNodeIdsOfFactorThreePipeline(initialPipeline.getNodes()))
           .build();
     }
   }
@@ -93,8 +91,6 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
         .setType(HddsProtos.ReplicationType.RATIS)
         .setFactor(factor)
         .setNodes(nodes)
-        .setNodeIdsHash(RatisPipelineUtils
-            .encodeNodeIdsOfFactorThreePipeline(nodes))
         .build();
   }
 }
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 2fff7d9..b9aa9af 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -65,10 +65,10 @@ public class TestPipelinePlacementPolicy {
 
     List<DatanodeDetails> excludedNodes =
         new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+    excludedNodes.add(anchor);
     DatanodeDetails nextNode = placementPolicy.chooseNodeFromNetworkTopology(
         nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes);
-    // excludedNodes should contain nextNode after being chosen.
-    Assert.assertTrue(excludedNodes.contains(nextNode));
+    Assert.assertFalse(excludedNodes.contains(nextNode));
     // nextNode should not be the same as anchor.
     Assert.assertTrue(anchor.getUuid() != nextNode.getUuid());
   }
@@ -83,7 +83,8 @@ public class TestPipelinePlacementPolicy {
     DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
         healthyNodes, new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
         topologyWithDifRacks, anchor);
-    Assert.assertFalse(topologyWithDifRacks.isSameParent(anchor, nextNode));
+    Assert.assertFalse(anchor.getNetworkLocation().equals(
+        nextNode.getNetworkLocation()));
   }
 
   private final static Node[] NODES = new NodeImpl[] {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index a17fc08..86d54b3 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -35,7 +35,6 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static org.apache.commons.collections.CollectionUtils.intersection;
@@ -84,7 +83,7 @@ public class TestRatisPipelineProvider {
         intersection(pipeline.getNodes(), pipeline1.getNodes())
             .size() < factor.getNumber());
     if (pipeline.getFactor() == HddsProtos.ReplicationFactor.THREE) {
-      assertNotEquals(pipeline.getNodeIdsHash(), pipeline1.getNodeIdsHash());
+      assertNotEquals(pipeline.getNodeSet(), pipeline1.getNodeSet());
     }
     stateManager.addPipeline(pipeline1);
     nodeManager.addPipeline(pipeline1);
@@ -105,7 +104,7 @@ public class TestRatisPipelineProvider {
     stateManager.addPipeline(pipeline1);
     // With enough pipeline quote on datanodes, they should not share
     // the same set of datanodes.
-    assertNotEquals(pipeline.getNodeIdsHash(), pipeline1.getNodeIdsHash());
+    assertNotEquals(pipeline.getNodeSet(), pipeline1.getNodeSet());
   }
 
   @Test
@@ -141,33 +140,6 @@ public class TestRatisPipelineProvider {
   }
 
   @Test
-  public void testComputeNodeIdsHash() {
-    int total = HddsProtos.ReplicationFactor.THREE.getNumber();
-    List<DatanodeDetails> nodes1 = new ArrayList<>();
-    for (int i = 0; i < total; i++) {
-      nodes1.add(MockDatanodeDetails.createDatanodeDetails(
-          UUID.fromString("00000-11000-00000-00000-0000" + (i + 1))));
-    }
-
-    Assert.assertEquals(total, nodes1.size());
-    Assert.assertNotEquals(0,
-        RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes1));
-
-    List<DatanodeDetails> nodes2 = new ArrayList<>();
-    for (int i = 0; i < total; i++) {
-      nodes2.add(MockDatanodeDetails.createDatanodeDetails(
-          UUID.fromString("00000-11000-00000-00000-0000" + (total - i))));
-    }
-    Assert.assertEquals(total, nodes2.size());
-    Assert.assertNotEquals(0,
-        RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes2));
-
-    Assert.assertEquals(
-        RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes1),
-        RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes2));
-  }
-
-  @Test
   public void testCreateFactorTHREEPipelineWithSameDatanodes() {
     List<DatanodeDetails> healthyNodes = nodeManager
         .getNodes(HddsProtos.NodeState.HEALTHY).stream()
@@ -178,9 +150,7 @@ public class TestRatisPipelineProvider {
     Pipeline pipeline2 = provider.create(
         HddsProtos.ReplicationFactor.THREE, healthyNodes);
 
-    Assert.assertTrue(pipeline1.getNodes().parallelStream()
-        .allMatch(pipeline2.getNodes()::contains));
-    Assert.assertEquals(pipeline1.getNodeIdsHash(), pipeline2.getNodeIdsHash());
+    Assert.assertEquals(pipeline1.getNodeSet(), pipeline2.getNodeSet());
   }
 
   @Test
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index deba91b..ab23153 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hdds.scm.pipeline;
 
-import static org.apache.commons.collections.CollectionUtils.intersection;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
 import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
@@ -116,15 +115,13 @@ public class TestSCMPipelineManager {
     List<Pipeline> pipelineList =
         pipelineManager.getPipelines(HddsProtos.ReplicationType.RATIS);
     Assert.assertEquals(pipelines, new HashSet<>(pipelineList));
-    // All NodeIdsHash from original pipeline list
-    List<Integer> originalPipelineHash = pipelineList.stream()
-        .map(Pipeline::getNodeIdsHash).collect(Collectors.toList());
-    // All NodeIdsHash from reloaded pipeline list
-    List<Integer> reloadedPipelineHash = pipelines.stream()
-        .map(Pipeline::getNodeIdsHash).collect(Collectors.toList());
-    // Original NodeIdsHash list should contain same items from reloaded one.
-    Assert.assertEquals(pipelineNum,
-        intersection(originalPipelineHash, reloadedPipelineHash).size());
+
+    Set<Set<DatanodeDetails>> originalPipelines = pipelineList.stream()
+        .map(Pipeline::getNodeSet).collect(Collectors.toSet());
+    Set<Set<DatanodeDetails>> reloadedPipelineHash = pipelines.stream()
+        .map(Pipeline::getNodeSet).collect(Collectors.toSet());
+    Assert.assertEquals(reloadedPipelineHash, originalPipelines);
+    Assert.assertEquals(pipelineNum, originalPipelines.size());
 
     // clean up
     for (Pipeline pipeline : pipelines) {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
index badfadc..e4060b3 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
@@ -114,8 +114,8 @@ public class ListInfoSubcommand implements Callable<Void> {
       pipelineListInfo.append("No pipelines in cluster.");
     }
     System.out.println("Datanode: " + datanode.getUuid().toString() +
-        " (" + datanode.getIpAddress() + "/"
-        + datanode.getHostName() + "/" + relatedPipelineNum +
+        " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
+        + "/" + datanode.getHostName() + "/" + relatedPipelineNum +
         " pipelines) \n" + "Related pipelines: \n" + pipelineListInfo);
   }
 }
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
index 69611fa..ccd131c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
@@ -72,6 +72,34 @@ services:
       networks:
          net:
             ipv4_address: 10.5.0.7
+   datanode_5:
+     image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+     privileged: true #required by the profiler
+     volumes:
+       - ../..:/opt/hadoop
+     ports:
+       - 9864
+       - 9882
+     command: ["/opt/hadoop/bin/ozone","datanode"]
+     env_file:
+       - ./docker-config
+     networks:
+       net:
+         ipv4_address: 10.5.0.8
+   datanode_6:
+     image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+     privileged: true #required by the profiler
+     volumes:
+       - ../..:/opt/hadoop
+     ports:
+       - 9864
+       - 9882
+     command: ["/opt/hadoop/bin/ozone","datanode"]
+     env_file:
+       - ./docker-config
+     networks:
+       net:
+         ipv4_address: 10.5.0.9
    om:
       image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
       privileged: true #required by the profiler


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message