hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinayakum...@apache.org
Subject hadoop git commit: HDFS-8209. Support different number of datanode directories in MiniDFSCluster. (Contributed by surendra singh lilhore)
Date Fri, 08 May 2015 09:35:23 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5cc036d47 -> eaa5f3e44


HDFS-8209. Support different number of datanode directories in MiniDFSCluster. (Contributed
by surendra singh lilhore)

(cherry picked from commit 4c6816faf83726c71c9c3981e7280b461f824a4f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eaa5f3e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eaa5f3e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eaa5f3e4

Branch: refs/heads/branch-2
Commit: eaa5f3e44801baba0f9cc79399bb72278fe12f14
Parents: 5cc036d
Author: Vinayakumar B <vinayakumarb@apache.org>
Authored: Fri May 8 15:03:44 2015 +0530
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Fri May 8 15:04:58 2015 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  4 +-
 .../apache/hadoop/hdfs/TestMiniDFSCluster.java  | 43 ++++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eaa5f3e4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index caae965..f1eb24d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -357,6 +357,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8175. Provide information on snapshotDiff for supporting the comparison
     between snapshot and current status (J.Andreina via vinayakumarb)
 
+    HDFS-8209. Support different number of datanode directories in MiniDFSCluster.
+    (surendra singh lilhore via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eaa5f3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index fd0ed10..6674dca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1211,8 +1211,10 @@ public class MiniDFSCluster {
 
   String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws IOException {
     StringBuilder sb = new StringBuilder();
-    assert storageTypes == null || storageTypes.length == storagesPerDatanode;
     for (int j = 0; j < storagesPerDatanode; ++j) {
+      if ((storageTypes != null) && (j >= storageTypes.length)) {
+        break;
+      }
       File dir = getInstanceStorageDir(dnIndex, j);
       dir.mkdirs();
       if (!dir.isDirectory()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eaa5f3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index cf29d97..3fa852e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -23,9 +23,13 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -139,4 +143,43 @@ public class TestMiniDFSCluster {
       MiniDFSCluster.shutdownCluster(cluster5);
     }
   }
+
+  @Test
+  public void testClusterSetDatanodeDifferentStorageType() throws IOException {
+    final Configuration conf = new HdfsConfiguration();
+    StorageType[][] storageType = new StorageType[][] {
+        {StorageType.DISK, StorageType.ARCHIVE}, {StorageType.DISK},
+        {StorageType.ARCHIVE}};
+    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(3).storageTypes(storageType).build();
+    try {
+      cluster.waitActive();
+      ArrayList<DataNode> dataNodes = cluster.getDataNodes();
+      // Check the number of directory in DN's
+      for (int i = 0; i < storageType.length; i++) {
+        assertEquals(DataNode.getStorageLocations(dataNodes.get(i).getConf())
+            .size(), storageType[i].length);
+      }
+    } finally {
+      MiniDFSCluster.shutdownCluster(cluster);
+    }
+  }
+
+  @Test
+  public void testClusterNoStorageTypeSetForDatanodes() throws IOException {
+    final Configuration conf = new HdfsConfiguration();
+    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(3).build();
+    try {
+      cluster.waitActive();
+      ArrayList<DataNode> dataNodes = cluster.getDataNodes();
+      // Check the number of directory in DN's
+      for (DataNode datanode : dataNodes) {
+        assertEquals(DataNode.getStorageLocations(datanode.getConf()).size(),
+            2);
+      }
+    } finally {
+      MiniDFSCluster.shutdownCluster(cluster);
+    }
+  }
 }


Mime
View raw message