hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r472652 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/fs/ src/test/org/apache/hadoop/mapred/
Date Wed, 08 Nov 2006 21:34:52 GMT
Author: cutting
Date: Wed Nov  8 13:34:51 2006
New Revision: 472652

URL: http://svn.apache.org/viewvc?view=rev&rev=472652
Log:
HADOOP-382.  Extend unit tests to run multiple datanodes.  Contributed by Milind.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSMkdirs.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Nov  8 13:34:51 2006
@@ -22,6 +22,9 @@
  6. HADOOP-683.  Remove a script dependency on bash, so it works with
     dash, the new default for /bin/sh on Ubuntu.  (James Todd via cutting)
 
+ 7. HADOOP-382.  Extend unit tests to run multiple datanodes.
+    (Milind Bhandarkar via cutting)
+
 
 Release 0.8.0 - 2006-11-03
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Wed Nov  8 13:34:51
2006
@@ -1996,17 +1996,17 @@
         //
         // Build a map of forbidden hostnames from the two forbidden sets.
         //
-        Collection<String> forbiddenMachines = new TreeSet<String>();
+        Collection<DatanodeDescriptor> forbiddenMachines = new TreeSet();
         if (forbidden1 != null) {
             for (Iterator<DatanodeDescriptor> it = forbidden1.iterator(); it.hasNext();
) {
                 DatanodeDescriptor cur = it.next();
-                forbiddenMachines.add(cur.getHost());
+                forbiddenMachines.add(cur);
             }
         }
         if (forbidden2 != null) {
             for (Iterator<DatanodeDescriptor> it = forbidden2.iterator(); it.hasNext();
) {
                 DatanodeDescriptor cur = it.next();
-                forbiddenMachines.add(cur.getHost());
+                forbiddenMachines.add(cur);
             }
         }
 
@@ -2017,7 +2017,7 @@
         List<DatanodeDescriptor> targetList = new ArrayList<DatanodeDescriptor>();
         for (Iterator<DatanodeDescriptor> it = heartbeats.iterator(); it.hasNext();
) {
             DatanodeDescriptor node = it.next();
-            if (! forbiddenMachines.contains(node.getHost())) {
+            if (! forbiddenMachines.contains(node)) {
                 targetList.add(node);
                 avgLoad += node.getXceiverCount();
             }

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java Wed Nov  8 13:34:51
2006
@@ -30,10 +30,11 @@
 public class MiniDFSCluster {
 
   private Configuration conf;
+  int nDatanodes;
   private Thread nameNodeThread;
-  private Thread dataNodeThread;
+  private Thread dataNodeThreads[];
   private NameNodeRunner nameNode;
-  private DataNodeRunner dataNode;
+  private DataNodeRunner dataNodes[];
   private int maxRetries = 10;
   private int MAX_RETRIES  = 10;
   private int MAX_RETRIES_PER_PORT = 10;
@@ -88,6 +89,15 @@
    */
   class DataNodeRunner implements Runnable {
     private DataNode node;
+    Configuration conf = null;
+    
+    public DataNodeRunner(Configuration conf, File dataDir, int index) {
+      this.conf = new Configuration(conf);
+      this.conf.set("dfs.data.dir",
+          new File(dataDir, "data"+(2*index+1)).getPath()+","+
+          new File(dataDir, "data"+(2*index+2)).getPath());
+    
+    }
     
     /**
      * Create and run the data node.
@@ -127,24 +137,39 @@
    * Create the config and start up the servers.  If either the rpc or info port is already

    * in use, we will try new ports.
    * @param namenodePort suggestion for which rpc port to use.  caller should use 
-   *                     getNameNodePort() to get the actual port used.   
+   *                     getNameNodePort() to get the actual port used.
    * @param dataNodeFirst should the datanode be brought up before the namenode?
    */
   public MiniDFSCluster(int namenodePort, 
                         Configuration conf,
                         boolean dataNodeFirst) throws IOException {
+    this(namenodePort, conf, 1, dataNodeFirst);
+  }
+  
+  /**
+   * Create the config and start up the servers.  If either the rpc or info port is already

+   * in use, we will try new ports.
+   * @param namenodePort suggestion for which rpc port to use.  caller should use 
+   *                     getNameNodePort() to get the actual port used.
+   * @param nDatanodes Number of datanodes   
+   * @param dataNodeFirst should the datanode be brought up before the namenode?
+   */
+  public MiniDFSCluster(int namenodePort, 
+                        Configuration conf,
+                        int nDatanodes,
+                        boolean dataNodeFirst) throws IOException {
 
     this.conf = conf;
 
+    this.nDatanodes = nDatanodes;
     this.nameNodePort = namenodePort;
     this.nameNodeInfoPort = 50080;   // We just want this port to be different from the default.

     File base_dir = new File(System.getProperty("test.build.data"),
                              "dfs/");
+    File data_dir = new File(base_dir, "data");
     conf.set("dfs.name.dir", new File(base_dir, "name1").getPath()+","+
         new File(base_dir, "name2").getPath());
-    conf.set("dfs.data.dir", new File(base_dir, "data1").getPath()+","+
-        new File(base_dir, "data2").getPath());
-    conf.setInt("dfs.replication", 1);
+    conf.setInt("dfs.replication", Math.min(3, nDatanodes));
     // this timeout seems to control the minimum time for the test, so
     // decrease it considerably.
     conf.setInt("ipc.client.timeout", 1000);
@@ -161,14 +186,22 @@
       NameNode.format(conf);
       nameNode = new NameNodeRunner();
       nameNodeThread = new Thread(nameNode);
-      dataNode = new DataNodeRunner();
-      dataNodeThread = new Thread(dataNode);
+      dataNodes = new DataNodeRunner[nDatanodes];
+      dataNodeThreads = new Thread[nDatanodes];
+      for (int idx = 0; idx < nDatanodes; idx++) {
+        dataNodes[idx] = new DataNodeRunner(conf, data_dir, idx);
+        dataNodeThreads[idx] = new Thread(dataNodes[idx]);
+      }
       if (dataNodeFirst) {
-        dataNodeThread.start();      
+        for (int idx = 0; idx < nDatanodes; idx++) {
+          dataNodeThreads[idx].start();
+        }
         nameNodeThread.start();      
       } else {
         nameNodeThread.start();
-        dataNodeThread.start();      
+        for (int idx = 0; idx < nDatanodes; idx++) {
+          dataNodeThreads[idx].start();
+        }
       }
 
       int retry = 0;
@@ -188,7 +221,9 @@
         System.out.println("\tNameNode info port: " + nameNodeInfoPort);
 
         nameNode.shutdown();
-        dataNode.shutdown();
+        for (int idx = 0; idx < nDatanodes; idx++) {
+          dataNodes[idx].shutdown();
+        }
         
       } else {
         foundPorts = true;
@@ -212,7 +247,9 @@
    * Shut down the servers.
    */
   public void shutdown() {
-    dataNode.shutdown();
+    for (int idx = 0; idx < nDatanodes; idx++) {
+      dataNodes[idx].shutdown();
+    }
     nameNode.shutdown();
   }
   

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSMkdirs.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSMkdirs.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSMkdirs.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSMkdirs.java Wed Nov  8 13:34:51
2006
@@ -43,7 +43,7 @@
    */
   public void testDFSMkdirs() throws IOException {
     Configuration conf = new Configuration();
-    MiniDFSCluster cluster = new MiniDFSCluster(65312, conf, false);
+    MiniDFSCluster cluster = new MiniDFSCluster(65312, conf, 2, false);
     FileSystem fileSys = cluster.getFileSystem();
     try {
     	// First create a new directory with mkdirs

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java Wed Nov  8 13:34:51 2006
@@ -139,7 +139,7 @@
     MiniDFSCluster cluster = null;
     try {
       Configuration conf = new Configuration();
-      cluster = new MiniDFSCluster(65314, conf, false);
+      cluster = new MiniDFSCluster(65314, conf, 4, false);
       namenode = conf.get("fs.default.name", "local");
       if (!"local".equals(namenode)) {
         MyFile[] files = createFiles(namenode, "/srcdat");

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestPread.java Wed Nov  8 13:34:51
2006
@@ -111,7 +111,7 @@
    */
   public void testPreadDFS() throws IOException {
     Configuration conf = new Configuration();
-    MiniDFSCluster cluster = new MiniDFSCluster(65312, conf, false);
+    MiniDFSCluster cluster = new MiniDFSCluster(65312, conf, 3, false);
     FileSystem fileSys = cluster.getFileSystem();
     try {
       Path file1 = new Path("preadtest.dat");

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java Wed Nov  8 13:34:51
2006
@@ -179,7 +179,7 @@
     MiniDFSCluster cluster = null;
     try {
       Configuration conf = new Configuration();
-      cluster = new MiniDFSCluster(65314, conf, false);
+      cluster = new MiniDFSCluster(65314, conf, 2, false);
       namenode = conf.get("fs.default.name", "local");
       if (!"local".equals(namenode)) {
         MyFile[] files = createFiles(namenode, "/srcdat");

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java?view=diff&rev=472652&r1=472651&r2=472652
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java Wed Nov 
8 13:34:51 2006
@@ -150,7 +150,7 @@
           final int jobTrackerPort = 60050;
 
           Configuration conf = new Configuration();
-          dfs = new MiniDFSCluster(65314, conf, true);
+          dfs = new MiniDFSCluster(65314, conf, 4, true);
           fileSys = dfs.getFileSystem();
           namenode = fileSys.getName();
           mr = new MiniMRCluster(jobTrackerPort, 50060, taskTrackers, 



Mime
View raw message