Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hdfs-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1326012 - in
 /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
 src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java
Date: Fri, 13 Apr 2012 23:30:52 -0000
To: hdfs-commits@hadoop.apache.org
From: atm@apache.org
Message-Id: <20120413233052.8C4B92388962@eris.apache.org>

Author: atm
Date: Fri Apr 13 23:30:52 2012
New Revision: 1326012

URL: http://svn.apache.org/viewvc?rev=1326012&view=rev
Log:
HDFS-2765. TestNameEditsConfigs is incorrectly swallowing IOE. Contributed by Aaron T. Myers.

Modified:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1326012&r1=1326011&r2=1326012&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Apr 13 23:30:52 2012
@@ -391,6 +391,8 @@ Release 2.0.0 - UNRELEASED
 
     HDFS-2799. Trim fs.checkpoint.dir values. (Amith D K via eli)
 
+    HDFS-2765. TestNameEditsConfigs is incorrectly swallowing IOE. (atm)
+
   BREAKDOWN OF HDFS-1623 SUBTASKS
 
     HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java?rev=1326012&r1=1326011&r2=1326012&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java Fri Apr 13 23:30:52 2012
@@ -17,20 +17,30 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import junit.framework.TestCase;
-import java.io.*;
-import java.util.Random;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
 import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import org.junit.Before;
+import org.junit.Test;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -39,7 +49,10 @@ import com.google.common.collect.Immutab
  * This class tests various combinations of dfs.namenode.name.dir 
  * and dfs.namenode.edits.dir configurations.
  */
-public class TestNameEditsConfigs extends TestCase {
+public class TestNameEditsConfigs {
+  
+  private static final Log LOG = LogFactory.getLog(FSEditLog.class);
+  
   static final long SEED = 0xDEADBEEFL;
   static final int BLOCK_SIZE = 4096;
   static final int FILE_SIZE = 8192;
@@ -51,15 +64,15 @@ public class TestNameEditsConfigs extend
   private File base_dir = new File(
       System.getProperty("test.build.data", "build/test/data"), "dfs/");
 
-  protected void setUp() throws java.lang.Exception {
-    if(base_dir.exists()) {
-      if (!FileUtil.fullyDelete(base_dir)) 
-        throw new IOException("Cannot remove directory " + base_dir);
+  @Before
+  public void setUp() throws IOException {
+    if(base_dir.exists() && !FileUtil.fullyDelete(base_dir)) {
+      throw new IOException("Cannot remove directory " + base_dir);
     }
   }
 
   private void writeFile(FileSystem fileSys, Path name, int repl)
-    throws IOException {
+      throws IOException {
     FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
         .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
         (short) repl, BLOCK_SIZE);
@@ -73,7 +86,7 @@ public class TestNameEditsConfigs extend
   void checkImageAndEditsFilesExistence(File dir, 
                                         boolean shouldHaveImages,
                                         boolean shouldHaveEdits)
-  throws IOException {
+      throws IOException {
     FSImageTransactionalStorageInspector ins = inspect(dir);
 
     if (shouldHaveImages) {
@@ -92,7 +105,7 @@ public class TestNameEditsConfigs extend
   }
 
   private void checkFile(FileSystem fileSys, Path name, int repl)
-    throws IOException {
+      throws IOException {
     assertTrue(fileSys.exists(name));
     int replication = fileSys.getFileStatus(name).getReplication();
     assertEquals("replication for " + name, repl, replication);
@@ -101,7 +114,7 @@ public class TestNameEditsConfigs extend
   }
 
   private void cleanupFile(FileSystem fileSys, Path name)
-    throws IOException {
+      throws IOException {
     assertTrue(fileSys.exists(name));
     fileSys.delete(name, true);
     assertTrue(!fileSys.exists(name));
@@ -126,6 +139,7 @@ public class TestNameEditsConfigs extend
    * sure we are reading proper edits and image.
    * @throws Exception 
    */
+  @Test
   public void testNameEditsConfigs() throws Exception {
     Path file1 = new Path("TestNameEditsConfigs1");
     Path file2 = new Path("TestNameEditsConfigs2");
@@ -310,12 +324,14 @@ public class TestNameEditsConfigs extend
    * This test tries to simulate failure scenarios.
    * 1. Start cluster with shared name and edits dir
    * 2. Restart cluster by adding separate name and edits dirs
-   * T3. Restart cluster by removing shared name and edits dir
+   * 3. Restart cluster by removing shared name and edits dir
    * 4. Restart cluster with old shared name and edits dir, but only latest 
-   *    name dir. This should fail since we dont have latest edits dir
+   *    name dir. This should fail since we don't have latest edits dir
    * 5. Restart cluster with old shared name and edits dir, but only latest
-   *    edits dir. This should fail since we dont have latest name dir
+   *    edits dir. This should succeed since the latest edits will have
+   *    segments leading all the way from the image in name_and_edits.
    */
+  @Test
   public void testNameEditsConfigsFailure() throws IOException {
     Path file1 = new Path("TestNameEditsConfigs1");
     Path file2 = new Path("TestNameEditsConfigs2");
@@ -323,28 +339,30 @@ public class TestNameEditsConfigs extend
     MiniDFSCluster cluster = null;
     Configuration conf = null;
     FileSystem fileSys = null;
-    File newNameDir = new File(base_dir, "name");
-    File newEditsDir = new File(base_dir, "edits");
-    File nameAndEdits = new File(base_dir, "name_and_edits");
+    File nameOnlyDir = new File(base_dir, "name");
+    File editsOnlyDir = new File(base_dir, "edits");
+    File nameAndEditsDir = new File(base_dir, "name_and_edits");
     
+    // 1
     // Start namenode with same dfs.namenode.name.dir and dfs.namenode.edits.dir
     conf = new HdfsConfiguration();
-    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameAndEdits.getPath());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEdits.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameAndEditsDir.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEditsDir.getPath());
     replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
-    // Manage our own dfs directories
-    cluster = new MiniDFSCluster.Builder(conf)
-                                .numDataNodes(NUM_DATA_NODES)
-                                .manageNameDfsDirs(false)
-                                .build();
-    cluster.waitActive();
-    
-    // Check that the dir has a VERSION file
-    assertTrue(new File(nameAndEdits, "current/VERSION").exists());
     
-    fileSys = cluster.getFileSystem();
-
     try {
+      // Manage our own dfs directories
+      cluster = new MiniDFSCluster.Builder(conf)
+                                  .numDataNodes(NUM_DATA_NODES)
+                                  .manageNameDfsDirs(false)
+                                  .build();
+      cluster.waitActive();
+      
+      // Check that the dir has a VERSION file
+      assertTrue(new File(nameAndEditsDir, "current/VERSION").exists());
+      
+      fileSys = cluster.getFileSystem();
+
       assertTrue(!fileSys.exists(file1));
       writeFile(fileSys, file1, replication);
       checkFile(fileSys, file1, replication);
@@ -353,32 +371,34 @@ public class TestNameEditsConfigs extend
       cluster.shutdown();
     }
 
+    // 2
     // Start namenode with additional dfs.namenode.name.dir and dfs.namenode.edits.dir
     conf =  new HdfsConfiguration();
-    assertTrue(newNameDir.mkdir());
-    assertTrue(newEditsDir.mkdir());
+    assertTrue(nameOnlyDir.mkdir());
+    assertTrue(editsOnlyDir.mkdir());
 
-    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameAndEdits.getPath() +
-              "," + newNameDir.getPath());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEdits.getPath() +
-              "," + newEditsDir.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameAndEditsDir.getPath() +
+              "," + nameOnlyDir.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEditsDir.getPath() +
+              "," + editsOnlyDir.getPath());
     replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
-    // Manage our own dfs directories. Do not format.
-    cluster = new MiniDFSCluster.Builder(conf)
-                                .numDataNodes(NUM_DATA_NODES)
-                                .format(false)
-                                .manageNameDfsDirs(false)
-                                .build();
-    cluster.waitActive();
-
-    // Check that the dirs have a VERSION file
-    assertTrue(new File(nameAndEdits, "current/VERSION").exists());
-    assertTrue(new File(newNameDir, "current/VERSION").exists());
-    assertTrue(new File(newEditsDir, "current/VERSION").exists());
-
-    fileSys = cluster.getFileSystem();
-
+    
     try {
+      // Manage our own dfs directories. Do not format.
+      cluster = new MiniDFSCluster.Builder(conf)
+                                  .numDataNodes(NUM_DATA_NODES)
+                                  .format(false)
+                                  .manageNameDfsDirs(false)
+                                  .build();
+      cluster.waitActive();
+  
+      // Check that the dirs have a VERSION file
+      assertTrue(new File(nameAndEditsDir, "current/VERSION").exists());
+      assertTrue(new File(nameOnlyDir, "current/VERSION").exists());
+      assertTrue(new File(editsOnlyDir, "current/VERSION").exists());
+  
+      fileSys = cluster.getFileSystem();
+
       assertTrue(fileSys.exists(file1));
       checkFile(fileSys, file1, replication);
       cleanupFile(fileSys, file1);
@@ -389,22 +409,23 @@ public class TestNameEditsConfigs extend
       cluster.shutdown();
     }
     
+    // 3
     // Now remove common directory both have and start namenode with 
     // separate name and edits dirs
-    conf =  new HdfsConfiguration();
-    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, newNameDir.getPath());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, newEditsDir.getPath());
-    replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
-    cluster = new MiniDFSCluster.Builder(conf)
-                                .numDataNodes(NUM_DATA_NODES)
-                                .format(false)
-                                .manageNameDfsDirs(false)
-                                .build();
-    cluster.waitActive();
-    fileSys = cluster.getFileSystem();
-
     try {
-      assertTrue(!fileSys.exists(file1));
+      conf =  new HdfsConfiguration();
+      conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameOnlyDir.getPath());
+      conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsOnlyDir.getPath());
+      replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
+      cluster = new MiniDFSCluster.Builder(conf)
+                                  .numDataNodes(NUM_DATA_NODES)
+                                  .format(false)
+                                  .manageNameDfsDirs(false)
+                                  .build();
+      cluster.waitActive();
+      fileSys = cluster.getFileSystem();
+
+      assertFalse(fileSys.exists(file1));
       assertTrue(fileSys.exists(file2));
       checkFile(fileSys, file2, replication);
       cleanupFile(fileSys, file2);
@@ -415,11 +436,12 @@ public class TestNameEditsConfigs extend
       cluster.shutdown();
     }
     
+    // 4
     // Add old shared directory for name and edits along with latest name
     conf = new HdfsConfiguration();
-    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, newNameDir.getPath() + "," + 
-             nameAndEdits.getPath());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEdits.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameOnlyDir.getPath() + "," + 
+             nameAndEditsDir.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameAndEditsDir.getPath());
     replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
     try {
       cluster = new MiniDFSCluster.Builder(conf)
@@ -427,21 +449,25 @@ public class TestNameEditsConfigs extend
                                   .format(false)
                                   .manageNameDfsDirs(false)
                                   .build();
-      assertTrue(false);
+      fail("Successfully started cluster but should not have been able to.");
     } catch (IOException e) { // expect to fail
-      System.out.println("cluster start failed due to missing " +
-                         "latest edits dir");
+      LOG.info("EXPECTED: cluster start failed due to missing " +
+                         "latest edits dir", e);
     } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
       cluster = null;
     }
 
+    // 5
     // Add old shared directory for name and edits along with latest edits. 
     // This is OK, since the latest edits will have segments leading all
     // the way from the image in name_and_edits.
     conf = new HdfsConfiguration();
-    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameAndEdits.getPath());
-    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, newEditsDir.getPath() +
-             "," + nameAndEdits.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameAndEditsDir.getPath());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsOnlyDir.getPath() +
+             "," + nameAndEditsDir.getPath());
     replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
     try {
       cluster = new MiniDFSCluster.Builder(conf)
@@ -449,14 +475,16 @@ public class TestNameEditsConfigs extend
                                   .format(false)
                                   .manageNameDfsDirs(false)
                                   .build();
-      assertTrue(!fileSys.exists(file1));
-      assertTrue(fileSys.exists(file2));
-      checkFile(fileSys, file2, replication);
-      cleanupFile(fileSys, file2);
+      
+      fileSys = cluster.getFileSystem();
+      
+      assertFalse(fileSys.exists(file1));
+      assertFalse(fileSys.exists(file2));
+      assertTrue(fileSys.exists(file3));
+      checkFile(fileSys, file3, replication);
+      cleanupFile(fileSys, file3);
       writeFile(fileSys, file3, replication);
       checkFile(fileSys, file3, replication);
-    } catch (IOException e) { // expect to fail
-      System.out.println("cluster start failed due to missing latest name dir");
     } finally {
       fileSys.close();
       cluster.shutdown();