hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1373178 - in /hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/qjournal/server/ src/test/java/org/apache/hadoop/hdfs/qjournal/ src/test/java/org/apache/hadoop/hdfs/qjournal/server/
Date Wed, 15 Aug 2012 00:52:24 GMT
Author: todd
Date: Wed Aug 15 00:52:24 2012
New Revision: 1373178

URL: http://svn.apache.org/viewvc?rev=1373178&view=rev
Log:
HDFS-3795. QJM: validate journal dir at startup. Contributed by Todd Lipcon.

Modified:
    hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
    hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
    hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
    hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
    hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java

Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
Wed Aug 15 00:52:24 2012
@@ -16,3 +16,5 @@ HDFS-3741. Exhaustive failure injection 
 HDFS-3773. TestNNWithQJM fails after HDFS-3741. (atm)
 
 HDFS-3793. Implement genericized format() in QJM (todd)
+
+HDFS-3795. QJM: validate journal dir at startup (todd)

Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
Wed Aug 15 00:52:24 2012
@@ -55,6 +55,8 @@ public class JournalNode implements Tool
   private JournalNodeHttpServer httpServer;
   private Map<String, Journal> journalsById = Maps.newHashMap();
 
+  private File localDir;
+
   static {
     HdfsConfiguration.init();
   }
@@ -82,8 +84,32 @@ public class JournalNode implements Tool
   @Override
   public void setConf(Configuration conf) {
     this.conf = conf;
+    this.localDir = new File(
+        conf.get(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY,
+        DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_DEFAULT).trim());
+  }
+
+  private static void validateAndCreateJournalDir(File dir) throws IOException {
+    if (!dir.isAbsolute()) {
+      throw new IllegalArgumentException(
+          "Journal dir '" + dir + "' should be an absolute path");
+    }
+
+    if (!dir.exists() && !dir.mkdirs()) {
+      throw new IOException("Could not create journal dir '" +
+          dir + "'");
+    } else if (!dir.isDirectory()) {
+      throw new IOException("Journal directory '" + dir + "' is not " +
+          "a directory");
+    }
+    
+    if (!dir.canWrite()) {
+      throw new IOException("Unable to write to journal dir '" +
+          dir + "'");
+    }
   }
 
+
   @Override
   public Configuration getConf() {
     return conf;
@@ -101,6 +127,8 @@ public class JournalNode implements Tool
   public void start() throws IOException {
     Preconditions.checkState(!isStarted(), "JN already running");
     
+    validateAndCreateJournalDir(localDir);
+    
     DefaultMetricsSystem.initialize("JournalNode");
     JvmMetrics.create("JournalNode",
         conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),

Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
Wed Aug 15 00:52:24 2012
@@ -156,7 +156,7 @@ public class MiniJournalCluster {
   }
 
   public File getStorageDir(int idx) {
-    return new File(baseDir, "journalnode-" + idx);
+    return new File(baseDir, "journalnode-" + idx).getAbsoluteFile();
   }
   
   public File getCurrentDir(int idx, String jid) {

Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
Wed Aug 15 00:52:24 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.qjournal;
 
 import static org.junit.Assert.*;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 
@@ -42,7 +43,9 @@ public class TestMiniJournalCluster {
       
       JournalNode node = c.getJournalNode(0);
       String dir = node.getConf().get(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY);
-      assertEquals(MiniDFSCluster.getBaseDirectory() + "journalnode-0",
+      assertEquals(
+          new File(MiniDFSCluster.getBaseDirectory() + "journalnode-0")
+            .getAbsolutePath(),
           dir);
     } finally {
       c.shutdown();

Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java?rev=1373178&r1=1373177&r2=1373178&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
Wed Aug 15 00:52:24 2012
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
 import java.net.URL;
@@ -234,6 +235,33 @@ public class TestJournalNode {
     }
   }
   
+  @Test
+  public void testFailToStartWithBadConfig() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, "non-absolute-path");
+    assertJNFailsToStart(conf, "should be an absolute path");
+    
+    // Existing file which is not a directory 
+    conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, "/dev/null");
+    assertJNFailsToStart(conf, "is not a directory");
+    
+    // Directory which cannot be created
+    conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, "/proc/does-not-exist");
+    assertJNFailsToStart(conf, "Could not create");
+
+  }
+
+  private static void assertJNFailsToStart(Configuration conf,
+      String errString) {
+    try {
+      JournalNode jn = new JournalNode();
+      jn.setConf(conf);
+      jn.start();
+    } catch (Exception e) {
+      GenericTestUtils.assertExceptionContains(errString, e);
+    }
+  }
+  
   // TODO:
   // - add test that checks formatting behavior
   // - add test that checks rejects newEpoch if nsinfo doesn't match



Mime
View raw message