hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject svn commit: r1127995 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/
Date Thu, 26 May 2011 17:34:40 GMT
Author: jitendra
Date: Thu May 26 17:34:39 2011
New Revision: 1127995

URL: http://svn.apache.org/viewvc?rev=1127995&view=rev
Log:
HDFS-1592. Datanode startup doesn't honor volumes.tolerated. Contributed by Bharath Mundlapudi.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1127995&r1=1127994&r2=1127995&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Thu May 26 17:34:39 2011
@@ -626,6 +626,9 @@ Trunk (unreleased changes)
 
     HDFS-1999. Tests use deprecated configs. (Aaron T. Myers via eli)
 
+    HDFS-1592. Datanode startup doesn't honor volumes.tolerated. 
+    (Bharath Mundlapudi via jitendra)
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1127995&r1=1127994&r2=1127995&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Thu May
26 17:34:39 2011
@@ -998,9 +998,14 @@ public class DataNode extends Configured
       if (blockScanner != null) {
         blockScanner.removeBlockPool(this.getBlockPoolId());
       }
-     
-      data.shutdownBlockPool(this.getBlockPoolId());
-      storage.removeBlockPoolStorage(this.getBlockPoolId());
+    
+      if (data != null) { 
+        data.shutdownBlockPool(this.getBlockPoolId());
+      }
+
+      if (storage != null) {
+        storage.removeBlockPoolStorage(this.getBlockPoolId());
+      }
     }
 
     /**

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1127995&r1=1127994&r2=1127995&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Thu
May 26 17:34:39 2011
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.pro
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Daemon;
 import org.apache.hadoop.util.DiskChecker;
+import org.apache.hadoop.util.StringUtils;
 
 /** 
  * Data storage information file.
@@ -170,14 +171,17 @@ public class DataStorage extends Storage
         }
       } catch (IOException ioe) {
         sd.unlock();
-        throw ioe;
+        LOG.warn("Ignoring storage directory "+ dataDir
+        		+ " due to an exception: " + StringUtils.stringifyException(ioe));
+        //continue with other good dirs
+        continue;
       }
       // add to the storage list
       addStorageDir(sd);
       dataDirStates.add(curState);
     }
 
-    if (dataDirs.size() == 0)  // none of the data dirs exist
+    if (dataDirs.size() == 0 || dataDirStates.size() == 0)  // none of the data dirs exist
       throw new IOException(
           "All specified directories are not accessible or do not exist.");
 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=1127995&r1=1127994&r2=1127995&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Thu May
26 17:34:39 2011
@@ -1145,12 +1145,21 @@ public class FSDataset implements FSCons
     final int volFailuresTolerated =
       conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
                   DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT);
-    this.validVolsRequired = storage.getNumStorageDirs() - volFailuresTolerated; 
-    if (validVolsRequired < 1 ||
-        validVolsRequired > storage.getNumStorageDirs()) {
-      DataNode.LOG.error("Invalid value " + volFailuresTolerated + " for " +
-          DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY);
+
+    String[] dataDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+
+    int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length;
+
+    this.validVolsRequired = volsConfigured - volFailuresTolerated;
+
+    if (validVolsRequired < 1
+        || validVolsRequired > storage.getNumStorageDirs()) {
+      throw new DiskErrorException("Too many failed volumes - "
+          + "current valid volumes: " + storage.getNumStorageDirs() 
+          + ", volumes configured: " + volsConfigured 
+          + ", volume failures tolerated: " + volFailuresTolerated );
     }
+
     FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()];
     for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
       volArray[idx] = new FSVolume(storage.getStorageDir(idx).getCurrentDir(),

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java?rev=1127995&r1=1127994&r2=1127995&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
Thu May 26 17:34:39 2011
@@ -18,10 +18,13 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 import java.io.File;
+import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -129,32 +132,91 @@ public class TestDataNodeVolumeFailureTo
     DFSTestUtil.waitReplication(fs, file2, (short)2);
   }
 
+  /** 
+   * Restart the cluster with a new volume tolerated value.
+   * @param volTolerated
+   * @param manageCluster
+   * @throws IOException
+   */
+  private void restartCluster(int volTolerated, boolean manageCluster)
+      throws IOException {
+    //Make sure no datanode is running
+    cluster.shutdownDataNodes();
+    conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, volTolerated);
+    cluster.startDataNodes(conf, 1, manageCluster, null, null);
+    cluster.waitActive();
+  }
+
   /**
-   * Test invalid DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY values.
+   * Test for different combination of volume configs and volumes tolerated 
+   * values.
    */
   @Test
-  public void testInvalidFailedVolumesConfig() throws Exception {
-    assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
-
-    /*
-     * Bring up another datanode that has an invalid value set.
-     * We should still be able to create a file with two replicas
-     * since the minimum valid volume parameter is only checked
-     * when we experience a disk error.
-     */
-    conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, -1);
-    cluster.startDataNodes(conf, 1, true, null, null);
-    cluster.waitActive();
-    Path file1 = new Path("/test1");
-    DFSTestUtil.createFile(fs, file1, 1024, (short)2, 1L);
-    DFSTestUtil.waitReplication(fs, file1, (short)2);
+  public void testVolumeAndTolerableConfiguration() throws Exception {
+    // Check if Block Pool Service exit for an invalid conf value.
+    testVolumeConfig(-1, 0, false, true);
 
     // Ditto if the value is too big.
-    conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 100);
-    cluster.startDataNodes(conf, 1, true, null, null);
-    cluster.waitActive();
-    Path file2 = new Path("/test1");
-    DFSTestUtil.createFile(fs, file2, 1024, (short)2, 1L);
-    DFSTestUtil.waitReplication(fs, file2, (short)2);
+    testVolumeConfig(100, 0, false, true);
+    
+    // Test for one failed volume
+    testVolumeConfig(0, 1, false, false);
+    
+    // Test for one failed volume with 1 tolerable volume
+    testVolumeConfig(1, 1, true, false);
+    
+    // Test all good volumes
+    testVolumeConfig(0, 0, true, false);
+    
+    // Test all failed volumes
+    testVolumeConfig(0, 2, false, false);
   }
+
+  /**
+   * Tests for a given volumes to be tolerated and volumes failed.
+   * 
+   * @param volumesTolerated
+   * @param volumesFailed
+   * @param expectedBPServiceState
+   * @param clusterManaged
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  private void testVolumeConfig(int volumesTolerated, int volumesFailed,
+      boolean expectedBPServiceState, boolean clusterManaged)
+      throws IOException, InterruptedException {
+    assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
+    final int dnIndex = 0;
+    File[] dirs = {
+        new File(MiniDFSCluster.getStorageDir(dnIndex, 0), "current"),
+        new File(MiniDFSCluster.getStorageDir(dnIndex, 1), "current") };
+
+    try {
+      for (int i = 0; i < volumesFailed; i++) {
+        prepareDirToFail(dirs[i]);
+      }
+      restartCluster(volumesTolerated, clusterManaged);
+      assertEquals(expectedBPServiceState, cluster.getDataNodes().get(0)
+          .isBPServiceAlive(cluster.getNamesystem().getBlockPoolId()));
+    } finally {
+      // restore its old permission
+      for (File dir : dirs) {
+        FileUtil.chmod(dir.toString(), "755");
+      }
+    }
+  }
+
+  /** 
+   * Prepare directories for a failure, set dir permission to 000
+   * @param dir
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  private void prepareDirToFail(File dir) throws IOException,
+      InterruptedException {
+    dir.mkdirs();
+    assertTrue("Couldn't chmod local vol", FileUtil
+        .chmod(dir.toString(), "000") == 0);
+  }
+
 }



Mime
View raw message