Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1574798 - in
 /hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-common/
 hadoop-common/src/ hadoop-common/src/main/java/
 hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/
 hadoop-common/src/test/java/org/apache/hadoop...
Date: Thu, 06 Mar 2014 08:26:26 -0000
To: common-commits@hadoop.apache.org
From: cmccabe@apache.org
Message-Id: <20140306082626.E659B23889EC@eris.apache.org>

Author: cmccabe
Date: Thu Mar  6 08:26:26 2014
New Revision: 1574798

URL: http://svn.apache.org/r1574798
Log:
HDFS-6061. Allow dfs.datanode.shared.file.descriptor.path to contain multiple entries and fall back when needed (cmccabe)

Modified:
    hadoop/common/branches/branch-2/hadoop-common-project/   (props changed)
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/   (props changed)
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/   (props changed)
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/   (props changed)
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java

Propchange: hadoop/common/branches/branch-2/hadoop-common-project/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project:r1574796

Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common:r1574796

Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src:r1574796

Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1574796

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java?rev=1574798&r1=1574797&r2=1574798&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java Thu Mar  6 08:26:26 2014
@@ -22,11 +22,11 @@ import java.io.IOException;
 import java.io.FileDescriptor;
 
 import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
-import com.google.common.base.Preconditions;
-
 /**
  * A factory for creating shared file descriptors inside a given directory.
  * Typically, the directory will be /dev/shm or /tmp.
@@ -45,6 +45,7 @@ import com.google.common.base.Preconditi
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class SharedFileDescriptorFactory {
+  public static final Log LOG = LogFactory.getLog(SharedFileDescriptorFactory.class);
   private final String prefix;
   private final String path;
 
@@ -59,17 +60,57 @@ public class SharedFileDescriptorFactory
   }
 
   /**
+   * Create a new SharedFileDescriptorFactory.
+   *
+   * @param prefix       The prefix to prepend to all the file names created
+   *                       by this factory.
+   * @param paths        An array of paths to use.  We will try each path in 
+   *                       succession, and return a factory using the first 
+   *                       usable path.
+   * @return             The factory.
+   * @throws IOException If a factory could not be created for any reason.
+   */
+  public static SharedFileDescriptorFactory create(String prefix,
+      String paths[]) throws IOException {
+    String loadingFailureReason = getLoadingFailureReason();
+    if (loadingFailureReason != null) {
+      throw new IOException(loadingFailureReason);
+    }
+    if (paths.length == 0) {
+      throw new IOException("no SharedFileDescriptorFactory paths were " +
+          "configured.");
+    }
+    StringBuilder errors = new StringBuilder();
+    String strPrefix = "";
+    for (String path : paths) {
+      try {
+        FileInputStream fis = 
+            new FileInputStream(createDescriptor0(prefix + "test", path, 1));
+        fis.close();
+        deleteStaleTemporaryFiles0(prefix, path);
+        return new SharedFileDescriptorFactory(prefix, path);
+      } catch (IOException e) {
+        errors.append(strPrefix).append("Error creating file descriptor in ").
+               append(path).append(": ").append(e.getMessage());
+        strPrefix = ", ";
+      }
+    }
+    throw new IOException(errors.toString());
+  }
+
+  /**
    * Create a SharedFileDescriptorFactory.
    *
    * @param prefix    Prefix to add to all file names we use.
    * @param path      Path to use.
    */
-  public SharedFileDescriptorFactory(String prefix, String path)
-      throws IOException {
-    Preconditions.checkState(getLoadingFailureReason() == null);
+  private SharedFileDescriptorFactory(String prefix, String path) {
     this.prefix = prefix;
     this.path = path;
-    deleteStaleTemporaryFiles0(prefix, path);
+  }
+
+  public String getPath() {
+    return path;
   }
 
   /**

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java?rev=1574798&r1=1574797&r2=1574798&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java Thu Mar  6 08:26:26 2014
@@ -20,9 +20,11 @@ package org.apache.hadoop.io.nativeio;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
 
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.Before;
 import org.junit.Test;
 import org.apache.commons.lang.SystemUtils;
 import org.apache.commons.logging.Log;
@@ -36,14 +38,19 @@ public class TestSharedFileDescriptorFac
   private static final File TEST_BASE =
       new File(System.getProperty("test.build.data", "/tmp"));
 
+  @Before
+  public void setup() throws Exception {
+    Assume.assumeTrue(null ==
+        SharedFileDescriptorFactory.getLoadingFailureReason());
+  }
+
   @Test(timeout=10000)
   public void testReadAndWrite() throws Exception {
-    Assume.assumeTrue(NativeIO.isAvailable());
-    Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
     File path = new File(TEST_BASE, "testReadAndWrite");
     path.mkdirs();
     SharedFileDescriptorFactory factory =
-        new SharedFileDescriptorFactory("woot_", path.getAbsolutePath());
+        SharedFileDescriptorFactory.create("woot_",
+            new String[] { path.getAbsolutePath() });
     FileInputStream inStream =
         factory.createDescriptor("testReadAndWrite", 4096);
     FileOutputStream outStream = new FileOutputStream(inStream.getFD());
@@ -73,11 +80,34 @@ public class TestSharedFileDescriptorFac
         Path.SEPARATOR + "woot2_remainder2";
     createTempFile(remainder1);
     createTempFile(remainder2);
-    new SharedFileDescriptorFactory("woot2_", path.getAbsolutePath());
+    SharedFileDescriptorFactory.create("woot2_", 
+        new String[] { path.getAbsolutePath() });
     // creating the SharedFileDescriptorFactory should have removed 
     // the remainders
     Assert.assertFalse(new File(remainder1).exists());
     Assert.assertFalse(new File(remainder2).exists());
     FileUtil.fullyDelete(path);
   }
+  
+  @Test(timeout=60000)
+  public void testDirectoryFallbacks() throws Exception {
+    File nonExistentPath = new File(TEST_BASE, "nonexistent");
+    File permissionDeniedPath = new File("/");
+    File goodPath = new File(TEST_BASE, "testDirectoryFallbacks");
+    goodPath.mkdirs();
+    try {
+      SharedFileDescriptorFactory.create("shm_", 
+          new String[] { nonExistentPath.getAbsolutePath(),
+                          permissionDeniedPath.getAbsolutePath() });
+      Assert.fail();
+    } catch (IOException e) {
+    }
+    SharedFileDescriptorFactory factory =
+        SharedFileDescriptorFactory.create("shm_", 
+            new String[] { nonExistentPath.getAbsolutePath(),
+                            permissionDeniedPath.getAbsolutePath(),
+                            goodPath.getAbsolutePath() } );
+    Assert.assertEquals(goodPath.getAbsolutePath(), factory.getPath());
+    FileUtil.fullyDelete(goodPath);
+  }
 }