Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BC3BA10745 for ; Thu, 6 Mar 2014 08:21:47 +0000 (UTC) Received: (qmail 57988 invoked by uid 500); 6 Mar 2014 08:21:46 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 57714 invoked by uid 500); 6 Mar 2014 08:21:44 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 57681 invoked by uid 99); 6 Mar 2014 08:21:42 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Mar 2014 08:21:42 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Mar 2014 08:21:39 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8A1E22388A5E; Thu, 6 Mar 2014 08:21:17 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1574796 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/server/datanode/ src/main/resources/ src/test/java/org/apache/hadoop/hdfs/client/ Date: Thu, 06 Mar 2014 08:21:17 -0000 To: hdfs-commits@hadoop.apache.org From: cmccabe@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140306082117.8A1E22388A5E@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cmccabe Date: Thu Mar 6 08:21:16 2014 New Revision: 1574796 URL: http://svn.apache.org/r1574796 Log: HDFS-6061. Allow dfs.datanode.shared.file.descriptor.path to contain multiple entries and fall back when needed (cmccabe) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1574796&r1=1574795&r2=1574796&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Mar 6 08:21:16 2014 @@ -527,6 +527,9 @@ Release 2.4.0 - UNRELEASED HDFS-6044. Add property for setting the NFS look up time for users (brandonli) + HDFS-6061. Allow dfs.datanode.shared.file.descriptor.path to contain + multiple entries and fall back when needed (cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1574796&r1=1574795&r2=1574796&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Mar 6 08:21:16 2014 @@ -477,8 +477,8 @@ public class DFSConfigKeys extends Commo public static final String DFS_NAMENODE_STARTUP_KEY = "dfs.namenode.startup"; public static final String DFS_DATANODE_KEYTAB_FILE_KEY = "dfs.datanode.keytab.file"; public static final String DFS_DATANODE_USER_NAME_KEY = "dfs.datanode.kerberos.principal"; - public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH = "dfs.datanode.shared.file.descriptor.path"; - public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT = "/dev/shm"; + public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS = "dfs.datanode.shared.file.descriptor.paths"; + public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT = "/dev/shm,/tmp"; public static final String DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS = "dfs.short.circuit.shared.memory.watcher.interrupt.check.ms"; public static final int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT = 60000; public static final String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file"; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java?rev=1574796&r1=1574795&r2=1574796&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java Thu Mar 6 08:21:16 2014 @@ -17,14 +17,15 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT; import java.io.Closeable; import java.io.FileInputStream; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Set; @@ -45,7 +46,9 @@ import org.apache.hadoop.net.unix.Domain import org.apache.hadoop.net.unix.DomainSocketWatcher; import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; import com.google.common.collect.HashMultimap; +import com.google.common.collect.Iterables; /* * Manages client short-circuit memory segments on the DataNode. @@ -149,38 +152,35 @@ public class ShortCircuitRegistry { SharedFileDescriptorFactory shmFactory = null; DomainSocketWatcher watcher = null; try { - String loadingFailureReason = - SharedFileDescriptorFactory.getLoadingFailureReason(); - if (loadingFailureReason != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Disabling ShortCircuitRegistry because " + - loadingFailureReason); - } - return; - } - String shmPath = conf.get(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH, - DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT); - if (shmPath.isEmpty()) { - LOG.debug("Disabling ShortCircuitRegistry because shmPath was not set."); - return; - } int interruptCheck = conf.getInt( DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT); if (interruptCheck <= 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("Disabling ShortCircuitRegistry because " + - "interruptCheckMs was set to " + interruptCheck); - } - return; + throw new IOException( + DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS + + " was set to " + interruptCheck); + } + String shmPaths[] = + conf.getTrimmedStrings(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS); + if (shmPaths.length == 0) { + shmPaths = + DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT.split(","); + } + shmFactory = SharedFileDescriptorFactory. + create("HadoopShortCircuitShm_", shmPaths); + String dswLoadingFailure = DomainSocketWatcher.getLoadingFailureReason(); + if (dswLoadingFailure != null) { + throw new IOException(dswLoadingFailure); } - shmFactory = - new SharedFileDescriptorFactory("HadoopShortCircuitShm_", shmPath); watcher = new DomainSocketWatcher(interruptCheck); enabled = true; if (LOG.isDebugEnabled()) { LOG.debug("created new ShortCircuitRegistry with interruptCheck=" + - interruptCheck + ", shmPath=" + shmPath); + interruptCheck + ", shmPath=" + shmFactory.getPath()); + } + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Disabling ShortCircuitRegistry", e); } } finally { this.enabled = enabled; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1574796&r1=1574795&r2=1574796&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Thu Mar 6 08:21:16 2014 @@ -1150,13 +1150,13 @@ - dfs.datanode.shared.file.descriptor.path - /dev/shm + dfs.datanode.shared.file.descriptor.paths + /dev/shm,/tmp - The path to use when creating file descriptors that will be shared - between the DataNode and the DFSClient. Typically we use /dev/shm, so - that the file descriptors will not be written to disk. Systems that - don't have /dev/shm should use /tmp. + A comma-separated list of paths to use when creating file descriptors that + will be shared between the DataNode and the DFSClient. Typically we use + /dev/shm, so that the file descriptors will not be written to disk. + Systems that don't have /dev/shm will fall back to /tmp by default. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java?rev=1574796&r1=1574795&r2=1574796&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java Thu Mar 6 08:21:16 2014 @@ -22,11 +22,9 @@ import java.io.FileInputStream; import java.util.ArrayList; import java.util.Iterator; -import org.apache.commons.lang.SystemUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory; import org.apache.hadoop.hdfs.ExtendedBlockId; import org.apache.hadoop.hdfs.ShortCircuitShm; @@ -45,8 +43,8 @@ public class TestShortCircuitShm { @Before public void before() { - Assume.assumeTrue(NativeIO.isAvailable()); - Assume.assumeTrue(SystemUtils.IS_OS_UNIX); + Assume.assumeTrue(null == + SharedFileDescriptorFactory.getLoadingFailureReason()); } @Test(timeout=60000) @@ -54,7 +52,8 @@ public class TestShortCircuitShm { File path = new File(TEST_BASE, "testStartupShutdown"); path.mkdirs(); SharedFileDescriptorFactory factory = - new SharedFileDescriptorFactory("shm_", path.getAbsolutePath()); + SharedFileDescriptorFactory.create("shm_", + new String[] { path.getAbsolutePath() } ); FileInputStream stream = factory.createDescriptor("testStartupShutdown", 4096); ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream); @@ -68,7 +67,8 @@ public class TestShortCircuitShm { File path = new File(TEST_BASE, "testAllocateSlots"); path.mkdirs(); SharedFileDescriptorFactory factory = - new SharedFileDescriptorFactory("shm_", path.getAbsolutePath()); + SharedFileDescriptorFactory.create("shm_", + new String[] { path.getAbsolutePath() }); FileInputStream stream = factory.createDescriptor("testAllocateSlots", 4096); ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);