From common-commits-return-14721-apmail-hadoop-common-commits-archive=hadoop.apache.org@hadoop.apache.org Fri Jul 15 20:58:32 2011 Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0A88F61C9 for ; Fri, 15 Jul 2011 20:58:32 +0000 (UTC) Received: (qmail 5827 invoked by uid 500); 15 Jul 2011 20:58:31 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 5647 invoked by uid 500); 15 Jul 2011 20:58:31 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 5640 invoked by uid 99); 15 Jul 2011 20:58:30 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Jul 2011 20:58:30 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Jul 2011 20:58:23 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B7C4D23888CF; Fri, 15 Jul 2011 20:58:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1147317 - in /hadoop/common/trunk/common: ./ src/java/org/apache/hadoop/net/ src/test/core/org/apache/hadoop/net/ Date: Fri, 15 Jul 2011 20:58:01 -0000 To: common-commits@hadoop.apache.org From: jitendra@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110715205801.B7C4D23888CF@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jitendra Date: Fri Jul 15 20:58:00 2011 New Revision: 1147317 URL: http://svn.apache.org/viewvc?rev=1147317&view=rev Log: HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't resolve. Contributed by Jeffrey Naisbitt. Modified: hadoop/common/trunk/common/CHANGES.txt hadoop/common/trunk/common/src/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java hadoop/common/trunk/common/src/java/org/apache/hadoop/net/DNSToSwitchMapping.java hadoop/common/trunk/common/src/java/org/apache/hadoop/net/ScriptBasedMapping.java hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/StaticMapping.java hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/TestScriptBasedMapping.java Modified: hadoop/common/trunk/common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/CHANGES.txt?rev=1147317&r1=1147316&r2=1147317&view=diff ============================================================================== --- hadoop/common/trunk/common/CHANGES.txt (original) +++ hadoop/common/trunk/common/CHANGES.txt Fri Jul 15 20:58:00 2011 @@ -266,6 +266,9 @@ Trunk (unreleased changes) HADOOP-7305. Eclipse project files are incomplete. (Niels Basjes via eli) + HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't + resolve. (Jeffrey Naisbitt via jitendra) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole Modified: hadoop/common/trunk/common/src/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java?rev=1147317&r1=1147316&r2=1147317&view=diff ============================================================================== --- hadoop/common/trunk/common/src/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java (original) +++ hadoop/common/trunk/common/src/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java Fri Jul 15 20:58:00 2011 @@ -17,6 +17,9 @@ */ package org.apache.hadoop.net; +import java.net.InetAddress; +import java.net.SocketException; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -42,16 +45,11 @@ public class CachedDNSToSwitchMapping im this.rawMapping = rawMapping; } - public List resolve(List names) { - // normalize all input names to be in the form of IP addresses - names = NetUtils.normalizeHostNames(names); - - List result = new ArrayList(names.size()); - if (names.isEmpty()) { - return result; - } - + /** + * Returns the hosts from 'names' that have not been cached previously + */ + private List getUncachedHosts(List names) { // find out all names without cached resolved location List unCachedHosts = new ArrayList(names.size()); for (String name : names) { @@ -59,27 +57,81 @@ public class CachedDNSToSwitchMapping im unCachedHosts.add(name); } } - - // Resolve those names - List rNames = rawMapping.resolve(unCachedHosts); - + return unCachedHosts; + } + + /** + * Caches the resolved hosts + */ + private void cacheResolvedHosts(List uncachedHosts, + List resolvedHosts) { // Cache the result - if (rNames != null) { - for (int i=0; i getCachedHosts(List names) { + List result = new ArrayList(names.size()); // Construct the result for (String name : names) { - //now everything is in the cache String networkLocation = cache.get(name); if (networkLocation != null) { result.add(networkLocation); - } else { //resolve all or nothing + } else { return null; } } return result; } + + /** + * Resolves host names and adds them to the cache. + * Unlike the 'resolve" method, this won't hide UnknownHostExceptions + * + * @param names to resolve + * @return List of resolved names + * @throws UnknownHostException if any hosts cannot be resolved + */ + public List resolveValidHosts(List names) + throws UnknownHostException { + if (names.isEmpty()) { + return new ArrayList(); + } + List addresses = new ArrayList(names.size()); + for (String name : names) { + addresses.add(InetAddress.getByName(name).getHostAddress()); + } + + List uncachedHosts = this.getUncachedHosts(names); + + // Resolve the uncached hosts + List resolvedHosts = rawMapping.resolveValidHosts(uncachedHosts); + this.cacheResolvedHosts(uncachedHosts, resolvedHosts); + return this.getCachedHosts(addresses); + } + + public List resolve(List names) { + // normalize all input names to be in the form of IP addresses + names = NetUtils.normalizeHostNames(names); + + List result = new ArrayList(names.size()); + if (names.isEmpty()) { + return result; + } + + List uncachedHosts = this.getUncachedHosts(names); + + // Resolve the uncached hosts + List resolvedHosts = rawMapping.resolve(uncachedHosts); + this.cacheResolvedHosts(uncachedHosts, resolvedHosts); + return this.getCachedHosts(names); + + } } Modified: hadoop/common/trunk/common/src/java/org/apache/hadoop/net/DNSToSwitchMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/java/org/apache/hadoop/net/DNSToSwitchMapping.java?rev=1147317&r1=1147316&r2=1147317&view=diff ============================================================================== --- hadoop/common/trunk/common/src/java/org/apache/hadoop/net/DNSToSwitchMapping.java (original) +++ hadoop/common/trunk/common/src/java/org/apache/hadoop/net/DNSToSwitchMapping.java Fri Jul 15 20:58:00 2011 @@ -18,6 +18,7 @@ package org.apache.hadoop.net; import java.util.List; +import java.net.UnknownHostException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -44,4 +45,23 @@ public interface DNSToSwitchMapping { * @return list of resolved network paths */ public List resolve(List names); + + /** + * Resolves a list of DNS-names/IP-addresses and returns back a list of + * switch information (network paths). One-to-one correspondence must be + * maintained between the elements in the lists. + * Consider an element in the argument list - x.y.com. The switch information + * that is returned must be a network path of the form /foo/rack, + * where / is the root, and 'foo' is the switch where 'rack' is connected. + * Note the hostname/ip-address is not part of the returned path. + * The network topology of the cluster would determine the number of + * components in the network path. Unlike 'resolve', names must be + * resolvable + * @param names + * @return list of resolved network paths + * @throws UnknownHostException if any hosts are not resolvable + */ + public List resolveValidHosts(List names) + throws UnknownHostException; + } Modified: hadoop/common/trunk/common/src/java/org/apache/hadoop/net/ScriptBasedMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/java/org/apache/hadoop/net/ScriptBasedMapping.java?rev=1147317&r1=1147316&r2=1147317&view=diff ============================================================================== --- hadoop/common/trunk/common/src/java/org/apache/hadoop/net/ScriptBasedMapping.java (original) +++ hadoop/common/trunk/common/src/java/org/apache/hadoop/net/ScriptBasedMapping.java Fri Jul 15 20:58:00 2011 @@ -20,6 +20,7 @@ package org.apache.hadoop.net; import java.util.*; import java.io.*; +import java.net.UnknownHostException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -123,6 +124,17 @@ implements Configurable return m; } + public List resolveValidHosts(List names) + throws UnknownHostException { + List result = this.resolve(names); + if (result != null) { + return result; + } else { + throw new UnknownHostException( + "Unknown host(s) returned from ScriptBasedMapping"); + } + } + private String runResolveCommand(List args) { int loopCount = 0; if (args.size() == 0) { Modified: hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/StaticMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/StaticMapping.java?rev=1147317&r1=1147316&r2=1147317&view=diff ============================================================================== --- hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/StaticMapping.java (original) +++ hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/StaticMapping.java Fri Jul 15 20:58:00 2011 @@ -18,6 +18,7 @@ package org.apache.hadoop.net; import java.util.*; +import java.net.UnknownHostException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -59,4 +60,19 @@ public class StaticMapping extends Confi return m; } } + public List resolveValidHosts(List names) + throws UnknownHostException { + List m = new ArrayList(); + synchronized (nameToRackMap) { + for (String name : names) { + String rackId; + if ((rackId = nameToRackMap.get(name)) != null) { + m.add(rackId); + } else { + throw new UnknownHostException(name); + } + } + return m; + } + } } Modified: hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/TestScriptBasedMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/TestScriptBasedMapping.java?rev=1147317&r1=1147316&r2=1147317&view=diff ============================================================================== --- hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/TestScriptBasedMapping.java (original) +++ hadoop/common/trunk/common/src/test/core/org/apache/hadoop/net/TestScriptBasedMapping.java Fri Jul 15 20:58:00 2011 @@ -19,6 +19,7 @@ package org.apache.hadoop.net; import java.util.ArrayList; import java.util.List; +import java.net.UnknownHostException; import org.apache.hadoop.conf.Configuration; @@ -26,21 +27,59 @@ import junit.framework.TestCase; public class TestScriptBasedMapping extends TestCase { - public void testNoArgsMeansNoResult() { - ScriptBasedMapping mapping = new ScriptBasedMapping(); + private ScriptBasedMapping mapping; + private Configuration conf; + private List names; + + public TestScriptBasedMapping() { + mapping = new ScriptBasedMapping(); - Configuration conf = new Configuration(); + conf = new Configuration(); conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY, ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1); conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename"); - mapping.setConf(conf); + mapping.setConf(conf); + } - List names = new ArrayList(); + public void testNoArgsMeansNoResult() { + names = new ArrayList(); names.add("some.machine.name"); names.add("other.machine.name"); - List result = mapping.resolve(names); assertNull(result); } + + public void testResolveValidInvalidHostException() { + names = new ArrayList(); + names.add("1.com"); // Add invalid hostname that doesn't resolve + boolean exceptionThrown = false; + try { + mapping.resolveValidHosts(names); + } catch (UnknownHostException e) { + exceptionThrown = true; + } + assertTrue( + "resolveValidHosts did not throw UnknownHostException for invalid host", + exceptionThrown); + } + + public void testResolveValidHostNoException() { + conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY, + ScriptBasedMapping.MIN_ALLOWABLE_ARGS); + conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "echo"); + mapping.setConf(conf); + + names = new ArrayList(); + names.add("some.machine.name"); + names.add("other.machine.name"); + + boolean exceptionThrown = false; + try { + mapping.resolveValidHosts(names); + } catch (UnknownHostException e) { + exceptionThrown = true; + } + assertFalse("resolveValidHosts threw Exception for valid host", exceptionThrown); + } }