From hdfs-commits-return-3362-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Fri Apr 6 00:21:20 2012 Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 488089951 for ; Fri, 6 Apr 2012 00:21:20 +0000 (UTC) Received: (qmail 6154 invoked by uid 500); 6 Apr 2012 00:21:20 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 6115 invoked by uid 500); 6 Apr 2012 00:21:20 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 6106 invoked by uid 99); 6 Apr 2012 00:21:20 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 Apr 2012 00:21:20 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 Apr 2012 00:21:17 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E4E7D2388865; Fri, 6 Apr 2012 00:20:56 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1310138 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ src/main/java/org/apach... Date: Fri, 06 Apr 2012 00:20:56 -0000 To: hdfs-commits@hadoop.apache.org From: eli@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120406002056.E4E7D2388865@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: eli Date: Fri Apr 6 00:20:55 2012 New Revision: 1310138 URL: http://svn.apache.org/viewvc?rev=1310138&view=rev Log: HDFS-3208. Bogus entries in hosts files are incorrectly displayed in the report. Contributed by Eli Collins Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Apr 6 00:20:55 2012 @@ -433,6 +433,9 @@ Release 2.0.0 - UNRELEASED HDFS-3210. JsonUtil#toJsonMap for for a DatanodeInfo should use "ipAddr" instead of "name". (eli) + HDFS-3208. Bogus entries in hosts files are incorrectly displayed + in the report. (eli) + BREAKDOWN OF HDFS-1623 SUBTASKS HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Fri Apr 6 00:20:55 2012 @@ -211,7 +211,8 @@ public class DFSConfigKeys extends Commo public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth"; public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; public static final String DFS_DATANODE_ADDRESS_KEY = "dfs.datanode.address"; - public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:50010"; + public static final int DFS_DATANODE_DEFAULT_PORT = 50010; + public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_DEFAULT_PORT; public static final String DFS_DATANODE_DATA_DIR_PERMISSION_KEY = "dfs.datanode.data.dir.perm"; public static final String DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT = "700"; public static final String DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY = "dfs.datanode.directoryscan.interval"; @@ -227,7 +228,8 @@ public class DFSConfigKeys extends Commo public static final String DFS_DATANODE_HANDLER_COUNT_KEY = "dfs.datanode.handler.count"; public static final int DFS_DATANODE_HANDLER_COUNT_DEFAULT = 3; public static final String DFS_DATANODE_HTTP_ADDRESS_KEY = "dfs.datanode.http.address"; - public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50075"; + public static final int DFS_DATANODE_HTTP_DEFAULT_PORT = 50075; + public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_HTTP_DEFAULT_PORT; public static final String DFS_DATANODE_MAX_RECEIVER_THREADS_KEY = "dfs.datanode.max.transfer.threads"; public static final int DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT = 4096; public static final String DFS_DATANODE_NUMBLOCKS_KEY = "dfs.datanode.numblocks"; @@ -251,13 +253,15 @@ public class DFSConfigKeys extends Commo public static final String DFS_HTTPS_ENABLE_KEY = "dfs.https.enable"; public static final boolean DFS_HTTPS_ENABLE_DEFAULT = false; public static final String DFS_HTTPS_PORT_KEY = "dfs.https.port"; - public static final int DFS_HTTPS_PORT_DEFAULT = 50470; public static final String DFS_DEFAULT_CHUNK_VIEW_SIZE_KEY = "dfs.default.chunk.view.size"; public static final int DFS_DEFAULT_CHUNK_VIEW_SIZE_DEFAULT = 32*1024; public static final String DFS_DATANODE_HTTPS_ADDRESS_KEY = "dfs.datanode.https.address"; - public static final String DFS_DATANODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:50475"; + public static final String DFS_DATANODE_HTTPS_PORT_KEY = "datanode.https.port"; + public static final int DFS_DATANODE_HTTPS_DEFAULT_PORT = 50475; + public static final String DFS_DATANODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_HTTPS_DEFAULT_PORT; public static final String DFS_DATANODE_IPC_ADDRESS_KEY = "dfs.datanode.ipc.address"; - public static final String DFS_DATANODE_IPC_ADDRESS_DEFAULT = "0.0.0.0:50020"; + public static final int DFS_DATANODE_IPC_DEFAULT_PORT = 50020; + public static final String DFS_DATANODE_IPC_ADDRESS_DEFAULT = "0.0.0.0" + DFS_DATANODE_IPC_DEFAULT_PORT; public static final String DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY = "dfs.block.access.token.enable"; public static final boolean DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT = false; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java Fri Apr 6 00:20:55 2012 @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; @@ -50,14 +51,21 @@ public class DatanodeID implements Writa protected int infoPort; // info server port protected int ipcPort; // IPC server port - /** Equivalent to DatanodeID(""). */ - public DatanodeID() {this("");} + public DatanodeID() { + this("", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT); + } - /** Equivalent to DatanodeID(ipAddr, "", -1, -1, -1). */ - public DatanodeID(String ipAddr) {this(ipAddr, "", "", -1, -1, -1);} + public DatanodeID(String ipAddr, int xferPort) { + this(ipAddr, "", "", xferPort, + DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, + DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT); + } - /** Equivalent to DatanodeID(ipAddr, "", xferPort, -1, -1). */ - public DatanodeID(String ipAddr, int xferPort) {this(ipAddr, "", "", xferPort, -1, -1);} + public DatanodeID(String ipAddr, String hostName, int xferPort) { + this(ipAddr, hostName, "", xferPort, + DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, + DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT); + } /** * DatanodeID copy constructor Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Fri Apr 6 00:20:55 2012 @@ -265,7 +265,7 @@ public class DatanodeInfo extends Datano long c = getCapacity(); long r = getRemaining(); long u = getDfsUsed(); - buffer.append(ipAddr); + buffer.append(getName()); if (!NetworkTopology.DEFAULT_RACK.equals(location)) { buffer.append(" "+location); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java Fri Apr 6 00:20:55 2012 @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.io.DataInput; -import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; @@ -26,13 +24,10 @@ import java.util.List; import java.util.Queue; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hdfs.DeprecatedUTF8; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.util.LightWeightHashSet; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableUtils; /** * This class extends the DatanodeInfo class with ephemeral information (eg @@ -158,18 +153,18 @@ public class DatanodeDescriptor extends */ private boolean disallowed = false; - /** Default constructor */ public DatanodeDescriptor() {} - /** DatanodeDescriptor constructor + /** + * DatanodeDescriptor constructor * @param nodeID id of the data node */ public DatanodeDescriptor(DatanodeID nodeID) { this(nodeID, 0L, 0L, 0L, 0L, 0, 0); } - /** DatanodeDescriptor constructor - * + /** + * DatanodeDescriptor constructor * @param nodeID id of the data node * @param networkLocation location of the data node in network */ @@ -178,8 +173,8 @@ public class DatanodeDescriptor extends this(nodeID, networkLocation, 0L, 0L, 0L, 0L, 0, 0); } - /** DatanodeDescriptor constructor - * + /** + * DatanodeDescriptor constructor * @param nodeID id of the data node * @param capacity capacity of the data node * @param dfsUsed space used by the data node @@ -199,8 +194,8 @@ public class DatanodeDescriptor extends failedVolumes); } - /** DatanodeDescriptor constructor - * + /** + * DatanodeDescriptor constructor * @param nodeID id of the data node * @param networkLocation location of the data node in network * @param capacity capacity of the data node, including space used by non-dfs Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Fri Apr 6 00:20:55 2012 @@ -71,6 +71,8 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.util.ReflectionUtils; +import com.google.common.net.InetAddresses; + /** * Manage datanodes, include decommission and other activities. */ @@ -353,14 +355,9 @@ public class DatanodeManager { private void resolveNetworkLocation (DatanodeDescriptor node) { List names = new ArrayList(1); if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) { - // get the node's IP address names.add(node.getIpAddr()); } else { - // get the node's host name - String hostName = node.getHostName(); - int colon = hostName.indexOf(":"); - hostName = (colon==-1)?hostName:hostName.substring(0,colon); - names.add(hostName); + names.add(node.getHostName()); } // resolve its network location @@ -771,6 +768,40 @@ public class DatanodeManager { } } + /** + * Parse a DatanodeID from a hosts file entry + * @param hostLine of form [hostname|ip][:port]? + * @return DatanodeID constructed from the given string + */ + private DatanodeID parseDNFromHostsEntry(String hostLine) { + DatanodeID dnId; + String hostStr; + int port; + int idx = hostLine.indexOf(':'); + + if (-1 == idx) { + hostStr = hostLine; + port = DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT; + } else { + hostStr = hostLine.substring(0, idx); + port = Integer.valueOf(hostLine.substring(idx)); + } + + if (InetAddresses.isInetAddress(hostStr)) { + // The IP:port is sufficient for listing in a report + dnId = new DatanodeID(hostStr, "", port); + } else { + String ipAddr = ""; + try { + ipAddr = InetAddress.getByName(hostStr).getHostAddress(); + } catch (UnknownHostException e) { + LOG.warn("Invalid hostname " + hostStr + " in hosts file"); + } + dnId = new DatanodeID(ipAddr, hostStr, port); + } + return dnId; + } + /** For generating datanode reports */ public List getDatanodeListForReport( final DatanodeReportType type) { @@ -782,7 +813,7 @@ public class DatanodeManager { HashMap mustList = new HashMap(); if (listDeadNodes) { - //first load all the nodes listed in include and exclude files. + // Put all nodes referenced in the hosts files in the map Iterator it = hostsReader.getHosts().iterator(); while (it.hasNext()) { mustList.put(it.next(), ""); @@ -805,7 +836,7 @@ public class DatanodeManager { if ( (isDead && listDeadNodes) || (!isDead && listLiveNodes) ) { nodes.add(dn); } - //Remove any form of the this datanode in include/exclude lists. + // Remove any nodes we know about from the map try { InetAddress inet = InetAddress.getByName(dn.getIpAddr()); // compare hostname(:port) @@ -814,7 +845,7 @@ public class DatanodeManager { // compare ipaddress(:port) mustList.remove(inet.getHostAddress().toString()); mustList.remove(inet.getHostAddress().toString()+ ":" +dn.getXferPort()); - } catch ( UnknownHostException e ) { + } catch (UnknownHostException e) { mustList.remove(dn.getName()); mustList.remove(dn.getIpAddr()); LOG.warn(e); @@ -825,9 +856,13 @@ public class DatanodeManager { if (listDeadNodes) { Iterator it = mustList.keySet().iterator(); while (it.hasNext()) { - DatanodeDescriptor dn = - new DatanodeDescriptor(new DatanodeID(it.next())); - dn.setLastUpdate(0); + // The remaining nodes are ones that are referenced by the hosts + // files but that we do not know about, ie that we have never + // head from. Eg. a host that is no longer part of the cluster + // or a bogus entry was given in the hosts files + DatanodeID dnId = parseDNFromHostsEntry(it.next()); + DatanodeDescriptor dn = new DatanodeDescriptor(dnId); + dn.setLastUpdate(0); // Consider this node dead for reporting nodes.add(dn); } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Apr 6 00:20:55 2012 @@ -668,8 +668,7 @@ public class DataNode extends Configured */ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { final String xferIp = streamingAddr.getAddress().getHostAddress(); - DatanodeRegistration bpRegistration = new DatanodeRegistration(xferIp); - bpRegistration.setXferPort(getXferPort()); + DatanodeRegistration bpRegistration = new DatanodeRegistration(xferIp, getXferPort()); bpRegistration.setInfoPort(getInfoPort()); bpRegistration.setIpcPort(getIpcPort()); bpRegistration.setHostName(hostName); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java Fri Apr 6 00:20:55 2012 @@ -62,7 +62,7 @@ public class FileChecksumServlets { ? ((DatanodeInfo)host).getHostName() : host.getIpAddr(); final String scheme = request.getScheme(); final int port = "https".equals(scheme) - ? (Integer)getServletContext().getAttribute("datanode.https.port") + ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY) : host.getInfoPort(); final String encodedPath = ServletUtil.getRawPath(request, "/fileChecksum"); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java Fri Apr 6 00:20:55 2012 @@ -27,6 +27,7 @@ import javax.servlet.http.HttpServletRes import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -62,7 +63,7 @@ public class FileDataServlet extends Dfs hostname = host.getIpAddr(); } final int port = "https".equals(scheme) - ? (Integer)getServletContext().getAttribute("datanode.https.port") + ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY) : host.getInfoPort(); String dtParam = ""; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Fri Apr 6 00:20:55 2012 @@ -165,10 +165,11 @@ public class NameNodeHttpServer { httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth, useKrb); // assume same ssl port for all datanodes - InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf - .get(DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 50475)); - httpServer.setAttribute("datanode.https.port", datanodeSslPort - .getPort()); + InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr( + conf.get(DFS_DATANODE_HTTPS_ADDRESS_KEY, + infoHost + ":" + DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT)); + httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, + datanodeSslPort.getPort()); } httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn); httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY, Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java Fri Apr 6 00:20:55 2012 @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.common.Storage; @@ -52,18 +53,9 @@ implements Writable, NodeRegistration { private StorageInfo storageInfo; private ExportedBlockKeys exportedKeys; - /** - * Default constructor. - */ public DatanodeRegistration() { - this(""); - } - - /** - * Create DatanodeRegistration - */ - public DatanodeRegistration(String ipAddr) { - this(ipAddr, new StorageInfo(), new ExportedBlockKeys()); + this("", DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, + new StorageInfo(), new ExportedBlockKeys()); } public DatanodeRegistration(DatanodeID dn, StorageInfo info, @@ -72,10 +64,14 @@ implements Writable, NodeRegistration { this.storageInfo = info; this.exportedKeys = keys; } - - public DatanodeRegistration(String ipAddr, StorageInfo info, + + public DatanodeRegistration(String ipAddr, int xferPort) { + this(ipAddr, xferPort, new StorageInfo(), new ExportedBlockKeys()); + } + + public DatanodeRegistration(String ipAddr, int xferPort, StorageInfo info, ExportedBlockKeys keys) { - super(ipAddr); + super(ipAddr, xferPort); this.storageInfo = info; this.exportedKeys = keys; } @@ -114,7 +110,7 @@ implements Writable, NodeRegistration { @Override public String toString() { return getClass().getSimpleName() - + "(" + ipAddr + + "(" + getIpAddr() + ", storageID=" + storageID + ", infoPort=" + infoPort + ", ipcPort=" + ipcPort Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Fri Apr 6 00:20:55 2012 @@ -395,9 +395,9 @@ public class TestDFSUtil { conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); String httpsport = DFSUtil.getInfoServer(null, conf, true); - assertEquals("0.0.0.0:50470", httpsport); + assertEquals("0.0.0.0:"+DFS_NAMENODE_HTTPS_PORT_DEFAULT, httpsport); String httpport = DFSUtil.getInfoServer(null, conf, false); - assertEquals("0.0.0.0:50070", httpport); + assertEquals("0.0.0.0:"+DFS_NAMENODE_HTTP_PORT_DEFAULT, httpport); } @Test Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java Fri Apr 6 00:20:55 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; @@ -516,7 +517,8 @@ public class TestDecommission { // Now empty hosts file and ensure the datanode is disallowed // from talking to namenode, resulting in it's shutdown. ArrayListlist = new ArrayList(); - list.add("invalidhost"); + final String badHostname = "BOGUSHOST"; + list.add(badHostname); writeConfigFile(hostsFile, list); for (int j = 0; j < numNameNodes; j++) { @@ -530,6 +532,17 @@ public class TestDecommission { info = client.datanodeReport(DatanodeReportType.LIVE); } assertEquals("Number of live nodes should be 0", 0, info.length); + + // Test that non-live and bogus hostnames are considered "dead". + // The dead report should have an entry for (1) the DN that is + // now considered dead because it is no longer allowed to connect + // and (2) the bogus entry in the hosts file (these entries are + // always added last) + info = client.datanodeReport(DatanodeReportType.DEAD); + assertEquals("There should be 2 dead nodes", 2, info.length); + DatanodeID id = cluster.getDataNodes().get(0).getDatanodeId(); + assertEquals(id.getHostName(), info[0].getHostName()); + assertEquals(badHostname, info[1].getHostName()); } } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java Fri Apr 6 00:20:55 2012 @@ -60,7 +60,7 @@ public class TestReplaceDatanodeOnFailur final DatanodeInfo[][] datanodes = new DatanodeInfo[infos.length + 1][]; datanodes[0] = new DatanodeInfo[0]; for(int i = 0; i < infos.length; ) { - infos[i] = new DatanodeInfo(new DatanodeID("dn" + i)); + infos[i] = new DatanodeInfo(new DatanodeID("dn" + i, 100)); i++; datanodes[i] = new DatanodeInfo[i]; System.arraycopy(infos, 0, datanodes[i], 0, datanodes[i].length); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Fri Apr 6 00:20:55 2012 @@ -115,7 +115,7 @@ public class TestBPOfferService { 0, HdfsConstants.LAYOUT_VERSION)) .when(mock).versionRequest(); - Mockito.doReturn(new DatanodeRegistration("fake-node")) + Mockito.doReturn(new DatanodeRegistration("fake-node", 100)) .when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class)); Mockito.doAnswer(new HeartbeatAnswer(nnIdx)) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1310138&r1=1310137&r2=1310138&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Fri Apr 6 00:20:55 2012 @@ -779,9 +779,9 @@ public class NNThroughputBenchmark { } TinyDatanode(int dnIdx, int blockCapacity) throws IOException { + String ipAddr = DNS.getDefaultIP("default"); String hostName = DNS.getDefaultHost("default", "default"); - dnRegistration = new DatanodeRegistration(hostName); - dnRegistration.setXferPort(getNodePort(dnIdx)); + dnRegistration = new DatanodeRegistration(ipAddr, getNodePort(dnIdx)); dnRegistration.setHostName(hostName); this.blocks = new ArrayList(blockCapacity); this.nrBlocks = 0; @@ -894,10 +894,10 @@ public class NNThroughputBenchmark { for(int t = 0; t < blockTargets.length; t++) { DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; - receivedDNReg = new DatanodeRegistration(dnInfo.getIpAddr()); + receivedDNReg = + new DatanodeRegistration(dnInfo.getIpAddr(), dnInfo.getXferPort()); receivedDNReg.setStorageInfo( - new DataStorage(nsInfo, dnInfo.getStorageID())); - receivedDNReg.setXferPort(dnInfo.getXferPort()); + new DataStorage(nsInfo, dnInfo.getStorageID())); receivedDNReg.setInfoPort(dnInfo.getInfoPort()); receivedDNReg.setIpcPort(dnInfo.getIpcPort()); ReceivedDeletedBlockInfo[] rdBlocks = {