From core-commits-return-4325-apmail-hadoop-core-commits-archive=hadoop.apache.org@hadoop.apache.org Wed Mar 19 01:45:20 2008 Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 84765 invoked from network); 19 Mar 2008 01:45:20 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 19 Mar 2008 01:45:20 -0000 Received: (qmail 15715 invoked by uid 500); 19 Mar 2008 01:45:17 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 15685 invoked by uid 500); 19 Mar 2008 01:45:17 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 15676 invoked by uid 99); 19 Mar 2008 01:45:17 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 18 Mar 2008 18:45:17 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Mar 2008 01:44:35 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D968A1A9838; Tue, 18 Mar 2008 18:44:54 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r638677 - in /hadoop/core/trunk: ./ conf/ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/mapred/ Date: Wed, 19 Mar 2008 01:44:53 -0000 To: core-commits@hadoop.apache.org From: cdouglas@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080319014454.D968A1A9838@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cdouglas Date: Tue Mar 18 18:44:50 2008 New Revision: 638677 URL: http://svn.apache.org/viewvc?rev=638677&view=rev Log: HADOOP-2239. Add HsftpFileSystem to permit transferring files over ssl. Added: hadoop/core/trunk/conf/sslinfo.xml.example hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java Modified: hadoop/core/trunk/CHANGES.txt hadoop/core/trunk/conf/hadoop-default.xml hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java Modified: hadoop/core/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/CHANGES.txt (original) +++ hadoop/core/trunk/CHANGES.txt Tue Mar 18 18:44:50 2008 @@ -108,6 +108,9 @@ HADOOP-2939. Make the automated patch testing process an executable Ant target, test-patch. (nigel) + HADOOP-2239. Add HsftpFileSystem to permit transferring files over ssl. + (cdouglas) + OPTIMIZATIONS HADOOP-2790. Fixed inefficient method hasSpeculativeTask by removing Modified: hadoop/core/trunk/conf/hadoop-default.xml URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/conf/hadoop-default.xml (original) +++ hadoop/core/trunk/conf/hadoop-default.xml Tue Mar 18 18:44:50 2008 @@ -175,6 +175,11 @@ + fs.hsftp.impl + org.apache.hadoop.dfs.HsftpFileSystem + + + fs.ramfs.impl org.apache.hadoop.fs.InMemoryFileSystem The FileSystem for ramfs: uris. @@ -242,6 +247,24 @@ The address and the base port where the dfs namenode web ui will listen on. If the port is 0 then the server will start on a free port. + + + + + dfs.datanode.https.address + 0.0.0.0:50475 + + + + dfs.https.address + 0.0.0.0:50470 + + + + https.keystore.info.rsrc + sslinfo.xml + The name of the resource from which ssl keystore information + will be extracted Added: hadoop/core/trunk/conf/sslinfo.xml.example URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/sslinfo.xml.example?rev=638677&view=auto ============================================================================== --- hadoop/core/trunk/conf/sslinfo.xml.example (added) +++ hadoop/core/trunk/conf/sslinfo.xml.example Tue Mar 18 18:44:50 2008 @@ -0,0 +1,21 @@ + + + + + + + https.keystore.location + ${user.home}/.keystore + + + + https.keystore.password + changeme + + + + https.keystore.keypassword + changeme + + + Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Tue Mar 18 18:44:50 2008 @@ -311,6 +311,16 @@ String infoHost = infoSocAddr.getHostName(); int tmpInfoPort = infoSocAddr.getPort(); this.infoServer = new StatusHttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0); + InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr( + conf.get("dfs.datanode.https.address", infoHost + ":" + 0)); + Configuration sslConf = new Configuration(conf); + sslConf.addResource(conf.get("https.keystore.info.rsrc", "sslinfo.xml")); + String keyloc = sslConf.get("https.keystore.location"); + if (null != keyloc) { + this.infoServer.addSslListener(secInfoSocAddr, keyloc, + sslConf.get("https.keystore.password", ""), + sslConf.get("https.keystore.keypassword", "")); + } this.infoServer.addServlet(null, "/streamFile/*", StreamFile.class); this.infoServer.setAttribute("datanode.blockScanner", blockScanner); this.infoServer.addServlet(null, "/blockScannerReport", Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Tue Mar 18 18:44:50 2008 @@ -310,6 +310,21 @@ int tmpInfoPort = infoSocAddr.getPort(); this.infoServer = new StatusHttpServer("dfs", infoHost, tmpInfoPort, tmpInfoPort == 0); + InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr( + conf.get("dfs.https.address", infoHost + ":" + 0)); + Configuration sslConf = new Configuration(conf); + sslConf.addResource(conf.get("https.keystore.info.rsrc", "sslinfo.xml")); + String keyloc = sslConf.get("https.keystore.location"); + if (null != keyloc) { + this.infoServer.addSslListener(secInfoSocAddr, keyloc, + sslConf.get("https.keystore.password", ""), + sslConf.get("https.keystore.keypassword", "")); + } + // assume same ssl port for all datanodes + InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr( + conf.get("dfs.datanode.https.address", infoHost + ":" + 50475)); + this.infoServer.setAttribute("datanode.https.port", + datanodeSslPort.getPort()); this.infoServer.setAttribute("name.system", this); this.infoServer.setAttribute("name.node", nn); this.infoServer.setAttribute("name.conf", conf); Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java Tue Mar 18 18:44:50 2008 @@ -34,11 +34,16 @@ * @see org.apache.hadoop.dfs.HftpFileSystem */ public class FileDataServlet extends DfsServlet { - private static URI createUri(DFSFileInfo i, UnixUserGroupInformation ugi, - ClientProtocol nnproxy) throws IOException, URISyntaxException { + + private URI createUri(DFSFileInfo i, UnixUserGroupInformation ugi, + ClientProtocol nnproxy, String scheme) + throws IOException, URISyntaxException { final DatanodeInfo host = pickSrcDatanode(i, nnproxy); - return new URI("http", null, host.getHostName(), host.getInfoPort(), - "/streamFile", "filename=" + i.getPath() + "&ugi=" + ugi, null); + return new URI(scheme, null, host.getHostName(), + "https".equals(scheme) + ? (Integer)getServletContext().getAttribute("datanode.https.port") + : host.getInfoPort(), + "/streamFile", "filename=" + i.getPath() + "&ugi=" + ugi, null); } private final static int BLOCK_SAMPLE = 5; @@ -97,7 +102,8 @@ ? request.getPathInfo() : "/"; DFSFileInfo info = nnproxy.getFileInfo(path); if (!info.isDir()) { - response.sendRedirect(createUri(info, ugi, nnproxy).toURL().toString()); + response.sendRedirect(createUri(info, ugi, nnproxy, + request.getScheme()).toURL().toString()); } else { response.sendError(400, "cat: " + path + ": is a directory"); } Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java Tue Mar 18 18:44:50 2008 @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.HttpURLConnection; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -64,10 +65,10 @@ HttpURLConnection.setFollowRedirects(true); } - private String fshostname = ""; - private int fsport = -1; + protected InetSocketAddress nnAddr; + protected UserGroupInformation ugi; + protected static final SimpleDateFormat df = ListPathsServlet.df; - private UserGroupInformation ugi; @Override public void initialize(URI name, Configuration conf) throws IOException { @@ -76,43 +77,43 @@ this.ugi = UnixUserGroupInformation.login(conf); } catch (LoginException le) { throw new IOException(StringUtils.stringifyException(le)); - } + } - this.fshostname = name.getHost(); - this.fsport = name.getPort(); - if(fsport >= 0) - return; - String infoAddr = - NetUtils.getServerAddress(conf, - "dfs.info.bindAddress", - "dfs.info.port", - "dfs.http.address"); - this.fsport = NetUtils.createSocketAddr(infoAddr).getPort(); + nnAddr = NetUtils.createSocketAddr(name.toString()); } @Override public URI getUri() { try { - return new URI("hftp", null, fshostname, fsport, null, null, null); + return new URI("hftp", null, nnAddr.getHostName(), nnAddr.getPort(), + null, null, null); } catch (URISyntaxException e) { return null; } } - @Override - public FSDataInputStream open(Path f, int buffersize) throws IOException { - HttpURLConnection connection = null; + /** + * Open an HTTP connection to the namenode to read file data and metadata. + * @param path The path component of the URL + * @param query The query component of the URL + */ + protected HttpURLConnection openConnection(String path, String query) + throws IOException { try { - final URL url = new URI("http", null, fshostname, fsport, - "/data" + f.toUri().getPath(), "ugi=" + ugi, null).toURL(); - connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod("GET"); - connection.connect(); + final URL url = new URI("http", null, nnAddr.getHostName(), + nnAddr.getPort(), path, query, null).toURL(); + return (HttpURLConnection)url.openConnection(); } catch (URISyntaxException e) { - IOException ie = new IOException("invalid url"); - ie.initCause(e); - throw ie; + throw (IOException)new IOException().initCause(e); } + } + + @Override + public FSDataInputStream open(Path f, int buffersize) throws IOException { + HttpURLConnection connection = null; + connection = openConnection("/data" + f.toUri().getPath(), "ugi=" + ugi); + connection.setRequestMethod("GET"); + connection.connect(); final InputStream in = connection.getInputStream(); return new FSDataInputStream(new FSInputStream() { public int read() throws IOException { @@ -160,13 +161,11 @@ Long.valueOf(attrs.getValue("blocksize")).longValue(), modif, FsPermission.valueOf(attrs.getValue("permission")), attrs.getValue("owner"), attrs.getValue("group"), - new Path("hftp", fshostname + ":" + fsport, - attrs.getValue("path"))) + new Path(getUri().toString(), attrs.getValue("path"))) : new FileStatus(0L, true, 0, 0L, modif, FsPermission.valueOf(attrs.getValue("permission")), attrs.getValue("owner"), attrs.getValue("group"), - new Path("hftp", fshostname + ":" + fsport, - attrs.getValue("path"))); + new Path(getUri().toString(), attrs.getValue("path"))); fslist.add(fs); } @@ -174,10 +173,8 @@ try { XMLReader xr = XMLReaderFactory.createXMLReader(); xr.setContentHandler(this); - final URL url = new URI("http", null, fshostname, fsport, - "/listPaths" + path, "ugi=" + ugi + (recur? "&recursive=yes" : ""), - null).toURL(); - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + HttpURLConnection connection = openConnection("/listPaths" + path, + "ugi=" + ugi + (recur? "&recursive=yes" : "")); connection.setRequestMethod("GET"); connection.connect(); @@ -185,10 +182,6 @@ xr.parse(new InputSource(resp)); } catch (SAXException e) { IOException ie = new IOException("invalid xml directory content"); - ie.initCause(e); - throw ie; - } catch (URISyntaxException e) { - IOException ie = new IOException("invalid url"); ie.initCause(e); throw ie; } Added: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java?rev=638677&view=auto ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java (added) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java Tue Mar 18 18:44:50 2008 @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.dfs; + +import java.io.IOException; + +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import org.apache.hadoop.fs.Path; + +/** An implementation of a protocol for accessing filesystems over HTTPS. + * The following implementation provides a limited, read-only interface + * to a filesystem over HTTPS. + * @see org.apache.hadoop.dfs.ListPathsServlet + * @see org.apache.hadoop.dfs.FileDataServlet + */ +public class HsftpFileSystem extends HftpFileSystem { + + @Override + protected HttpURLConnection openConnection(String path, String query) + throws IOException { + try { + final URL url = new URI("https", null, nnAddr.getHostName(), + nnAddr.getPort(), path, query, null).toURL(); + return (HttpURLConnection)url.openConnection(); + } catch (URISyntaxException e) { + throw (IOException)new IOException().initCause(e); + } + } + + @Override + public URI getUri() { + try { + return new URI("hsftp", null, nnAddr.getHostName(), nnAddr.getPort(), + null, null, null); + } catch (URISyntaxException e) { + return null; + } + } + +} Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java?rev=638677&r1=638676&r2=638677&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java Tue Mar 18 18:44:50 2008 @@ -21,6 +21,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.net.URL; +import java.net.InetSocketAddress; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -32,6 +33,7 @@ import org.apache.hadoop.util.ReflectionUtils; import org.mortbay.http.HttpContext; import org.mortbay.http.SocketListener; +import org.mortbay.http.SslListener; import org.mortbay.http.handler.ResourceHandler; import org.mortbay.jetty.servlet.WebApplicationContext; @@ -46,6 +48,7 @@ public class StatusHttpServer { private org.mortbay.jetty.Server webServer; private SocketListener listener; + private SslListener sslListener; private boolean findPort; private WebApplicationContext webAppContext; private static final Log LOG = @@ -172,6 +175,28 @@ listener.setMinThreads(min); listener.setMaxThreads(max); } + + /** + * Configure an ssl listener on the server. + * @param addr address to listen on + * @param keystore location of the keystore + * @param storPass password for the keystore + * @param keyPass password for the key + */ + public void addSslListener(InetSocketAddress addr, String keystore, + String storPass, String keyPass) throws IOException { + if (sslListener != null || webServer.isStarted()) { + throw new IOException("Failed to add ssl listener"); + } + sslListener = new SslListener(); + sslListener.setHost(addr.getHostName()); + sslListener.setPort(addr.getPort()); + sslListener.setKeystore(keystore); + sslListener.setPassword(storPass); + sslListener.setKeyPassword(keyPass); + webServer.addListener(sslListener); + } + /** * Start the server. Does not wait for the server to start. */