hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r758495 - in /hadoop/core/trunk: ./ src/contrib/hdfsproxy/ src/contrib/hdfsproxy/conf/ src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/ src/contrib/hdfsproxy/src/test/re...
Date Thu, 26 Mar 2009 01:42:01 GMT
Author: cdouglas
Date: Thu Mar 26 01:41:57 2009
New Revision: 758495

URL: http://svn.apache.org/viewvc?rev=758495&view=rev
Log:
HADOOP-5363. Add support for proxying connections to multiple clusters with
different versions to hdfsproxy. Contributed by Zhiyong Zhang

Added:
    hadoop/core/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileForward.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/SimpleServlet.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyForwardServlet.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-site.xml
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/hdfsproxy/build.xml
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/cactus-web.xml
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Mar 26 01:41:57 2009
@@ -63,6 +63,9 @@
 
     HADOOP-4539. Introduce backup node and checkpoint node. (shv)
 
+    HADOOP-5363. Add support for proxying connections to multiple clusters with
+    different versions to hdfsproxy. (Zhiyong Zhang via cdouglas)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/build.xml?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/build.xml (original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/build.xml Thu Mar 26 01:41:57 2009
@@ -18,7 +18,7 @@
 -->
 
 <project name="hdfsproxy" default="jar" xmlns:ivy="antlib:org.apache.ivy.ant">
-	<property name="hdfsproxyVersion" value="1.0"/>
+	<property name="hdfsproxyVersion" value="2.0"/>
 	<property name="final.name" value="${ant.project.name}-${hdfsproxyVersion}"/>
 	<property name="javac.debug" value="on"/>
 	<property name="javac.optimize" value="on"/>
@@ -44,8 +44,7 @@
 	<property name="tomcat.container.id" value="tomcat5x"/>
 	<property name="cargo.logging" value="high"/>
 	<property name="cactus.formatter.type" value="xml"/>
-	<property name="cactus.warfile.name" value="test"/>
-	
+	<property name="cactus.warfile.name" value="test"/>	
   
 	<available file="${hadoop.root}/build/classes" type="dir" property="test.available"/>
 	<property environment="env"/>
@@ -62,6 +61,7 @@
 	    <or>
 	    	<equals arg1="${testcase}" arg2="TestProxyFilter" />
 				<equals arg1="${testcase}" arg2="TestProxyUtil" />
+				<equals arg1="${testcase}" arg2="TestProxyForwardServlet" />
 				<not>
 					<isset property="testcase"/>
 				</not>
@@ -103,7 +103,6 @@
                                 <attribute name="Implementation-Vendor" value="Apache"/>
                             </section>
                         </manifest>
-
 		</jar>
 	</target>
 	
@@ -112,7 +111,7 @@
 	<!-- Make war file                                              -->
 	<!-- ================================================================== -->
 	
-	<target name="war" depends="local-package" description="Create war">
+	<target name="war" depends="compile" description="Create war">
 		<echo>
 			Building the .war file
 		</echo>
@@ -129,9 +128,29 @@
 	    <classes dir="${proxy.conf.dir}" excludes="**/*.example **/*.template **/*.sh hadoop-site.xml"/>
 	    <classes dir="${build.classes}"/>
 	    <classes dir="${hadoop.root}/build/classes"/>
-			<classes dir="${test.build.dir}"/>
 	  </war>
 	</target>
+	
+	<target name="forward" depends="compile" description="Create forward war">
+		<echo>
+			Building the forward war file
+		</echo>
+	  <war destfile="${build.dir}/${final.name}-forward.war" webxml="${basedir}/conf/tomcat-forward-web.xml">
+	    <lib dir="${common.ivy.lib.dir}">
+	      <include name="commons-logging-${commons-logging.version}.jar"/>
+        <include name="junit-${junit.version}.jar"/>
+        <include name="log4j-${log4j.version}.jar"/>
+        <include name="slf4j-api-${slf4j-api.version}.jar"/>
+        <include name="slf4j-log4j12-${slf4j-log4j12.version}.jar"/>
+        <include name="xmlenc-${xmlenc.version}.jar"/>
+        <include name="core-${core.vesion}.jar"/> 
+	    </lib>
+	    <classes dir="${proxy.conf.dir}" excludes="**/*.example **/*.template **/*.sh hadoop-site.xml"/>
+	    <classes dir="${build.classes}"/>
+	    <classes dir="${hadoop.root}/build/classes"/>
+	  </war>
+	</target>	
+	
 	<target name="cactifywar" depends="war,load-tasks,cactifywar-pure,cactifywar-clover"
description="To include clover coverage test use -Dclover.home ..."/>
 	
 	<target name="cactifywar-pure" depends="war,load-tasks" unless="useClover">
@@ -145,6 +164,7 @@
           mapping="/ServletRedirectorSecure" roles="test"/>
       <filterredirector mapping="/test/filterRedirector.jsp"/>
       <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
+      <classes dir="${test.build.dir}"/>
     </cactifywar>    	
 	</target>
 
@@ -159,6 +179,7 @@
           mapping="/ServletRedirectorSecure" roles="test"/>
       <filterredirector mapping="/test/filterRedirector.jsp"/>
       <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
+      <classes dir="${test.build.dir}"/>
       <lib dir="${clover.home}/lib">
       	<include name="clover.jar"/> 
       </lib>
@@ -196,6 +217,7 @@
             	<include name="**/${testcase}.java"/>
             	<exclude name="**/TestProxyFilter.java"/>
             	<exclude name="**/TestProxyUtil.java"/>
+            	<exclude name="**/TestProxyForwardServlet.java"/>
             </fileset>
          </batchtest>
     </junit>    	
@@ -304,7 +326,7 @@
 	<!-- ================================================================== -->
 	<!--                                                                    -->
 	<!-- ================================================================== -->
-	<target name="local-package" depends="jar" description="Package in local build directory">
+	<target name="local-package" depends="jar,war" description="Package in local build directory">
 		<mkdir dir="${build.dir}/${final.name}"/>
 		<mkdir dir="${build.dir}/${final.name}/logs"/>
 		<copy todir="${build.dir}/${final.name}" includeEmptyDirs="false">
@@ -367,22 +389,22 @@
 		</chmod>
 
 	</target>
-	
 	<target name="package" depends="local-package" description="Build distribution">
     <mkdir dir="${dist.dir}/contrib/${name}"/>
     <copy todir="${dist.dir}/contrib/${name}">
       <fileset dir="${build.dir}/${final.name}">
       	<exclude name="**/lib/**" />
       	<exclude name="**/src/**" />
+      	<exclude name="*.war" />
       </fileset>
     </copy>
-    <chmod dir="${dist.dir}/contrib/${name}/bin" perm="a+x" includes="*"/>        
       
+    <chmod dir="${dist.dir}/contrib/${name}/bin" perm="a+x" includes="*"/>
 	</target>
 
 	<!-- ================================================================== -->
 	<!-- Make release tarball                                               -->
 	<!-- ================================================================== -->
-	<target name="tar" depends="local-package" description="Make release tarball">
+	<target name="tar" depends="local-package,war" description="Make release tarball">
 		<macro_tar param.destfile="${build.dir}/${final.name}.tar.gz">
 			<param.listofitems>
 				<tarfileset dir="${build.dir}" mode="664">
@@ -396,7 +418,7 @@
 		</macro_tar>
 	</target>
 
-	<target name="binary" depends="local-package" description="Make tarball without source
and documentation">
+	<target name="binary" depends="local-package,war" description="Make tarball without source
and documentation">
 		<macro_tar param.destfile="${build.dir}/${final.name}-bin.tar.gz">
 			<param.listofitems>
 				<tarfileset dir="${build.dir}" mode="664">

Added: hadoop/core/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml?rev=758495&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml (added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml Thu Mar 26 01:41:57
2009
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<web-app>
+
+
+    <!-- General description of your web application -->
+
+    <display-name>HDFS Proxy</display-name>
+    <description>
+      get data from grid forward war
+    </description>
+
+    <context-param>
+      <param-name>webmaster</param-name>
+      <param-value>zhiyong1@yahoo-inc.com</param-value>
+      <description>
+        The EMAIL address of the administrator to whom questions
+        and comments about this application should be addressed.
+      </description>
+    </context-param>
+    
+    <filter>
+	   	<filter-name>proxyFilter</filter-name>
+	   	<filter-class>org.apache.hadoop.hdfsproxy.ProxyFilter</filter-class>
+	   	<init-param>
+	      <param-name>filteraddress</param-name>
+	      <param-value>10</param-value>
+	   	</init-param>
+		</filter>
+
+		<filter-mapping>
+        <filter-name>proxyFilter</filter-name>
+				<url-pattern>/*</url-pattern>
+    </filter-mapping>
+
+    
+    <servlet>
+    	<servlet-name>proxyForward</servlet-name>
+      <description>forward data access to specifc servlets</description>
+      <servlet-class>org.apache.hadoop.hdfsproxy.ProxyForwardServlet</servlet-class>
+    </servlet>
+    
+    <servlet-mapping>
+        <servlet-name>proxyForward</servlet-name>
+        <url-pattern>/listPaths/*</url-pattern>
+    </servlet-mapping>
+	  <servlet-mapping>
+        <servlet-name>proxyForward</servlet-name>
+        <url-pattern>/data/*</url-pattern>
+    </servlet-mapping>
+    <servlet-mapping>
+        <servlet-name>proxyForward</servlet-name>
+        <url-pattern>/streamFile/*</url-pattern>
+    </servlet-mapping>
+    
+    <servlet>
+    	<servlet-name>fileForward</servlet-name>
+      <description>forward file data access to streamFile</description>
+      <servlet-class>org.apache.hadoop.hdfsproxy.ProxyFileForward</servlet-class>
+    </servlet>
+    
+    <servlet-mapping>
+        <servlet-name>fileForward</servlet-name>
+        <url-pattern>/file/*</url-pattern>
+    </servlet-mapping>
+    
+
+		<welcome-file-list>
+		  <welcome-file>index.html</welcome-file>
+		</welcome-file-list>
+
+    <!-- Define the default session timeout for your application,
+         in minutes.  From a servlet or JSP page, you can modify
+         the timeout for a particular session dynamically by using
+         HttpSession.getMaxInactiveInterval(). -->
+
+    <session-config>
+      <session-timeout>30</session-timeout>    <!-- 30 minutes -->
+    </session-config>    
+
+
+</web-app>
+
+
+
+
+
+
+
+

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java
(original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java
Thu Mar 26 01:41:57 2009
@@ -21,8 +21,12 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.server.namenode.FileDataServlet;
@@ -32,6 +36,15 @@
 public class ProxyFileDataServlet extends FileDataServlet {
   /** For java.io.Serializable */
   private static final long serialVersionUID = 1L;
+  
+  /** {@inheritDoc} */
+  @Override
+  public void init() throws ServletException {
+    ServletContext context = getServletContext();
+    if (context.getAttribute("name.conf") == null) { 
+      context.setAttribute("name.conf", new Configuration());
+    }    
+  }
 
   /** {@inheritDoc} */
   @Override
@@ -46,6 +59,8 @@
   /** {@inheritDoc} */
   @Override
   protected UnixUserGroupInformation getUGI(HttpServletRequest request) {
-    return (UnixUserGroupInformation) request.getAttribute("authorized.ugi");
+    String userID = (String) request.getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
+    UnixUserGroupInformation ugi = ProxyUgiManager.getUgiForUser(userID);
+    return ugi;
   }
 }

Added: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileForward.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileForward.java?rev=758495&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileForward.java
(added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileForward.java
Thu Mar 26 01:41:57 2009
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfsproxy;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.security.UnixUserGroupInformation;
+
+
+public class ProxyFileForward extends ProxyForwardServlet {
+  /** For java.io.Serializable */
+  private static final long serialVersionUID = 1L;
+
+  /** {@inheritDoc} */
+  @Override
+  protected String buildForwardPath(HttpServletRequest request, String pathInfo) {
+    String path = "/streamFile";
+    path += "?filename=" + request.getPathInfo();
+    UnixUserGroupInformation ugi = (UnixUserGroupInformation)request.getAttribute("authorized.ugi");
+    if (ugi != null) {
+      path += "&ugi=" + ugi.toString();
+    }
+    return path;
+  }
+  
+}

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
(original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFilter.java
Thu Mar 26 01:41:57 2009
@@ -61,14 +61,14 @@
       .compile("^(/clearUgiCache)$");
   /** Pattern for a filter to find out if a request is HFTP/HSFTP request */
   protected static final Pattern HFTP_PATTERN = Pattern
-      .compile("^(/listPaths|/data|/streamFile)$");
+      .compile("^(/listPaths|/data|/streamFile|/file)$");
   /**
    * Pattern for a filter to find out if an HFTP/HSFTP request stores its file
    * path in the extra path information associated with the URL; if not, the
    * file path is stored in request parameter "filename"
    */
   protected static final Pattern FILEPATH_PATTERN = Pattern
-      .compile("^(/listPaths|/data)$");
+      .compile("^(/listPaths|/data|/file)$");
 
   private static volatile Map<String, Set<Path>> permsMap;
   private static volatile Map<String, Set<BigInteger>> certsMap;
@@ -88,14 +88,16 @@
     Configuration conf = new Configuration(false);
     conf.addResource("hdfsproxy-default.xml");
     conf.addResource("ssl-server.xml");
+    conf.addResource("hdfsproxy-site.xml");
     String nn = conf.get("hdfsproxy.dfs.namenode.address");
     if (nn == null) {
       throw new ServletException("Proxy source cluster name node address not speficied");
     }
     InetSocketAddress nAddr = NetUtils.createSocketAddr(nn);
     context.setAttribute("name.node.address", nAddr);
-    context.setAttribute("name.conf", new Configuration());
-           
+    context.setAttribute("name.conf", new Configuration());   
+    
+    context.setAttribute("org.apache.hadoop.hdfsproxy.conf", conf);
     LOG.info("proxyFilter initialization success: " + nn);
   }
 
@@ -165,7 +167,7 @@
 
     HttpServletRequest rqst = (HttpServletRequest) request;
     HttpServletResponse rsp = (HttpServletResponse) response;
-
+    
     if (LOG.isDebugEnabled()) {
       StringBuilder b = new StringBuilder("Request from ").append(
           rqst.getRemoteHost()).append("/").append(rqst.getRemoteAddr())
@@ -264,7 +266,10 @@
       userID = userID.substring(3);
       
       String servletPath = rqst.getServletPath();
-      if (unitTest) servletPath = rqst.getParameter("TestSevletPathInfo");
+      if (unitTest) { 
+        servletPath = rqst.getParameter("TestSevletPathInfo");
+        LOG.info("this is for unit test purpose only");
+      }
       
       if (HFTP_PATTERN.matcher(servletPath).matches()) {
         // request is an HSFTP request
@@ -317,11 +322,16 @@
         return;
       }
       rqst.setAttribute("authorized.ugi", ugi);
+      rqst.setAttribute("org.apache.hadoop.hdfsproxy.authorized.userID", userID);
     } else if(rqst.getScheme().equalsIgnoreCase("http")) { // http request, set ugi for servlets,
only for testing purposes
       String ugi = rqst.getParameter("ugi");
       if (ugi != null) {
         rqst.setAttribute("authorized.ugi", new UnixUserGroupInformation(ugi
           .split(",")));
+        String[] ugiStr = ugi.split(",");
+        if(ugiStr.length > 0) {
+          rqst.setAttribute("org.apache.hadoop.hdfsproxy.authorized.userID", ugiStr[0]);
+        }
       } 
     }
     chain.doFilter(request, response);

Added: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java?rev=758495&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java
(added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java
Thu Mar 26 01:41:57 2009
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfsproxy;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import javax.servlet.ServletException;
+import javax.servlet.ServletContext;
+import javax.servlet.RequestDispatcher;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+
+/**
+ * 
+ *
+ */
+public class ProxyForwardServlet extends HttpServlet {
+  /**
+   * 
+   */
+  private static final long serialVersionUID = 1L;
+  private static Configuration configuration = null;
+  public static final Log LOG = LogFactory.getLog(ProxyForwardServlet.class);
+  
+  /** {@inheritDoc} */
+  @Override
+  public void init() throws ServletException {
+    ServletContext context = getServletContext();
+    configuration = (Configuration) context.getAttribute("org.apache.hadoop.hdfsproxy.conf");
+  }
+  
+  /** {@inheritDoc} */
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+    throws IOException, ServletException {  
+    String hostname = request.getServerName(); 
+    
+    String version = configuration.get(hostname);
+    if (version != null) {
+      ServletContext curContext = getServletContext();
+      ServletContext dstContext = curContext.getContext(version);
+      
+      if (dstContext == null) {
+        LOG.info("Context non-exist or restricted from access: " + version);
+        response.sendError(HttpServletResponse.SC_NOT_FOUND);
+        return;
+      }
+      LOG.debug("Request to " + hostname + " is forwarded to version " + version);
+      forwardRequest(request, response, dstContext, request.getServletPath());
+
+    } else {
+      LOG.info("not a valid context path");
+      response.sendError(HttpServletResponse.SC_NOT_IMPLEMENTED); 
+    }
+  } 
+  /** {@inheritDoc} */
+  public void forwardRequest(HttpServletRequest request, HttpServletResponse response, ServletContext
context, String pathInfo) 
+    throws IOException, ServletException{
+    String path = buildForwardPath(request, pathInfo);    
+    RequestDispatcher dispatcher = context.getRequestDispatcher(path);
+    if (dispatcher == null) {
+      LOG.info("There was no such dispatcher");
+      response.sendError(HttpServletResponse.SC_NO_CONTENT);
+      return;
+    }
+    dispatcher.forward(request, response);
+  }
+  
+  /** {@inheritDoc} */
+  protected String buildForwardPath(HttpServletRequest request, String pathInfo) {
+    String path = pathInfo;
+    if (request.getPathInfo() != null) {
+      path += request.getPathInfo();
+    }
+    if (request.getQueryString() != null) {
+      path += "?" + request.getQueryString();
+    }
+    return path;
+  }
+}

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java
(original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java
Thu Mar 26 01:41:57 2009
@@ -17,8 +17,11 @@
  */
 package org.apache.hadoop.hdfsproxy;
 
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.namenode.ListPathsServlet;
 import org.apache.hadoop.security.UnixUserGroupInformation;
 
@@ -26,10 +29,21 @@
 public class ProxyListPathsServlet extends ListPathsServlet {
   /** For java.io.Serializable */
   private static final long serialVersionUID = 1L;
+  
+  /** {@inheritDoc} */
+  @Override
+  public void init() throws ServletException {
+    ServletContext context = getServletContext();
+    if (context.getAttribute("name.conf") == null) { 
+      context.setAttribute("name.conf", new Configuration());
+    }    
+  }
 
   /** {@inheritDoc} */
   @Override
   protected UnixUserGroupInformation getUGI(HttpServletRequest request) {
-    return (UnixUserGroupInformation) request.getAttribute("authorized.ugi");
+    String userID = (String) request.getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
+    UnixUserGroupInformation ugi = ProxyUgiManager.getUgiForUser(userID);
+    return ugi;
   }
 }

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java
(original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java
Thu Mar 26 01:41:57 2009
@@ -21,6 +21,7 @@
 import java.net.InetSocketAddress;
 
 import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.hadoop.hdfs.DFSClient;
@@ -32,6 +33,14 @@
 public class ProxyStreamFile extends StreamFile {
   /** For java.io.Serializable */
   private static final long serialVersionUID = 1L;
+  /** {@inheritDoc} */
+  @Override
+  public void init() throws ServletException {
+    ServletContext context = getServletContext();
+    if (context.getAttribute("name.conf") == null) { 
+      context.setAttribute("name.conf", new Configuration());
+    }    
+  }
 
   /** {@inheritDoc} */
   @Override
@@ -50,6 +59,8 @@
   /** {@inheritDoc} */
   @Override
   protected UnixUserGroupInformation getUGI(HttpServletRequest request) {
-    return (UnixUserGroupInformation) request.getAttribute("authorized.ugi");
+    String userID = (String) request.getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
+    UnixUserGroupInformation ugi = ProxyUgiManager.getUgiForUser(userID);
+    return ugi;
   }
 }

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java
(original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java
Thu Mar 26 01:41:57 2009
@@ -19,6 +19,8 @@
 package org.apache.hadoop.hdfsproxy;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -35,6 +37,10 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.util.HostsFileReader;
 
@@ -46,7 +52,7 @@
   public static final Log LOG = LogFactory.getLog(ProxyUtil.class);
   
   private static enum UtilityOption {
-    RELOAD("-reloadPermFiles"), CLEAR("-clearUgiCache");
+    RELOAD("-reloadPermFiles"), CLEAR("-clearUgiCache"), GET("-get");
 
     private String name = null;
 
@@ -163,14 +169,48 @@
     }
     return true;
   }
+  
+  
+  static FSDataInputStream open(Configuration conf, String hostname, int port, String path)
throws IOException {
+    setupSslProps(conf);
+    HttpURLConnection connection = null;
+    connection = openConnection(hostname, port, path);
+    connection.connect();
+    final InputStream in = connection.getInputStream();
+    return new FSDataInputStream(new FSInputStream() {
+        public int read() throws IOException {
+          return in.read();
+        }
+        public int read(byte[] b, int off, int len) throws IOException {
+          return in.read(b, off, len);
+        }
+
+        public void close() throws IOException {
+          in.close();
+        }
+
+        public void seek(long pos) throws IOException {
+          throw new IOException("Can't seek!");
+        }
+        public long getPos() throws IOException {
+          throw new IOException("Position unknown!");
+        }
+        public boolean seekToNewSource(long targetPos) throws IOException {
+          return false;
+        }
+      });
+  }
 
   public static void main(String[] args) throws Exception {
-    if(args.length != 1 || 
+    if(args.length < 1 || 
         (!UtilityOption.RELOAD.getName().equalsIgnoreCase(args[0]) 
-            && !UtilityOption.CLEAR.getName().equalsIgnoreCase(args[0]))) {
+            && !UtilityOption.CLEAR.getName().equalsIgnoreCase(args[0])
+            && !UtilityOption.GET.getName().equalsIgnoreCase(args[0])) ||
+            (UtilityOption.GET.getName().equalsIgnoreCase(args[0]) && args.length
!= 4)) {
       System.err.println("Usage: ProxyUtil ["
           + UtilityOption.RELOAD.getName() + "] | ["
-          + UtilityOption.CLEAR.getName() + "]");
+          + UtilityOption.CLEAR.getName() + "] | ["
+          + UtilityOption.GET.getName() + " <hostname> <#port> <path> ]");
       System.exit(0);      
     }
     Configuration conf = new Configuration(false);   
@@ -179,10 +219,17 @@
      
     if (UtilityOption.RELOAD.getName().equalsIgnoreCase(args[0])) {
       // reload user-certs.xml and user-permissions.xml files
-      boolean error = sendCommand(conf, "/reloadPermFiles");
-    } else {
+      sendCommand(conf, "/reloadPermFiles");
+    } else if (UtilityOption.CLEAR.getName().equalsIgnoreCase(args[0])) {
       // clear UGI caches
-      boolean error = sendCommand(conf, "/clearUgiCache");
+      sendCommand(conf, "/clearUgiCache");
+    } else {
+      String hostname = args[1];
+      int port = Integer.parseInt(args[2]);
+      String path = args[3];
+      InputStream in = open(conf, hostname, port, path);
+      IOUtils.copyBytes(in, System.out, conf, false);
+      in.close();
     }
   }
         

Added: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/SimpleServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/SimpleServlet.java?rev=758495&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/SimpleServlet.java
(added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/SimpleServlet.java
Thu Mar 26 01:41:57 2009
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfsproxy;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.PrintWriter;
+import java.io.IOException;
+
+
+/**
+ * simple servlet for forward testing purpose
+ */
+
+public class SimpleServlet extends HttpServlet {
+ 
+  /**
+   * 
+   */
+  private static final long serialVersionUID = 1L;
+
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+    throws IOException {
+	  response.setContentType("text/html");
+	  PrintWriter out = response.getWriter();	  
+	  out.print("<html><head/><body>");
+    out.print("A GET request");
+    out.print("</body></html>");
+	  out.close();
+	  return;
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyForwardServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyForwardServlet.java?rev=758495&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyForwardServlet.java
(added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyForwardServlet.java
Thu Mar 26 01:41:57 2009
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfsproxy;
+
+import org.apache.cactus.ServletTestCase;
+import org.apache.cactus.WebRequest;
+import org.apache.cactus.WebResponse;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.IOException;
+import javax.servlet.ServletException;
+
+/** Unit tests for ProxyUtil */
+public class TestProxyForwardServlet extends ServletTestCase {
+  public static final Log LOG = LogFactory.getLog(TestProxyForwardServlet.class);
+  
+ 
+  public void beginDoGet(WebRequest theRequest) {
+    theRequest.setURL("proxy-test:0", null, "/simple", null, null);
+  }
+  
+  public void testDoGet() throws IOException, ServletException {
+    ProxyForwardServlet servlet = new ProxyForwardServlet();
+    
+    servlet.init(config);
+    servlet.doGet(request, response);
+  }
+  
+  public void endDoGet(WebResponse theResponse)
+    throws IOException {
+    String expected = "<html><head/><body>A GET request</body></html>";
+    String result = theResponse.getText();
+
+    assertEquals(expected, result);
+  }
+  
+  
+  public void testForwardRequest() throws Exception  {
+    ProxyForwardServlet servlet = new ProxyForwardServlet();
+
+    servlet.forwardRequest(request, response, config.getServletContext(), "/simple");
+  }
+  
+  public void endForwardRequest(WebResponse theResponse) throws IOException  {
+    String expected = "<html><head/><body>A GET request</body></html>";
+    String result = theResponse.getText();
+    
+    assertEquals(expected, result);
+    
+  } 
+ 
+}

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/cactus-web.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/cactus-web.xml?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/cactus-web.xml (original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/cactus-web.xml Thu Mar 26 01:41:57
2009
@@ -57,10 +57,21 @@
           <param-value>value1 used for testing</param-value>
         </init-param>
     </servlet>
-
+		
+		<servlet>
+      <servlet-name>Simple</servlet-name>
+      <description> A Simple Servlet </description>
+      <servlet-class>org.apache.hadoop.hdfsproxy.SimpleServlet</servlet-class>
+    </servlet>
+    
     <servlet-mapping>
         <servlet-name>ServletRedirector_TestOverride</servlet-name>
         <url-pattern>/ServletRedirectorOverride</url-pattern>
     </servlet-mapping>
+    
+    <servlet-mapping>
+        <servlet-name>Simple</servlet-name>
+        <url-pattern>/simple/*</url-pattern>
+    </servlet-mapping>
 
 </web-app>

Added: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-site.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-site.xml?rev=758495&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-site.xml
(added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-site.xml
Thu Mar 26 01:41:57 2009
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put hdfsproxy specific properties in this file. -->
+
+<configuration>
+<property>
+  <name>proxy-test</name>
+  <value>/test</value>
+  <description>one hostname corresponds to one web application archive 
+  </description>
+</property>
+
+</configuration>
+

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Thu Mar 26 01:41:57
2009
@@ -91,26 +91,15 @@
     nnAddr = NetUtils.createSocketAddr(name.toString());
   }
   
-  /** randomly pick one from all available IP addresses of a given hostname */
-  protected String pickOneAddress(String hostname) throws UnknownHostException {
-    if ("localhost".equals(hostname))
-      return hostname;
-    InetAddress[] addrs = InetAddress.getAllByName(hostname);
-    if (addrs.length > 1)
-      return addrs[ran.nextInt(addrs.length)].getHostAddress();
-    return addrs[0].getHostAddress();
-  }
 
   @Override
   public URI getUri() {
     try {
-      return new URI("hftp", null, pickOneAddress(nnAddr.getHostName()), nnAddr.getPort(),
+      return new URI("hftp", null, nnAddr.getHostName(), nnAddr.getPort(),
                      null, null, null);
     } catch (URISyntaxException e) {
       return null;
-    } catch (UnknownHostException e) {
-      return null;
-    }
+    } 
   }
 
   /**
@@ -121,7 +110,7 @@
   protected HttpURLConnection openConnection(String path, String query)
       throws IOException {
     try {
-      final URL url = new URI("http", null, pickOneAddress(nnAddr.getHostName()),
+      final URL url = new URI("http", null, nnAddr.getHostName(),
           nnAddr.getPort(), path, query, null).toURL();
       if (LOG.isTraceEnabled()) {
         LOG.trace("url=" + url);

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java?rev=758495&r1=758494&r2=758495&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java Thu Mar 26 01:41:57
2009
@@ -69,7 +69,7 @@
   protected HttpURLConnection openConnection(String path, String query)
       throws IOException {
     try {
-      final URL url = new URI("https", null, pickOneAddress(nnAddr.getHostName()),
+      final URL url = new URI("https", null, nnAddr.getHostName(),
           nnAddr.getPort(), path, query, null).toURL();
       HttpsURLConnection conn = (HttpsURLConnection)url.openConnection();
       // bypass hostname verification
@@ -83,13 +83,11 @@
   @Override
   public URI getUri() {
     try {
-      return new URI("hsftp", null, pickOneAddress(nnAddr.getHostName()), nnAddr.getPort(),
+      return new URI("hsftp", null, nnAddr.getHostName(), nnAddr.getPort(),
                      null, null, null);
     } catch (URISyntaxException e) {
       return null;
-    } catch (UnknownHostException e) {
-      return null;
-    }
+    } 
   }
 
   /**



Mime
View raw message