hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r752682 - in /hadoop/core/trunk: ./ src/contrib/hdfsproxy/ src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/ src/contrib/hdfsproxy/src/test/resources/tomcat-config/
Date Wed, 11 Mar 2009 23:02:27 GMT
Author: cdouglas
Date: Wed Mar 11 23:02:26 2009
New Revision: 752682

URL: http://svn.apache.org/viewvc?rev=752682&view=rev
Log:
HADOOP-5386. Modify hdfsproxy unit test to start on a random port,
implement clover instrumentation. Contributed by Zhiyong Zhang

Added:
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/FindFreePort.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/hdfsproxy/build.xml
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyUtil.java
    hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-config/server.xml

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=752682&r1=752681&r2=752682&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Mar 11 23:02:26 2009
@@ -291,6 +291,9 @@
 
     HADOOP-5458. Remove leftover Chukwa entries from build, etc. (cdouglas)
 
+    HADOOP-5386. Modify hdfsproxy unit test to start on a random port,
+    implement clover instrumentation. (Zhiyong Zhang via cdouglas)
+
 Release 0.20.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/build.xml?rev=752682&r1=752681&r2=752682&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/build.xml (original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/build.xml Wed Mar 11 23:02:26 2009
@@ -42,25 +42,41 @@
 	<property name="reports.dir" value="${target.dir}/reports"/>
 	<property name="tomcatconfig.dir" value="${target.dir}/tomcat-config"/>
 	<property name="tomcat.container.id" value="tomcat5x"/>
-	<property name="cargo.servlet.port" value="8087"/>
 	<property name="cargo.logging" value="high"/>
 	<property name="cactus.formatter.type" value="xml"/>
 	<property name="cactus.warfile.name" value="test"/>
-	<property environment="env"/>
 	
+  
+	<available file="${hadoop.root}/build/classes" type="dir" property="test.available"/>
+	<property environment="env"/>
 	<!-- check if environment has been set -->
-  <condition property="source.conf.dir" value="${env.HADOOP_CONF_DIR}" else="${basedir}/conf">
-    <and>
-        <isset property="env.HADOOP_CONF_DIR"/>
-        <available file="${env.HADOOP_CONF_DIR}/core-site.xml"/>        
-    </and>
-  </condition>
   <condition property="proxy.conf.dir" value="${env.HDFSPROXY_CONF_DIR}" else="${basedir}/conf">
     <and>
         <isset property="env.HDFSPROXY_CONF_DIR"/>
         <available file="${env.HDFSPROXY_CONF_DIR}/hdfsproxy-default.xml"/>
     </and>
+  </condition> 
+  
+  <condition property="startCactus">
+  	<and>
+	    <or>
+	    	<equals arg1="${testcase}" arg2="TestProxyFilter" />
+				<equals arg1="${testcase}" arg2="TestProxyUtil" />
+				<not>
+					<isset property="testcase"/>
+				</not>
+	    </or>
+    	<isset property="test.available"/>
+    </and>
+  </condition>
+  
+  <condition property="useClover">
+  	<and>
+    	<isset property="clover.home"/>
+    	<available file="${clover.home}/lib/clover.jar"/>
+    </and>
   </condition>
+
 	<property name="ivy.settings.file" location="${hadoop.root}/ivy/ivysettings.xml"/>
   
   <target name="ivy-init" depends="ivy-init-antlib">
@@ -113,17 +129,17 @@
         <include name="xmlenc-${xmlenc.version}.jar"/>
         <include name="core-${core.vesion}.jar"/> 
 	    </lib>
-	    <classes dir="${source.conf.dir}" excludes="**/*.example **/*.sh"/>
-	    <classes dir="${proxy.conf.dir}" excludes="**/*.template **/*.sh"/>
+	    <classes dir="${proxy.conf.dir}" excludes="**/*.example **/*.template **/*.sh hadoop-site.xml"/>
 	    <classes dir="${build.classes}"/>
 	    <classes dir="${hadoop.root}/build/classes"/>
 			<classes dir="${test.build.dir}"/>
 	  </war>
 	</target>
+	<target name="cactifywar" depends="war,load-tasks,cactifywar-pure,cactifywar-clover"
description="To include clover coverage test use -Dclover.home ..."/>
 	
-	
-	<target name="cactifywar" depends="war,load-tasks">
+	<target name="cactifywar-pure" depends="war,load-tasks" unless="useClover">
 		<mkdir dir="${target.dir}" />
+		<echo> no clover found ...</echo>
     <cactifywar srcfile="${build.dir}/${final.name}.war"
         destfile="${target.dir}/${cactus.warfile.name}.war"
         mergewebxml="${src.test.resources}/cactus-web.xml">
@@ -131,10 +147,77 @@
       <servletredirector name="ServletRedirectorSecure"
           mapping="/ServletRedirectorSecure" roles="test"/>
       <filterredirector mapping="/test/filterRedirector.jsp"/>
+      <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
     </cactifywar>    	
 	</target>
 
-	<target name="test" depends="compile,compile-test,cactifywar" if="test.available">
+	<target name="cactifywar-clover" depends="war,load-tasks" if="useClover">
+		<mkdir dir="${target.dir}" />
+		<echo> Including clover.jar in the war file ...</echo>
+    <cactifywar srcfile="${build.dir}/${final.name}.war"
+        destfile="${target.dir}/${cactus.warfile.name}.war"
+        mergewebxml="${src.test.resources}/cactus-web.xml">
+      <servletredirector/>
+      <servletredirector name="ServletRedirectorSecure"
+          mapping="/ServletRedirectorSecure" roles="test"/>
+      <filterredirector mapping="/test/filterRedirector.jsp"/>
+      <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
+      <lib dir="${clover.home}/lib">
+      	<include name="clover.jar"/> 
+      </lib>
+    </cactifywar>    	
+	</target>
+	
+	<target name="test" depends="compile,compile-test,test-junit,test-cactus" description="Automated
Test Framework" if="test.available"/>
+	
+	<target name="test-junit" depends="compile,compile-test" if="test.available">
+		<junit fork="yes" printsummary="yes" errorProperty="tests.failed" failureProperty="tests.failed">
+        <classpath refid="test.classpath"/>
+        <sysproperty key="test.build.data" value="${build.test}/data"/>
+	      <sysproperty key="build.test" value="${build.test}"/>
+	      <sysproperty key="user.dir" value="${build.test}/data"/>
+	      <sysproperty key="fs.default.name" value="${fs.default.name}"/>
+	      <sysproperty key="hadoop.test.localoutputfile" value="${hadoop.test.localoutputfile}"/>
+	      <sysproperty key="hadoop.log.dir" value="${hadoop.log.dir}"/> 
+        <sysproperty key="test.src.dir" value="${test.src.dir}"/>
+        <sysproperty key="javax.net.ssl.trustStore" value="${ssl.keystore.proxy}"/>
+				<sysproperty key="javax.net.ssl.trustStorePassword" value="changeme"/>
+				<sysproperty key="javax.net.ssl.keyStore.proxy" value="${ssl.keystore.proxy}"/>
+				<sysproperty key="javax.net.ssl.keyStore" value="${ssl.keystore.client}"/>
+				<sysproperty key="javax.net.ssl.keyStorePassword" value="changeme"/>
+				<sysproperty key="javax.net.ssl.keyPassword" value="changeme"/>			
+				<sysproperty key="javax.net.ssl.clientCert" value="${ssl.client.cert}"/>
+        <formatter type="xml" />
+        <batchtest todir="${test.build.dir}" unless="testcase">
+           <fileset dir="${src.test}">
+             <include name="**/TestHdfsProxy.java"/>
+             <include name="**/TestProxyUgiManager.java"/>
+           </fileset>
+        </batchtest>
+        <batchtest todir="${test.build.dir}" if="testcase">
+            <fileset dir="${src.test}">
+            	<include name="**/${testcase}.java"/>
+            	<exclude name="**/TestProxyFilter.java"/>
+            	<exclude name="**/TestProxyUtil.java"/>
+            </fileset>
+         </batchtest>
+    </junit>    	
+    <fail if="tests.failed">Tests failed!</fail>
+	</target>
+
+
+	<target name="test-cactus" depends="compile,compile-test,cactifywar" if="startCactus">
+		<exec executable="${env.JAVA_HOME}/bin/java" outputproperty="cargo.servlet.admin.port">
+	    <arg line="-cp ${build.test} org.apache.hadoop.hdfsproxy.FindFreePort -random"/>
+		</exec>
+		<exec executable="${env.JAVA_HOME}/bin/java" outputproperty="cargo.servlet.http.port">
+	    <arg line="-cp ${build.test} org.apache.hadoop.hdfsproxy.FindFreePort ${cargo.servlet.admin.port}"/>
+		</exec>
+		<exec executable="${env.JAVA_HOME}/bin/java" outputproperty="cargo.servlet.https.port">
+	    <arg line="-cp ${build.test} org.apache.hadoop.hdfsproxy.FindFreePort ${cargo.servlet.http.port}"/>
+		</exec>
+		
+		<echo> Free Ports: startup-${cargo.servlet.admin.port} / http-${cargo.servlet.http.port}
/ https-${cargo.servlet.https.port}</echo>
 	  <echo>Please take a deep breath while Cargo gets the Tomcat for running the servlet
tests...</echo>
 	  
 	  <mkdir dir="${tomcatconfig.dir}" />
@@ -143,75 +226,68 @@
 	  <mkdir dir="${tomcatconfig.dir}/temp" />
 	  <mkdir dir="${logs.dir}" />
 	  <mkdir dir="${reports.dir}" />	  
-	  <copy file="${tomcat.conf.test}/server.xml" tofile="${tomcatconfig.dir}/conf/server.xml"/>
+	  <copy file="${tomcat.conf.test}/server.xml" tofile="${tomcatconfig.dir}/conf/server.xml"
overwrite="true">
+		  <filterset>
+		    <filter token="ADMIN.PORT" value="${cargo.servlet.admin.port}"/>
+		    <filter token="HTTP.PORT" value="${cargo.servlet.http.port}"/>
+		    <filter token="HTTPS.PORT" value="${cargo.servlet.https.port}"/>
+		  </filterset>
+		</copy>		
 		<copy file="${tomcat.conf.test}/web.xml" tofile="${tomcatconfig.dir}/conf/web.xml"/>
 		<copy file="${tomcat.conf.test}/tomcat-users.xml" tofile="${tomcatconfig.dir}/conf/tomcat-users.xml"/>
-        
-		<cactus warfile="${target.dir}/${cactus.warfile.name}.war" printsummary="yes" failureproperty="tests.failed">
+	
+		<cactus warfile="${target.dir}/${cactus.warfile.name}.war" fork="yes" haltonfailure="no"
printsummary="yes" failureproperty="tests.failed">
 			<classpath>
 				<path refid="cactus.classpath"/>
 				<pathelement location="${build.classes}"/>
 				<pathelement location="${proxy.conf.dir}"/>
 				<pathelement location="${src.test.resources}"/>
 				<pathelement location="${src.test.resources}/proxy-config"/>
-			</classpath>		
-		
+			</classpath>			
 			<containerset>
-				<cargo containerId="${tomcat.container.id}" output="${logs.dir}/output.log" log="${logs.dir}/cargo.log">
+				<cargo containerId="${tomcat.container.id}" timeout="30000" output="${logs.dir}/output.log"
log="${logs.dir}/cargo.log">
 				 <zipUrlInstaller
             installUrl="http://apache.osuosl.org/tomcat/tomcat-6/v6.0.18/bin/apache-tomcat-6.0.18.zip"
             installDir="${target.dir}/${tomcat.container.id}"/>
-				  <!--<configuration type="standalone" home="${tomcatconfig.dir}">-->
 				  <configuration type="existing" home="${tomcatconfig.dir}">
-						<property name="cargo.servlet.port" value="${cargo.servlet.port}"/>
+						<property name="cargo.servlet.port" value="${cargo.servlet.http.port}"/>
 						<property name="cargo.logging" value="${cargo.logging}"/>
 						<property name="cactus.toDir" value="${build.test}"/>
 						<deployable type="war" file="${target.dir}/${cactus.warfile.name}.war"/>
 					</configuration>
 				</cargo>
 			</containerset>
+			<sysproperty key="test.build.data" value="${build.test}/data"/>
+      <sysproperty key="build.test" value="${build.test}"/>
+      <sysproperty key="build.target" value="${target.dir}"/>
 			<sysproperty key="javax.net.ssl.trustStore" value="${ssl.keystore.proxy}"/>
 			<sysproperty key="javax.net.ssl.trustStorePassword" value="changeme"/>
 			<sysproperty key="javax.net.ssl.keyStore.proxy" value="${ssl.keystore.proxy}"/>
 			<sysproperty key="javax.net.ssl.keyStore" value="${ssl.keystore.client}"/>
 			<sysproperty key="javax.net.ssl.keyStorePassword" value="changeme"/>
-			<sysproperty key="javax.net.ssl.keyPassword" value="changeme"/>
-			
-			<sysproperty key="javax.net.ssl.clientCert" value="${ssl.client.cert}"/>
-			
-			<sysproperty key="test.build.data" value="${build.test}/data"/>
-      <sysproperty key="build.test" value="${build.test}"/>
-      <sysproperty key="build.target" value="${target.dir}"/>
-      
+			<sysproperty key="javax.net.ssl.keyPassword" value="changeme"/>			
+			<sysproperty key="javax.net.ssl.clientCert" value="${ssl.client.cert}"/>     
       <sysproperty key="test.proxy.conf.dir" value="${proxy.conf.test}"/>
-      <!-- requires fork=yes for: 
-        relative File paths to use the specified user.dir 
-        classpath to use build/contrib/*.jar
-      -->
-      <sysproperty key="user.dir" value="${build.test}/data"/>
+      <sysproperty key="test.proxy.https.port" value="${cargo.servlet.https.port}"/>
       
-      <sysproperty key="fs.default.name" value="${fs.default.name}"/>
-      <sysproperty key="hadoop.test.localoutputfile" value="${hadoop.test.localoutputfile}"/>
-      <sysproperty key="hadoop.log.dir" value="${hadoop.log.dir}"/> 
-      
-			
 			<formatter type="${cactus.formatter.type}"/>
 			<batchtest todir="${reports.dir}" unless="testcase">
 				<fileset dir="${src.test}">
 					<include name="**/Test*.java"/>
+					<exclude name="**/TestHdfsProxy.java"/>
+					<exclude name="**/TestProxyUgiManager.java"/>
 				</fileset>
 			</batchtest>
 			<batchtest todir="${reports.dir}" if="testcase">
-        <fileset dir="${src.test}" includes="**/${testcase}.java"/>
+        <fileset dir="${src.test}">
+        	<include name="**/${testcase}.java"/>
+        	<exclude name="**/TestHdfsProxy.java"/>
+        	<exclude name="**/TestProxyUgiManager.java"/>
+        </fileset>
       </batchtest>
-		</cactus>
-		<junitreport todir="${reports.dir}">
-			<fileset dir="${target.dir}/${tomcat.container.id}" includes="TEST-*.xml"/>
-      <report todir="${reports.dir}" format="frames"/>
-    </junitreport>
+		</cactus>		
     <fail if="tests.failed">Tests failed!</fail>
 	</target>
-
 	<!-- ====================================================== -->
 	<!-- Macro definitions                                      -->
 	<!-- ====================================================== -->
@@ -347,6 +423,7 @@
     <pathelement location="${hadoop.root}/build/classes"/>
     <pathelement location="${hadoop.root}/build/tools"/>
     <pathelement location="${build.examples}"/>
+    <pathelement path="${clover.jar}"/>
     <path refid="contrib-classpath"/>
   </path>
   

Added: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/FindFreePort.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/FindFreePort.java?rev=752682&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/FindFreePort.java
(added)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/FindFreePort.java
Wed Mar 11 23:02:26 2009
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfsproxy;
+
+import java.net.ServerSocket;
+import java.io.IOException;
+import java.net.BindException;
+import java.util.Random;
+
+public class FindFreePort {
+  private static final int MIN_AVAILABLE_PORT = 10000;
+  private static final int MAX_AVAILABLE_PORT = 65535;
+  private static Random random = new Random();
+  /**
+   * 
+   * @param num <= 0, find a single free port
+   * @return free port next to port (>port)
+   * @throws IOException
+   */
+  public static int findFreePort(int port) throws IOException {
+    ServerSocket server;
+    if (port < 0) {
+      server =  new ServerSocket(0);      
+    } else {
+      int freePort = port+1;
+      while (true) {
+        try {
+          server =  new ServerSocket(freePort);
+          break;
+        } catch (IOException e) {
+          if (e instanceof BindException) {
+            if (freePort >= MAX_AVAILABLE_PORT || 
+                freePort < MIN_AVAILABLE_PORT) {
+              throw e;
+            }
+          } else {
+            throw e;
+          }
+          freePort += 1;
+        }
+      }
+    }
+    int fport = server.getLocalPort();
+    server.close();
+    return fport;    
+  }
+ /**
+  * 
+  * @return
+  * @throws IOException
+  */
+  public static int findFreePortRandom() throws IOException {
+    return findFreePort(MIN_AVAILABLE_PORT + random.nextInt(MAX_AVAILABLE_PORT - MIN_AVAILABLE_PORT
+ 1));
+  }
+   
+
+  public static void main(String[] args) throws Exception {
+    if(args.length < 1) {       
+      System.err.println("Usage: FindFreePort < -random / <#port> >");      
 
+      System.exit(0);      
+    }
+    int j = 0;
+    String cmd = args[j++];
+    if ("-random".equals(cmd)) {
+      System.out.println(findFreePortRandom());
+    } else {
+      System.out.println(findFreePort(Integer.parseInt(cmd)));
+    }   
+  }
+        
+}

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyUtil.java?rev=752682&r1=752681&r2=752682&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyUtil.java
(original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestProxyUtil.java
Wed Mar 11 23:02:26 2009
@@ -26,13 +26,15 @@
 public class TestProxyUtil extends TestCase {
   
   private static String TEST_PROXY_CONF_DIR = System.getProperty("test.proxy.conf.dir", "./conf");
+  private static String TEST_PROXY_HTTPS_PORT = System.getProperty("test.proxy.https.port",
"8443");
 
   public void testSendCommand() throws Exception {
       
     Configuration conf = new Configuration(false);  
     conf.addResource("ssl-client.xml");
     conf.addResource("hdfsproxy-default.xml");
-    conf.set("hdfsproxy.https.address", "localhost:8443");
+    String address = "localhost:" + TEST_PROXY_HTTPS_PORT;
+    conf.set("hdfsproxy.https.address", address);
     String hostFname = TEST_PROXY_CONF_DIR + "/hdfsproxy-hosts";
     conf.set("hdfsproxy.hosts", hostFname);    
     

Modified: hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-config/server.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-config/server.xml?rev=752682&r1=752681&r2=752682&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-config/server.xml (original)
+++ hadoop/core/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-config/server.xml Wed
Mar 11 23:02:26 2009
@@ -15,17 +15,17 @@
    limitations under the License.
 -->
 
-<Server port="8205" shutdown="SHUTDOWN" debug="4">
+<Server port="@ADMIN.PORT@" shutdown="SHUTDOWN" debug="4">
 
   <Service name="Catalina" debug="4">
 
-    <Connector port="8087"
+    <Connector port="@HTTP.PORT@"
         maxThreads="150" minSpareThreads="25" maxSpareThreads="75"
         enableLookups="false" redirectPort="8443" acceptCount="100"
         connectionTimeout="20000" disableUploadTimeout="true"
         scheme="http" secure="false"
         debug="4"/>
-    <Connector port="8443" protocol="HTTP/1.1" SSLEnabled="true"
+    <Connector port="@HTTPS.PORT@" protocol="HTTP/1.1" SSLEnabled="true"
                maxThreads="150" scheme="https" secure="true" 
                keystoreFile="${javax.net.ssl.keyStore.proxy}" keystorePass="changeme" keystoreType="JKS"
clientAuth="true" sslProtocol="TLS" />
 



Mime
View raw message