Return-Path: 
 <mapreduce-commits-return-1338-apmail-hadoop-mapreduce-commits-archive=hadoop.apache.org@hadoop.apache.org>
Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org
Received: (qmail 82592 invoked from network); 8 Mar 2011 05:57:51 -0000
Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3)
  by minotaur.apache.org with SMTP; 8 Mar 2011 05:57:51 -0000
Received: (qmail 61831 invoked by uid 500); 8 Mar 2011 05:57:51 -0000
Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org
Received: (qmail 61792 invoked by uid 500); 8 Mar 2011 05:57:51 -0000
Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:mapreduce-commits-help@hadoop.apache.org>
List-Unsubscribe: <mailto:mapreduce-commits-unsubscribe@hadoop.apache.org>
List-Post: <mailto:mapreduce-commits@hadoop.apache.org>
List-Id: <mapreduce-commits.hadoop.apache.org>
Reply-To: mapreduce-dev@hadoop.apache.org
Delivered-To: mailing list mapreduce-commits@hadoop.apache.org
Received: (qmail 61783 invoked by uid 99); 8 Mar 2011 05:57:51 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 05:57:51 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=5.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 05:57:50 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id 8C4A02388A66; Tue,  8 Mar 2011 05:57:25 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1079220 - in /hadoop/mapreduce/branches/yahoo-merge/src:
 contrib/build-contrib.xml contrib/build.xml contrib/streaming/build.xml
 test/aop/build/aop.xml
Date: Tue, 08 Mar 2011 05:57:25 -0000
To: mapreduce-commits@hadoop.apache.org
From: omalley@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20110308055725.8C4A02388A66@eris.apache.org>

Author: omalley
Date: Tue Mar  8 05:57:25 2011
New Revision: 1079220

URL: http://svn.apache.org/viewvc?rev=1079220&view=rev
Log:
commit 7b91419a3dd35dee878f90a2f1629a5998b5bc03
Author: Vinay Kumar Thota <vinayt@yahoo-inc.com>
Date:   Thu Dec 16 19:58:35 2010 -0800

    MAPREDUCE:1809 from https://issues.apache.org/jira/secure/attachment/12457421/MAPREDUCE-1809.patch

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/build-contrib.xml
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/build.xml
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/streaming/build.xml
    hadoop/mapreduce/branches/yahoo-merge/src/test/aop/build/aop.xml

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/build-contrib.xml?rev=1079220&r1=1079219&r2=1079220&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/build-contrib.xml (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/build-contrib.xml Tue Mar  8 05:57:25 2011
@@ -36,9 +36,16 @@
   <property name="src.test" location="${root}/src/test"/>
   <property name="src.test.data" location="${root}/src/test/data"/>
   <property name="src.examples" location="${root}/src/examples"/>
+  <property name="build-fi.dir" location="${hadoop.root}/build-fi"/>
+  <property name="system-test-build-dir" location="${build-fi.dir}/system"/>
+  <!-- Property added for contrib system tests -->
+  <property name="src.test.system" location="${root}/src/test/system"/>
 
   <available file="${src.examples}" type="dir" property="examples.available"/>
   <available file="${src.test}" type="dir" property="test.available"/>
+  <!-- Property added for contrib system tests -->
+  <available file="${src.test.system}" type="dir" 
+       property="test.system.available"/>
 
   <property name="conf.dir" location="${hadoop.root}/conf"/>
   <property name="test.junit.output.format" value="plain"/>
@@ -63,6 +70,10 @@
   <property name="dest.jar" value="hadoop-${version}-${name}.jar"/>
 
   <fileset id="lib.jars" dir="${root}" includes="lib/*.jar"/>
+  <!-- Property added for contrib system tests -->
+  <property name="build.test.system" location="${build.dir}/system"/>
+  <property name="build.system.classes" 
+       location="${build.test.system}/classes"/>
 
 
    <!-- IVY properties set here -->
@@ -87,10 +98,15 @@
     <pathelement location="${build.classes}"/>
     <fileset refid="lib.jars"/>
     <pathelement location="${hadoop.root}/build/classes"/>
+    <pathelement location="${system-test-build-dir}/classes"/>
+    <pathelement location="${system-test-build-dir}/tools"/>
     <pathelement location="${hadoop.root}/build/tools"/>
     <fileset dir="${hadoop.root}/lib">
       <include name="**/*.jar" />
     </fileset>
+    <fileset dir="${build.ivy.lib.dir}">
+      <include name="**/*.jar" />
+    </fileset>
     <path refid="${ant.project.name}.common-classpath"/>
     <pathelement path="${clover.jar}"/>
   </path>
@@ -112,6 +128,36 @@
     <path refid="contrib-classpath"/>
   </path>
 
+  <!-- The system test classpath -->
+  <path id="test.system.classpath">
+      <pathelement location="${hadoop.root}/src/contrib/${name}/src/test/system" />
+      <pathelement location="${build.test.system}" />
+      <pathelement location="${build.test.system}/classes"/>
+      <pathelement location="${build.examples}"/>
+      <pathelement location="${system-test-build-dir}/classes" />
+      <pathelement location="${system-test-build-dir}/test/mapred/classes" />
+      <pathelement location="${system-test-build-dir}" />
+      <pathelement location="${system-test-build-dir}/tools" />
+      <pathelement location="${hadoop.home}"/>
+      <pathelement location="${hadoop.conf.dir}"/>
+      <pathelement location="${hadoop.conf.dir.deployed}"/>
+      <pathelement location="${hadoop.root}/build"/>
+      <pathelement location="${hadoop.root}/build/examples"/>
+      <pathelement location="${hadoop.root}/build/test/classes" />
+      <path refid="contrib-classpath"/>
+      <fileset dir="${system-test-build-dir}">
+          <include name="**/*.jar" />
+          <exclude name="**/excluded/" />
+      </fileset>
+      <fileset dir="${system-test-build-dir}/test/mapred/testjar">
+          <include name="**/*.jar" />
+          <exclude name="**/excluded/" />
+      </fileset>
+      <fileset dir="${hadoop.root}/build/contrib/${name}">
+          <include name="**/*.jar" />
+          <exclude name="**/excluded/" />
+      </fileset>
+  </path>
 
   <!-- to be overridden by sub-projects -->
   <target name="check-contrib"/>
@@ -128,6 +174,9 @@
     <mkdir dir="${build.test}/extraconf"/>
     <mkdir dir="${build.examples}"/>
     <mkdir dir="${hadoop.log.dir}"/>
+    <!-- The below two tags  added for contrib system tests -->
+    <mkdir dir="${build.test.system}"/>
+    <mkdir dir="${build.system.classes}"/> 
     <antcall target="init-contrib"/>
   </target>
 
@@ -141,6 +190,7 @@
      encoding="${build.encoding}"
      srcdir="${src.dir}"
      includes="**/*.java"
+     excludes="system/**/*.java"
      destdir="${build.classes}"
      debug="${javac.debug}"
      deprecation="${javac.deprecation}">
@@ -174,12 +224,28 @@
      encoding="${build.encoding}"
      srcdir="${src.test}"
      includes="**/*.java"
+     excludes="system/**/*.java"
      destdir="${build.test}"
      debug="${javac.debug}">
     <classpath refid="test.classpath"/>
     </javac>
   </target>
   
+  <!-- ================================================================== -->
+  <!-- Compile system test code                                           -->
+  <!-- ================================================================== -->
+  <target name="compile-test-system" depends="compile-examples, ivy-retrieve-test" 
+       if="test.system.available">
+     <echo message="contrib: ${name}"/>
+     <javac
+         encoding="${build.encoding}"
+         srcdir="${src.test.system}"
+         includes="**/*.java"
+         destdir="${build.system.classes}"
+         debug="${javac.debug}">
+        <classpath refid="test.system.classpath"/>
+    </javac>
+  </target>
 
   <!-- ====================================================== -->
   <!-- Make a Hadoop contrib's jar                            -->
@@ -264,15 +330,133 @@
       <formatter type="${test.junit.output.format}" />
       <batchtest todir="${build.test}" unless="testcase">
         <fileset dir="${src.test}"
-                 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+                 includes="**/Test*.java" excludes="**/${test.exclude}.java, system/**/*.java" />
       </batchtest>
       <batchtest todir="${build.test}" if="testcase">
-        <fileset dir="${src.test}" includes="**/${testcase}.java"/>
+        <fileset dir="${src.test}" includes="**/${testcase}.java" excludes="system/**/*.java" />
       </batchtest>
     </junit>
     <antcall target="checkfailure"/>
   </target>
 
+  <!-- ================================================================== -->
+  <!-- Run system tests                                                   -->
+  <!-- ================================================================== -->
+  <target name="test-system" depends="compile-test-system, jar"
+       if="test.system.available">
+      <delete dir="${build.test.system}/extraconf"/>
+      <mkdir dir="${build.test.system}/extraconf"/>
+     <property name="test.src.dir" location="${hadoop.root}/src/test"/>
+     <property name="test.junit.printsummary" value="yes" />
+     <property name="test.junit.haltonfailure" value="no" />
+     <property name="test.junit.maxmemory" value="512m" />
+     <property name="test.junit.fork.mode" value="perTest" />
+     <property name="test.all.tests.file" value="${test.src.dir}/all-tests" />
+     <property name="test.build.dir" value="${hadoop.root}/build/test"/>
+     <property name="basedir" value="${hadoop.root}"/>
+     <property name="test.timeout" value="900000"/>
+     <property name="test.junit.output.format" value="plain"/>
+     <property name="test.tools.input.dir" value="${basedir}/src/test/tools/data"/>
+     <property name="c++.src" value="${basedir}/src/c++"/>
+     <property name="test.include" value="Test*"/>
+     <property name="c++.libhdfs.src" value="${c++.src}/libhdfs"/>
+     <property name="test.build.data" value="${build.test.system}/data"/>
+     <property name="test.cache.data" value="${build.test.system}/cache"/>
+     <property name="test.debug.data" value="${build.test.system}/debug"/>
+     <property name="test.log.dir" value="${build.test.system}/logs"/>
+         <exec executable="sed" inputstring="${os.name}"
+              outputproperty="nonspace.os">
+            <arg value="s/ /_/g"/>
+         </exec>
+     <property name="build.platform"
+          value="${nonspace.os}-${os.arch}-${sun.arch.data.model}"/>
+     <property name="build.native" 
+          value="${hadoop.root}/build/native/${build.platform}"/>
+     <property name="lib.dir" value="${hadoop.root}/lib"/>
+     <property name="install.c++.examples"
+          value="${hadoop.root}/build/c++-examples/${build.platform}"/>
+     <condition property="tests.testcase">
+         <and>
+           <isset property="testcase" />
+         </and>
+  </condition>
+  <property name="test.junit.jvmargs" value="-ea" />
+  <macro-system-test-runner test.file="${test.all.tests.file}"
+                     classpath="test.system.classpath"
+                     test.dir="${build.test.system}"
+                     fileset.dir="${hadoop.root}/src/contrib/${name}/src/test/system"
+                     hadoop.conf.dir.deployed="${hadoop.conf.dir.deployed}">
+  </macro-system-test-runner>
+</target>
+<macrodef name="macro-system-test-runner">
+    <attribute name="test.file" />
+    <attribute name="classpath" />
+    <attribute name="test.dir" />
+    <attribute name="fileset.dir" />
+    <attribute name="hadoop.conf.dir.deployed" default="" />    
+    <sequential>
+      <delete dir="@{test.dir}/data"/>
+      <mkdir dir="@{test.dir}/data"/>
+      <delete dir="@{test.dir}/logs"/>
+      <mkdir dir="@{test.dir}/logs"/>
+      <copy file="${test.src.dir}/hadoop-policy.xml"
+        todir="@{test.dir}/extraconf" />
+      <copy file="${test.src.dir}/fi-site.xml"
+        todir="@{test.dir}/extraconf" />
+      <junit showoutput="${test.output}"
+        printsummary="${test.junit.printsummary}"
+        haltonfailure="${test.junit.haltonfailure}"
+        fork="yes"
+        forkmode="${test.junit.fork.mode}"
+        maxmemory="${test.junit.maxmemory}"
+        dir="${basedir}" timeout="${test.timeout}"
+        errorProperty="tests.failed" failureProperty="tests.failed">
+        <jvmarg value="${test.junit.jvmargs}" />
+        <sysproperty key="test.build.data" value="@{test.dir}/data"/>
+        <sysproperty key="test.tools.input.dir" value = "${test.tools.input.dir}"/>
+        <sysproperty key="test.cache.data" value="${test.cache.data}"/>
+        <sysproperty key="test.debug.data" value="${test.debug.data}"/>
+        <sysproperty key="hadoop.log.dir" value="@{test.dir}/logs"/>
+        <sysproperty key="test.src.dir" value="@{fileset.dir}"/>
+        <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
+        <sysproperty key="taskcontroller-ugi" value="${taskcontroller-ugi}"/>
+        <sysproperty key="test.build.extraconf" value="@{test.dir}/extraconf" />
+        <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
+        <sysproperty key="java.library.path"
+          value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
+        <sysproperty key="install.c++.examples" value="${install.c++.examples}"/>
+        <syspropertyset dynamic="no">
+          <propertyref name="hadoop.tmp.dir"/>
+        </syspropertyset>
+        <!-- set compile.c++ in the child jvm only if it is set -->
+        <syspropertyset dynamic="no">
+          <propertyref name="compile.c++"/>
+        </syspropertyset>
+
+        <!-- Pass probability specifications to the spawn JVM -->
+        <syspropertyset id="FaultProbabilityProperties">
+          <propertyref regex="fi.*"/>
+        </syspropertyset>
+        <sysproperty key="test.system.hdrc.deployed.hadoopconfdir"
+                     value="@{hadoop.conf.dir.deployed}" />
+        <classpath refid="@{classpath}"/>
+        <formatter type="${test.junit.output.format}" />
+        <batchtest todir="@{test.dir}" unless="testcase">
+          <fileset dir="@{fileset.dir}"
+            excludes="**/${test.exclude}.java aop/** system/**">
+            <patternset>
+              <includesfile name="@{test.file}"/>
+            </patternset>
+          </fileset>
+        </batchtest>
+        <batchtest todir="@{test.dir}" if="testcase">
+          <fileset dir="@{fileset.dir}" includes="**/${testcase}.java"/>
+        </batchtest>
+      </junit>
+      <antcall target="checkfailure"/>
+    </sequential>
+  </macrodef>
+
   <target name="docs" depends="forrest.check" description="Generate forrest-based documentation. To use, specify -Dforrest.home=&lt;base of Apache Forrest installation&gt; on the command line." if="forrest.home">
 	<!-- Nothing by default -->
   </target>

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/build.xml?rev=1079220&r1=1079219&r2=1079220&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/build.xml (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/build.xml Tue Mar  8 05:57:25 2011
@@ -73,6 +73,29 @@
     <fail if="testsfailed">Tests failed!</fail>
   </target>
 
+  <!-- ====================================================== -->
+  <!-- Test all the contrib system tests                     -->
+  <!-- ====================================================== -->
+  <target name="test-system-contrib">
+      <property name="hadoop.root" location="${root}/../../../"/>
+      <property name="build.contrib.dir" location="${hadoop.root}/build/contrib"/>
+      <delete file="${build.contrib.dir}/testsfailed"/>
+      <subant target="test-system">
+          <property name="continueOnFailure" value="true"/>
+          <property name="hadoop.home" value="${hadoop.home}"/>
+          <property name="hadoop.conf.dir" value="${hadoop.conf.dir}"/>
+          <property name="hadoop.conf.dir.deployed"
+               value="${hadoop.conf.dir.deployed}"/>
+          <fileset dir="." includes="hdfsproxy/build.xml"/>
+          <fileset dir="." includes="streaming/build.xml"/>
+          <fileset dir="." includes="fairscheduler/build.xml"/>
+         <fileset dir="." includes="capacity-scheduler/build.xml"/>
+         <fileset dir="." includes="gridmix/build.xml"/>
+      </subant>
+      <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
+      <fail if="testsfailed">Tests failed!</fail>
+  </target>
+
   <target name="docs">
     <subant target="docs">
       <fileset dir="." includes="capacity-scheduler/build.xml"/> 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/streaming/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/streaming/build.xml?rev=1079220&r1=1079219&r2=1079220&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/streaming/build.xml (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/streaming/build.xml Tue Mar  8 05:57:25 2011
@@ -43,4 +43,10 @@ to call at top-level: ant deploy-contrib
    </antcall>
   </target>  
  
+  <!--Run all system tests.-->
+  <target name="test-system">
+    <antcall target="hadoopbuildcontrib.test-system">
+    </antcall>
+  </target>
+
 </project>

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/aop/build/aop.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/aop/build/aop.xml?rev=1079220&r1=1079219&r2=1079220&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/aop/build/aop.xml (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/aop/build/aop.xml Tue Mar  8 05:57:25 2011
@@ -139,6 +139,14 @@
   <target name="test-system" depends="ivy-retrieve-common, ivy-retrieve-system"
     description="Run system tests">
     <subant buildpath="build.xml" target="jar-test-system"/>
+    <subant target="test-system-contrib">
+        <property name="hadoop.home" value="${hadoop.home}"/>
+       <property name="hadoop.conf.dir" value="${hadoop.conf.dir}"/>
+       <property name="version" value="${version}"/>
+       <property name="hadoop.conf.dir.deployed" 
+            value="${hadoop.conf.dir.deployed}"/>
+       <fileset file="${contrib.dir}/build.xml"/>
+    </subant>
     <macro-test-runner test.file="${test.mapred.all.tests.file}"
                        suite.type="system/test"
                        classpath="test.system.classpath"