pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1144763 - in /pig/trunk: CHANGES.txt test/e2e/harness/Makefile test/e2e/harness/build.xml test/e2e/pig/Makefile test/e2e/pig/build.xml test/e2e/pig/tests/multiquery.conf test/e2e/pig/udfs/java/build.xml
Date Sat, 09 Jul 2011 23:58:44 GMT
Author: gates
Date: Sat Jul  9 23:58:43 2011
New Revision: 1144763

URL: http://svn.apache.org/viewvc?rev=1144763&view=rev
Log:
PIG-2121 e2e test harness should use ant instead of make

Added:
    pig/trunk/test/e2e/harness/build.xml
    pig/trunk/test/e2e/pig/build.xml
Removed:
    pig/trunk/test/e2e/harness/Makefile
    pig/trunk/test/e2e/pig/Makefile
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/test/e2e/pig/tests/multiquery.conf
    pig/trunk/test/e2e/pig/udfs/java/build.xml

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1144763&r1=1144762&r2=1144763&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Jul  9 23:58:43 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2121: e2e test harness should use ant instead of make (gates)
+
 PIG-2142: Allow registering multiple jars from DFS via single statement (rangadi via dvryaboy)
 
 PIG-1926: Sample/Limit should take scalar (azaroth via thejas)

Added: pig/trunk/test/e2e/harness/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/build.xml?rev=1144763&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/build.xml (added)
+++ pig/trunk/test/e2e/harness/build.xml Sat Jul  9 23:58:43 2011
@@ -0,0 +1,99 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="TestHarness" default="tar">
+
+  <property name="tar.dir" value="${basedir}/tar"/>
+  <property name="pig.testharness.dir" value="${basedir}"/>
+  <property name="tar.name" value="pigharness.tar"/>
+  <property name="test.dist.dir" value="${basedir}/testdist"/>
+  <property name="test.dist.conf.dir" value="${test.dist.dir}/conf"/>
+  <property name="test.dir" value="${basedir}/test"/>
+  <property name="test.testfiles" value="${test.dir}/tests"/>
+  <property name="test.conf" value="${test.dir}/conf"/>
+  <property name="test.src" value="${basedir}"/>
+
+  <target name="tar"
+      description="Create tar file with test harness modules and script">
+    <mkdir dir="${tar.dir}"/>
+    <copy todir="${tar.dir}">
+        <fileset dir="${pig.testharness.dir}">
+            <include name="Insert2Mysql.pm"/>
+            <include name="TestDeployerFactory.pm"/>
+            <include name="TestDeployer.pm"/>
+            <include name="TestDriverFactory.pm"/>
+            <include name="TestDriver.pm"/>
+            <include name="TestReport.pm"/>
+            <include name="Log.pm"/>
+            <include name="Properties.pm"/>
+            <include name="pig_test_harness.pl"/>
+        </fileset>
+    </copy>
+
+    <tar destfile="${tar.name}" basedir="${tar.dir}"/>
+  </target>
+
+  <target name="test-init" depends="tar">
+    <mkdir dir="${test.dist.dir}"/>
+    <mkdir dir="${test.dist.conf.dir}"/>
+
+    <copy todir="${test.dist.dir}">
+      <fileset dir="${test.testfiles}">
+        <include name="*.conf"/>
+      </fileset>
+      <fileset dir="${test.src}">
+        <include name="TestDriverTest.pm"/>
+        <include name="TestDeployerTest.pm"/>
+      </fileset>
+    </copy>
+
+    <copy todir="${test.dist.conf.dir}">
+        <fileset file="${test.conf}/default.conf"/>
+    </copy>
+
+    <untar src="${tar.name}" dest="${test.dist.dir}"/>
+    <chmod file="${test.dist.dir}/pig_test_harness.pl" perm="u+x"/>
+  </target>
+
+  <!-- Test the ability to run tests -->
+  <target name="test-test" depends="test-init">
+    <exec executable="./pig_test_harness.pl" dir="${test.dist.dir}">
+        <env key="PIG_HARNESS_ROOT" value="."/>
+        <arg value="test.conf"/>
+    </exec>
+  </target>
+
+  <!-- Test the deployer interface -->
+  <target name="test-deploy" depends="test-init">
+    <exec executable="./pig_test_harness.pl" dir="${test.dist.dir}">
+        <env key="PIG_HARNESS_ROOT" value="."/>
+        <arg value="-deploycfg"/>
+        <arg value="deploy.conf"/>
+        <arg value="-deploy"/>
+        <arg value="-undeploy"/>
+        <arg value="test.conf"/>
+    </exec>
+  </target>
+
+  <target name="test" depends="test-test, test-deploy"/>
+
+  <target name="clean">
+    <delete dir="${test.dist.dir}"/>
+    <delete file="${tar.name}"/>
+    <delete dir="${tar.dir}"/>
+  </target>
+</project>

Added: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1144763&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (added)
+++ pig/trunk/test/e2e/pig/build.xml Sat Jul  9 23:58:43 2011
@@ -0,0 +1,231 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="TestHarnessPigTests" default="test">
+
+  <property name="pig.dir" value="${basedir}/../../.."/>
+  <property name="pig.jar" value="${pig.dir}/pig.jar"/>
+  <property name="jython.jar"
+    value="${pig.dir}/build/ivy/lib/Pig/jython-2.5.0.jar"/>
+
+  <!-- Separate property name for udfs' build.xml -->
+  <property name="pig.jarfile" value="${pig.jar}"/>
+  <property name="udf.dir" value="${basedir}/udfs"/>
+  <property name="udf.java.dir" value="${udf.dir}/java"/>
+  <property name="udf.jar" value="${udf.java.dir}/testudf.jar"/>
+  <property name="python.udf.dir" value="${udf.dir}/python"/>
+  <property name="params.dir" value="${basedir}/paramfiles"/>
+  <property name="lib.dir" value="${basedir}/lib"/>
+  <property name="streaming.dir" value="${basedir}/streaming"/>
+
+  <property name="tar.name" value="${basedir}/pigtests.tar"/>
+  <property name="tar.dir" value="${basedir}/tar"/>
+  <property name="test.src" value="${basedir}/tests"/>
+  <property name="driver.src" value="${basedir}/drivers"/>
+  <property name="deployer.src" value="${basedir}/deployers"/>
+  <property name="conf.src" value="${basedir}/conf"/>
+  <property name="tool.src" value="${basedir}/tools"/>
+
+  <property name="harness.dir" value="${basedir}/../harness"/>
+  <property name="harness.tar" value="${harness.dir}/pigharness.tar"/>
+
+  <property name="test.location" value="${basedir}/testdist"/>
+  <property name="benchmark.location" value="${test.location}/benchmarks"/>
+
+
+  <!-- Build the UDFs -->
+  <target name="pig-jar-available">
+    <available property="pig.jar.available" file="${pig.jar}"/>
+    <fail message="You must build the main Pig jar before running these tests"
+        unless="pig.jar.available"/>
+  </target>
+
+  <target name="udfs" depends="pig-jar-available">
+    <ant dir="${udf.java.dir}"/>
+  </target>
+
+  <!-- Build an archive to use in the tests -->
+  <target name="tar" description="Create tar file with pig modules">
+    <mkdir dir="${tar.dir}"/>
+    <mkdir dir="${tar.dir}/tests"/>
+    <mkdir dir="${tar.dir}/drivers"/>
+    <mkdir dir="${tar.dir}/deployers"/>
+    <mkdir dir="${tar.dir}/conf"/>
+    <mkdir dir="${tar.dir}/libexec"/>
+    <mkdir dir="${tar.dir}/libexec/PigTest"/>
+    <mkdir dir="${tar.dir}/libexec/PigTest/test"/>
+    <mkdir dir="${tar.dir}/libexec/PigTest/generate"/>
+    <mkdir dir="${tar.dir}/libexec/python"/>
+    <mkdir dir="${tar.dir}/lib"/>
+    <mkdir dir="${tar.dir}/lib/java"/>
+    <mkdir dir="${tar.dir}/paramfiles"/>
+
+    <copy todir="${tar.dir}/tests">
+        <fileset dir="${test.src}">
+          <exclude name="bigdata.conf"/>
+          <exclude name="grunt.conf"/>
+          <exclude name="streaming_local.conf"/>
+        </fileset>
+    </copy>
+
+    <copy todir="${tar.dir}">
+      <fileset dir="${driver.src}">
+        <exclude name="TestDriverScript.pm"/>
+      </fileset>
+      <fileset dir="${deployer.src}"/>
+    </copy>
+
+
+    <copy todir="${tar.dir}/conf">
+      <fileset dir="${conf.src}"/>
+    </copy>
+
+    <copy todir="${tar.dir}/libexec/PigTest">
+      <fileset dir="${tool.src}/test"/>
+      <fileset dir="${tool.src}/generate"/>
+    </copy>
+
+    <copy todir="${tar.dir}/lib">
+      <fileset dir="${lib.dir}"/>
+    </copy>
+
+    <copy todir="${tar.dir}/lib/java">
+      <fileset file="${udf.jar}"/>
+    </copy>
+
+    <copy todir="${tar.dir}/libexec">
+      <fileset dir="${streaming.dir}"/>
+    </copy>
+
+    <copy todir="${tar.dir}/libexec/python">
+      <fileset dir="${python.udf.dir}"/>
+    </copy>
+
+    <copy todir="${tar.dir}/paramfiles">
+      <fileset file="${params.dir}/params_3"/>
+    </copy>
+
+    <tar destfile="${tar.name}" basedir="${tar.dir}"/>
+  </target>
+
+  <!-- Get the tarball for the harness -->
+  <target name="build-harness">
+    <ant dir="${harness.dir}" inheritAll="false"/>
+  </target>
+
+  <!-- Check that the necessary properties are setup -->
+  <target name="property-check">
+    <fail message="Please set the property pig.harness.old.pig to the directory where
your old version of Pig is installed"
+      unless="pig.harness.old.pig"/>
+    <fail message="Please set the property pig.harness.cluster to the directory of your
hadoop installation"
+      unless="pig.harness.cluster"/>
+  </target>
+
+  <!-- Prep the test area -->
+  <target name="init-test" depends="build-harness">
+    <mkdir dir="${test.location}"/>
+    <mkdir dir="${benchmark.location}"/>
+
+    <untar src="${tar.name}" dest="${test.location}"/>
+    <untar src="${harness.tar}" dest="${test.location}"/>
+
+    <chmod perm="ugo+x" type="file">
+      <fileset dir="${test.location}/libexec" />
+      <fileset file="${test.location}/pig_test_harness.pl"/>
+    </chmod>
+
+  </target>
+
+
+  <target name="test" depends="property-check, udfs, tar, init-test">
+    <!-- If they have not specified tests to run then null it out -->
+    <property name="tests.to.run" value=""/>
+
+    <exec executable="./pig_test_harness.pl" dir="${test.location}">
+      <env key="PIG_HARNESS_ROOT" value="."/>
+      <env key="PH_LOCAL" value="."/>
+      <env key="PH_OUT" value="."/>
+      <env key="PH_ROOT" value="."/>
+      <env key="PH_PIG" value="${pig.dir}"/>
+      <env key="PH_OLDPIG" value="${pig.harness.old.pig}"/>
+      <env key="PH_CLUSTER" value="${pig.harness.cluster}"/>
+      <env key="PH_JYTHON_JAR" value="${jython.jar}"/>
+      <arg line="${tests.to.run}"/>
+      <arg value="${test.location}/tests/cmdline.conf"/>
+      <arg value="${test.location}/tests/multiquery.conf"/>
+      <arg value="${test.location}/tests/negative.conf"/>
+      <arg value="${test.location}/tests/nightly.conf"/>
+      <arg value="${test.location}/tests/streaming.conf"/>
+      <arg value="${test.location}/tests/turing_jython.conf"/>
+    </exec>
+  </target>
+
+  <target name="init-deploy">
+     <!-- For now default to the existing cluster deployer, since 
+    it's all there is.  Once the local deployer is available that
+    should be the default. -->
+    <property name="deploy.conf"
+        value="${test.location}/conf/existing_deployer.conf"/>
+  </target>
+
+  <target name="deploy-base" depends="property-check, tar, init-test, init-deploy">
+    <exec executable="./pig_test_harness.pl" dir="${test.location}">
+      <env key="PIG_HARNESS_ROOT" value="."/>
+      <env key="PH_LOCAL" value="."/>
+      <env key="PH_OUT" value="."/>
+      <env key="PH_ROOT" value="."/>
+      <env key="PH_PIG" value="${pig.dir}"/>
+      <env key="PH_OLDPIG" value="${pig.harness.old.pig}"/>
+      <env key="PH_CLUSTER" value="${pig.harness.cluster}"/>
+      <env key="PH_JYTHON_JAR" value="${jython.jar}"/>
+
+      <arg value="-deploycfg"/>
+      <arg value="${deploy.conf}"/>
+      <arg value="${deploy.opt}"/>
+      <!-- Give a bogus test so it just does the deployment -->
+      <arg value="-t"/>
+      <arg value="NoSuchTest"/>
+      <arg value="${test.location}/tests/nightly.conf"/>
+    </exec>
+  </target>
+
+  <target name="deploy">
+    <antcall target="deploy-base">
+      <param name="deploy.opt" value="-deploy"/>
+    </antcall>
+  </target>
+
+  <target name="undeploy">
+    <antcall target="deploy-base">
+      <param name="deploy.opt" value="-undeploy"/>
+    </antcall>
+  </target>
+
+  <target name="deploy-test" depends="deploy, test"/>
+
+  <target name="deploy-test-undeploy" depends="deploy, test, undeploy"/>
+
+  <target name="clean">
+    <delete dir="${test.location}"/>
+    <delete file="${tar.name}"/>
+    <delete dir="${tar.dir}"/>
+    <ant dir="${udf.java.dir}" target="clean"/>
+  </target>
+
+</project>
+
+

Modified: pig/trunk/test/e2e/pig/tests/multiquery.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/multiquery.conf?rev=1144763&r1=1144762&r2=1144763&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/multiquery.conf (original)
+++ pig/trunk/test/e2e/pig/tests/multiquery.conf Sat Jul  9 23:58:43 2011
@@ -282,7 +282,7 @@ $cfg = {
                         store c into ':OUTPATH:.1';
                         d = filter b by gpa > 3.0; 
                         store d into ':OUTPATH:.2';
-                        e = join c by gpa, d by gpa using "repl";
+                        e = join c by gpa, d by gpa using 'repl';
                         store e into ':OUTPATH:.3'; #,
             'sql' => "select name, age, gpa from studenttab10k where age > 50;
                       select name, age, gpa from studenttab10k where gpa > 3.0;

Modified: pig/trunk/test/e2e/pig/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/build.xml?rev=1144763&r1=1144762&r2=1144763&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/java/build.xml (original)
+++ pig/trunk/test/e2e/pig/udfs/java/build.xml Sat Jul  9 23:58:43 2011
@@ -35,6 +35,7 @@
 
     <target name="clean">
         <delete dir="${build.dir}" />
+        <delete file="${udf.jarfile}" />
     </target>
 
     <target name="udf-compile" depends="init, pig.jar.check">



Mime
View raw message