Return-Path: 
 <pig-commits-return-3034-apmail-hadoop-pig-commits-archive=hadoop.apache.org@hadoop.apache.org>
Delivered-To: apmail-hadoop-pig-commits-archive@www.apache.org
Received: (qmail 22939 invoked from network); 9 Jun 2010 18:38:42 -0000
Received: from unknown (HELO mail.apache.org) (140.211.11.3)
  by 140.211.11.9 with SMTP; 9 Jun 2010 18:38:42 -0000
Received: (qmail 28372 invoked by uid 500); 9 Jun 2010 18:38:42 -0000
Delivered-To: apmail-hadoop-pig-commits-archive@hadoop.apache.org
Received: (qmail 28330 invoked by uid 500); 9 Jun 2010 18:38:42 -0000
Mailing-List: contact pig-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:pig-commits-help@hadoop.apache.org>
List-Unsubscribe: <mailto:pig-commits-unsubscribe@hadoop.apache.org>
List-Post: <mailto:pig-commits@hadoop.apache.org>
List-Id: <pig-commits.hadoop.apache.org>
Reply-To: pig-dev@hadoop.apache.org
Delivered-To: mailing list pig-commits@hadoop.apache.org
Received: (qmail 28323 invoked by uid 500); 9 Jun 2010 18:38:42 -0000
Delivered-To: apmail-incubator-pig-commits@incubator.apache.org
Received: (qmail 28320 invoked by uid 99); 9 Jun 2010 18:38:42 -0000
Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Jun 2010 18:38:42 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Jun 2010 18:38:38 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id 4207823889CB; Wed,  9 Jun 2010 18:37:55 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r953109 - in /hadoop/pig/trunk: CHANGES.txt build.xml
 test/org/apache/pig/test/TestJoinSmoke.java test/smoke-tests test/unit-tests
Date: Wed, 09 Jun 2010 18:37:55 -0000
To: pig-commits@incubator.apache.org
From: olga@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20100609183755.4207823889CB@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Author: olga
Date: Wed Jun  9 18:37:54 2010
New Revision: 953109

URL: http://svn.apache.org/viewvc?rev=953109&view=rev
Log:
PIG-1441: new test targets (olgan)

Added:
    hadoop/pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java
    hadoop/pig/trunk/test/smoke-tests
    hadoop/pig/trunk/test/unit-tests
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/build.xml

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=953109&r1=953108&r2=953109&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Jun  9 18:37:54 2010
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-1441: new test targets (olgan)
+
 PIG-282: Custom Partitioner (aniket486 via daijy)
 
 PIG-283: Allow to set arbitrary jobconf key-value pairs inside pig program (hashutosh)

Modified: hadoop/pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=953109&r1=953108&r2=953109&view=diff
==============================================================================
--- hadoop/pig/trunk/build.xml (original)
+++ hadoop/pig/trunk/build.xml Wed Jun  9 18:37:54 2010
@@ -80,6 +80,8 @@
     <property name="test.timeout" value="2700000" />
     <property name="test.junit.output.format" value="plain" />
     <property name="test.commit.file" value="${test.src.dir}/commit-tests"/>
+    <property name="test.unit.file" value="${test.src.dir}/unit-tests"/>
+    <property name="test.smoke.file" value="${test.src.dir}/smoke-tests"/>
     <property name="test.all.file" value="${test.src.dir}/all-tests"/>
 
 
@@ -536,6 +538,14 @@
         <macro-test-runner test.file="${test.commit.file}" />
     </target>
 
+    <target name="test-unit" depends="compile-test,jar-withouthadoop" description="Run all true unit tests">
+        <macro-test-runner test.file="${test.unit.file}" />
+    </target>
+
+    <target name="test-smoke" depends="compile-test,jar-withouthadoop" description="Run 30 min smoke tests">
+        <macro-test-runner test.file="${test.smoke.file}" />
+    </target>
+
     <macrodef name="macro-test-runner">
       <attribute name="test.file" />
       <sequential>

Added: hadoop/pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java?rev=953109&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java Wed Jun  9 18:37:54 2010
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.ExecType;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.test.utils.TestHelper;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class TestJoinSmoke extends TestCase{
+    private static final String FR_INPUT_FILE = "testFrJoinInput.txt";
+
+    private static final String SKEW_INPUT_FILE1 = "SkewedJoinInput1.txt";
+    private static final String SKEW_INPUT_FILE2 = "SkewedJoinInput2.txt";
+    private static final String SKEW_INPUT_FILE5 = "SkewedJoinInput5.txt";
+
+    private PigServer pigServer;
+    private static MiniCluster cluster = MiniCluster.buildCluster();
+    private File tmpFile;
+    
+    public TestJoinSmoke() throws ExecException, IOException{
+        pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+//        pigServer = new PigServer(ExecType.LOCAL);
+        
+    }
+    
+    @Before
+    public void setUp() throws Exception {
+        setupFRJoin();
+	setupSkewJoin();
+    }
+
+    private void setupFRJoin() throws Exception {
+        int LOOP_SIZE = 2;
+        String[] input = new String[2*LOOP_SIZE];
+        int k = 0;
+        for(int i = 1; i <= LOOP_SIZE; i++) {
+            String si = i + "";
+            for(int j=1;j<=LOOP_SIZE;j++)
+                input[k++] = si + "\t" + j;
+        }
+        Util.createInputFile(cluster, FR_INPUT_FILE, input);
+    }
+
+    private void setupSkewJoin() throws IOException {
+        PrintWriter w = new PrintWriter(new FileWriter(SKEW_INPUT_FILE1));
+
+        int k = 0;
+        for(int j=0; j<120; j++) {
+                w.println("100\tapple1\taaa" + k);
+            k++;
+            w.println("200\torange1\tbbb" + k);
+            k++;
+            w.println("300\tstrawberry\tccc" + k);
+            k++;
+        }
+
+        w.close();
+
+        PrintWriter w2 = new PrintWriter(new FileWriter(SKEW_INPUT_FILE2));
+        w2.println("100\tapple1");
+        w2.println("100\tapple2");
+        w2.println("100\tapple2");
+        w2.println("200\torange1");
+        w2.println("200\torange2");
+        w2.println("300\tstrawberry");
+        w2.println("400\tpear");
+
+        w2.close();
+
+        // Create a file with null keys
+        PrintWriter w5 = new PrintWriter(new FileWriter(SKEW_INPUT_FILE5));
+        for(int i=0; i < 10; i++) {
+                w5.println("\tapple1");
+        }
+        w5.println("100\tapple2");
+        for(int i=0; i < 10; i++) {
+                w5.println("\torange1");
+        }
+        w5.println("\t");
+        w5.println("100\t");
+        w5.close();
+
+	Util.copyFromLocalToCluster(cluster,SKEW_INPUT_FILE1,SKEW_INPUT_FILE1);
+        Util.copyFromLocalToCluster(cluster,SKEW_INPUT_FILE2,SKEW_INPUT_FILE2);
+        Util.copyFromLocalToCluster(cluster,SKEW_INPUT_FILE5,SKEW_INPUT_FILE5);
+    }
+
+    @AfterClass
+    public static void oneTimeTearDown() throws Exception {
+        cluster.shutDown();
+    }
+    
+    @After
+    public void tearDown() throws Exception {
+        Util.deleteFile(cluster, FR_INPUT_FILE);
+        tearDownSkewJoin();
+    }
+
+    private void tearDownSkewJoin() throws Exception {
+        new File(SKEW_INPUT_FILE1).delete();
+        new File(SKEW_INPUT_FILE2).delete();
+        new File(SKEW_INPUT_FILE5).delete();
+        Util.deleteDirectory(new File("skewedjoin"));
+
+        Util.deleteFile(cluster,SKEW_INPUT_FILE1);
+        Util.deleteFile(cluster,SKEW_INPUT_FILE2);
+        Util.deleteFile(cluster,SKEW_INPUT_FILE5);
+    }
+
+    @Test
+    public void testFRJoin() throws IOException{
+        pigServer.registerQuery("A = LOAD '" + FR_INPUT_FILE + "' as (x:int,y:int);");
+        pigServer.registerQuery("B = LOAD '" + FR_INPUT_FILE + "' as (x:int,y:int);");
+        DataBag dbfrj = BagFactory.getInstance().newDefaultBag(), dbshj = BagFactory.getInstance().newDefaultBag();
+        {
+            pigServer.registerQuery("C = join A by $0, B by $0 using \"replicated\";");
+            Iterator<Tuple> iter = pigServer.openIterator("C");
+            
+            while(iter.hasNext()) {
+                dbfrj.add(iter.next());
+            }
+        }
+        {
+            pigServer.registerQuery("C = join A by $0, B by $0;");
+            Iterator<Tuple> iter = pigServer.openIterator("C");
+            
+            while(iter.hasNext()) {
+                dbshj.add(iter.next());
+            }
+        }
+        Assert.assertTrue(dbfrj.size()>0 && dbshj.size()>0);
+        Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
+    }
+
+    @Test
+    public void testSkewedJoinWithGroup() throws IOException{
+        pigServer.registerQuery("A = LOAD '" + SKEW_INPUT_FILE1 + "' as (id, name, n);");
+        pigServer.registerQuery("B = LOAD '" + SKEW_INPUT_FILE2 + "' as (id, name);");
+        pigServer.registerQuery("C = GROUP A by id;");
+        pigServer.registerQuery("D = GROUP B by id;");
+
+        DataBag dbfrj = BagFactory.getInstance().newDefaultBag(), dbshj = BagFactory.getInstance().newDefaultBag();
+        {
+            pigServer.registerQuery("E = join C by group, D by group using \"skewed\" parallel 5;");
+            Iterator<Tuple> iter = pigServer.openIterator("E");
+
+            while(iter.hasNext()) {
+                dbfrj.add(iter.next());
+            }
+        }
+        {
+            pigServer.registerQuery("E = join C by group, D by group;");
+            Iterator<Tuple> iter = pigServer.openIterator("E");
+
+            while(iter.hasNext()) {
+                dbshj.add(iter.next());
+            }
+        }
+        Assert.assertTrue(dbfrj.size()>0 && dbshj.size()>0);
+        Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
+    }
+
+    @Test
+    public void testSkewedJoinOuter() throws IOException {
+        pigServer.registerQuery("A = LOAD '" + SKEW_INPUT_FILE5 + "' as (id,name);");
+        pigServer.registerQuery("B = LOAD '" + SKEW_INPUT_FILE5 + "' as (id,name);");
+        try {
+            DataBag dbfrj = BagFactory.getInstance().newDefaultBag();
+            {
+                pigServer.registerQuery("C = join A by id left, B by id using \"skewed\";");
+                Iterator<Tuple> iter = pigServer.openIterator("C");
+
+                while(iter.hasNext()) {
+                    dbfrj.add(iter.next());
+                }
+            }
+            {
+                pigServer.registerQuery("C = join A by id right, B by id using \"skewed\";");
+                Iterator<Tuple> iter = pigServer.openIterator("C");
+
+                while(iter.hasNext()) {
+                    dbfrj.add(iter.next());
+                }
+            }
+            {
+                pigServer.registerQuery("C = join A by id full, B by id using \"skewed\";");
+                Iterator<Tuple> iter = pigServer.openIterator("C");
+
+                while(iter.hasNext()) {
+                    dbfrj.add(iter.next());
+                }
+            }
+        } catch(Exception e) {
+            System.out.println(e.getMessage());
+            e.printStackTrace();
+            fail("Should support outer join in skewed join");
+        }
+        return;
+    }
+}

Added: hadoop/pig/trunk/test/smoke-tests
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/smoke-tests?rev=953109&view=auto
==============================================================================
--- hadoop/pig/trunk/test/smoke-tests (added)
+++ hadoop/pig/trunk/test/smoke-tests Wed Jun  9 18:37:54 2010
@@ -0,0 +1,4 @@
+**/TestCommit.java
+**/TestMultiQueryLocal.java
+**/TestStreaming.java
+**/TestJoinSmoke.java

Added: hadoop/pig/trunk/test/unit-tests
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/unit-tests?rev=953109&view=auto
==============================================================================
--- hadoop/pig/trunk/test/unit-tests (added)
+++ hadoop/pig/trunk/test/unit-tests Wed Jun  9 18:37:54 2010
@@ -0,0 +1,83 @@
+**/TestAdd.java
+**/TestAlgebraicEvalLocal.java
+**/TestBagFormat.java
+**/TestBoolean.java
+**/TestCmdLineParser.java
+**/TestConstExpr.java
+**/TestConversions.java
+**/TestDataBag.java
+**/TestDataModel.java
+**/TestDeleteOnFail.java
+**/TestDivide.java
+**/TestEqualTo.java
+**/TestEvalPipelineLocal.java
+**/TestExperimentalColumnPrune.java
+**/TestExperimentalFilterAboveForeach.java
+**/TestExperimentalFilterRule.java
+**/TestExperimentalListener.java
+**/TestExperimentalLogicalOptimizer.java
+**/TestExperimentalLogToPhyTranslationVisitor.java
+**/TestExperimentalOperatorPlan.java
+**/TestExperimentalPruneMapKeys.java
+**/TestExperimentalRule.java
+**/TestFilter.java
+**/TestForEach.java
+**/TestForEachNestedPlanLocal.java
+**/TestFuncSpec.java
+**/TestGreaterThan.java
+**/TestGTOrEqual.java
+**/TestInstantiateFunc.java
+**/TestInvoker.java
+**/TestLessThan.java
+**/TestLoadFunc.java
+**/TestLocal2.java
+**/TestLocal.java
+**/TestLocalPOSplit.java
+**/TestLogicalOptimizer.java
+**/TestLogicalPlanMigrationVisitor.java
+**/TestLogToPhyCompiler.java
+**/TestLTOrEqual.java
+**/TestMod.java
+**/TestMultiply.java
+**/TestMultiQueryLocal.java
+**/TestNotEqualTo.java
+**/TestNull.java
+**/TestOperatorPlan.java
+**/TestPackage.java
+**/TestParamSubPreproc.java
+**/TestPartitionFilterOptimization.java
+**/TestPhyOp.java
+**/TestPigScriptParser.java
+**/TestPigSplit.java
+**/TestPigStats.java
+**/TestPinOptions.java
+**/TestPOBinCond.java
+**/TestPOCast.java
+**/TestPOCogroup.java
+**/TestPOCross.java
+**/TestPODistinct.java
+**/TestPOGenerate.java
+**/TestPOMapLookUp.java
+**/TestPOSort.java
+**/TestPOUserFunc.java
+**/TestProjectionMap.java
+**/TestProject.java
+**/TestPruneColumn.java
+**/TestPruneColumnNewLogicalPlan.java
+**/TestPushDownForeachFlatten.java
+**/TestPushUpFilter.java
+**/TestRegexp.java
+**/TestRelevantFields.java
+**/TestRequiredFields.java
+**/TestResourceSchema.java
+**/TestRewire.java
+**/TestSchemaParser.java
+**/TestSchemaUtil.java
+**/TestStreamingLocal.java
+**/TestSubtract.java
+**/TestTextDataParser.java
+**/TestTupleFormat.java
+**/TestTypeChecking.java
+**/TestTypeCheckingValidator.java
+**/TestTypeCheckingValidatorNoSchema.java
+**/TestUDFWithoutParameter.java