Mailing-List: contact commits-help@pig.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@pig.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1148963 - in /pig/branches/branch-0.9: CHANGES.txt
 src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
 test/org/apache/pig/test/TestLimitSchemaStore.java
 test/org/apache/pig/test/TestMRCompiler.java
Date: Wed, 20 Jul 2011 22:56:26 -0000
To: commits@pig.apache.org
From: thejas@apache.org
Message-Id: <20110720225626.71D8E2388A2C@eris.apache.org>

Author: thejas
Date: Wed Jul 20 22:56:25 2011
New Revision: 1148963

URL: http://svn.apache.org/viewvc?rev=1148963&view=rev
Log:
PIG-2146: POStore.getSchema() returns null because of which PigOutputCommitter
 is not storing schema while cleanup

Added:
    pig/branches/branch-0.9/test/org/apache/pig/test/TestLimitSchemaStore.java
Modified:
    pig/branches/branch-0.9/CHANGES.txt
    pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestMRCompiler.java

Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1148963&r1=1148962&r2=1148963&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Wed Jul 20 22:56:25 2011
@@ -192,6 +192,9 @@ PIG-1696: Performance: Use System.arrayc
 
 BUG FIXES
 
+PIG-2146: POStore.getSchema() returns null because of which PigOutputCommitter
+ is not storing schema while cleanup (thejas)
+
 PIG-2159: New logical plan uses incorrect class for SUM causing for ClassCastException (daijy)
 
 PIG-1890: Fix piggybank unit test TestAvroStorage (kengoodhope via daijy)

Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=1148963&r1=1148962&r2=1148963&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Wed Jul 20 22:56:25 2011
@@ -2902,6 +2902,7 @@ public class MRCompiler extends PhyPlanV
                 POStore st = getStore();
                 st.setSFile(oldSpec);
                 st.setIsTmpStore(oldIsTmpStore);
+                st.setSchema(((POStore)mpLeaf).getSchema());
                 limitAdjustMROp.reducePlan.addAsLeaf(st);
                 limitAdjustMROp.requestedParallelism = 1;
                 limitAdjustMROp.setLimitOnly(true);

Added: pig/branches/branch-0.9/test/org/apache/pig/test/TestLimitSchemaStore.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestLimitSchemaStore.java?rev=1148963&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestLimitSchemaStore.java (added)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestLimitSchemaStore.java Wed Jul 20 22:56:25 2011
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.File;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.eclipse.jdt.internal.core.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestLimitSchemaStore{
+    
+    
+    private PigServer pigServer;
+
+    @Before
+    public void setUp() throws Exception{
+        pigServer = new PigServer(ExecType.LOCAL);
+    }
+    
+    
+    //PIG-2146
+    @Test //end to end test
+    public void testLimitStoreSchema1() throws Exception{
+        Util.createLocalInputFile("student", new String[]{"joe smith:18:3.5","amy brown:25:2.5","jim fox:20:4.0","leo fu:55:3.0"});
+        
+        pigServer.registerQuery("a = load 'student' using " + PigStorage.class.getName() + "(':') as (name, age, gpa);");
+        pigServer.registerQuery("d = distinct a;");
+        pigServer.registerQuery("lim = limit d 1;");
+        String outFile = "limitSchemaOut";
+        Util.deleteDirectory(new File(outFile));
+        pigServer.store("lim", outFile,  "PigStorage('\\t', '-schema')");
+        pigServer.dumpSchema("lim");
+        
+        pigServer.registerQuery("b = LOAD '" + outFile + "' using PigStorage('\\t', '-schema');");
+        Schema genSchema = pigServer.dumpSchema("b");
+        System.err.println(genSchema);
+        Assert.isNotNull(genSchema);
+        
+    } 
+  
+      
+}

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestMRCompiler.java?rev=1148963&r1=1148962&r2=1148963&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestMRCompiler.java Wed Jul 20 22:56:25 2011
@@ -55,6 +55,7 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc;
 import org.apache.pig.impl.plan.NodeIdGenerator;
+import org.apache.pig.impl.util.Utils;
 import org.apache.pig.test.utils.GenPhyOp;
 import org.junit.After;
 import org.junit.Before;
@@ -1118,4 +1119,25 @@ public class TestMRCompiler extends juni
         
         assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName()));
     }
+    
+    //PIG-2146
+    @Test
+    public void testSchemaInStoreForDistinctLimit() throws Exception {
+        //test if the POStore in the 2nd mr plan (that stores the actual output)
+        // has a schema 
+        String query = "a = load 'input1' as (a : int,b :float ,c : int);" + 
+            "b  = distinct a;" +
+            "c = limit b 10;" +
+            "store c into 'output';";
+        
+        PhysicalPlan pp = Util.buildPp(pigServer, query);
+        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
+        MapReduceOper secondMrOper = mrPlan.getLeaves().get(0);
+        POStore store = (POStore)secondMrOper.reducePlan.getLeaves().get(0);
+        assertEquals(
+                "compare load and store schema", 
+                store.getSchema(), 
+                Utils.getSchemaFromString("a : int,b :float ,c : int")
+        );
+    }
 }