Mailing-List: contact pig-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: pig-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r945250 - in /hadoop/pig/trunk: CHANGES.txt
 src/org/apache/pig/impl/logicalLayer/LOLoad.java
 test/org/apache/pig/test/PigStorageWithSchema.java
 test/org/apache/pig/test/TestLogicalPlanBuilder.java
Date: Mon, 17 May 2010 17:21:13 -0000
To: pig-commits@incubator.apache.org
From: daijy@apache.org
Message-Id: <20100517172113.DBFBA2388A36@eris.apache.org>

Author: daijy
Date: Mon May 17 17:21:13 2010
New Revision: 945250

URL: http://svn.apache.org/viewvc?rev=945250&view=rev
Log:
PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes

Added:
    hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=945250&r1=945249&r2=945250&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon May 17 17:21:13 2010
@@ -276,6 +276,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes (daijy)
+
 PIG-1403: Make Pig work with remote HDFS in secure mode (daijy)
 
 PIG-1394: POCombinerPackage hold too much memory for InternalCachedBag (daijy)

Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=945250&r1=945249&r2=945250&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java Mon May 17 17:21:13 2010
@@ -59,6 +59,7 @@ public class LOLoad extends RelationalOp
     transient private Configuration conf;
     private static Log log = LogFactory.getLog(LOLoad.class);
     private Schema mDeterminedSchema = null;
+    private Schema scriptSchema = null;
     private RequiredFieldList requiredFieldList;
 
     private boolean mDeterminedSchemaCached = false;
@@ -149,6 +150,20 @@ public class LOLoad extends RelationalOp
                 if(null == mDeterminedSchema) {
                     mSchema = determineSchema();
                 }
+                if (mSchema == null) {
+                    log.debug("Operator schema is null; Setting it to new schema");
+                    mSchema = scriptSchema;
+                } else {
+                    log.debug("Reconciling schema");
+                    log.debug("mSchema: " + mSchema + " schema: " + scriptSchema);
+                    try {
+                        mSchema = mSchema.mergePrefixSchema(scriptSchema, true, true);
+                    } catch (SchemaMergeException e) {
+                        int errCode = 1019;
+                        String msg = "Unable to merge schemas";
+                        throw new FrontendException(msg, errCode, PigException.INPUT, false, null, e);
+                    }
+                }
                 mIsSchemaComputed = true;
             } catch (IOException ioe) {
                 int errCode = 1018;
@@ -182,27 +197,7 @@ public class LOLoad extends RelationalOp
      */
     @Override
     public void setSchema(Schema schema) throws FrontendException {
-        // In general, operators don't generate their schema until they're
-        // asked, so ask them to do it.
-        try {
-            getSchema();
-        } catch (FrontendException ioe) {
-            // It's fine, it just means we don't have a schema yet.
-        }
-        if (mSchema == null) {
-            log.debug("Operator schema is null; Setting it to new schema");
-            mSchema = schema;
-        } else {
-            log.debug("Reconciling schema");
-            log.debug("mSchema: " + mSchema + " schema: " + schema);
-            try {
-                mSchema = mSchema.mergePrefixSchema(schema, true, true);
-            } catch (SchemaMergeException e) {
-                int errCode = 1019;
-                String msg = "Unable to merge schemas";
-                throw new FrontendException(msg, errCode, PigException.INPUT, false, null, e);
-            }
-        }
+        scriptSchema = schema;
     }
     
 

Added: hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java?rev=945250&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java Mon May 17 17:21:13 2010
@@ -0,0 +1,46 @@
+package org.apache.pig.test;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.Expression;
+import org.apache.pig.LoadMetadata;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.ResourceStatistics;
+import org.apache.pig.builtin.PigStorage;
+
+public class PigStorageWithSchema extends PigStorage implements LoadMetadata {
+    private String signature;
+
+    @Override
+    public String[] getPartitionKeys(String location, Job job)
+            throws IOException {
+        return null;
+    }
+
+    @Override
+    public ResourceSchema getSchema(String location, Job job)
+            throws IOException {
+        return null;
+    }
+
+    @Override
+    public ResourceStatistics getStatistics(String location, Job job)
+            throws IOException {
+        return null;
+    }
+
+    @Override
+    public void setPartitionFilter(Expression partitionFilter)
+            throws IOException {
+    }
+    
+    @Override
+    public void setUDFContextSignature(String signature) {
+        this.signature = signature;
+    }
+    
+    public String getUDFContextSignature() {
+        return signature;
+    }
+}

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=945250&r1=945249&r2=945250&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Mon May 17 17:21:13 2010
@@ -2145,6 +2145,14 @@ public class TestLogicalPlanBuilder exte
                 "not occur", true, exceptionThrown);
     }
     
+    @Test
+    public void testLoaderSignature() {
+        LogicalPlan plan = buildPlan(" a = load '1.txt' using org.apache.pig.test.PigStorageWithSchema() as (a0:int, a1:int);");
+        assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("a"));
+        plan = buildPlan(" b = load '1.txt' using org.apache.pig.test.PigStorageWithSchema();");
+        assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("b"));
+    }
+    
     private void printPlan(LogicalPlan lp) {
         LOPrinter graphPrinter = new LOPrinter(System.err, lp);
         System.err.println("Printing the logical plan");