pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1199416 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ src/org/apache/pig/newplan/logical/expression/ src/org/apache/pig/scripting/js/ test/e2e/pig/tests/ test/e...
Date Tue, 08 Nov 2011 19:56:17 GMT
Author: daijy
Date: Tue Nov  8 19:56:16 2011
New Revision: 1199416

URL: http://svn.apache.org/viewvc?rev=1199416&view=rev
Log:
PIG-2337: Provide UDF with input schema

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/EvalFunc.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
    pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
    pig/trunk/src/org/apache/pig/scripting/js/JsFunction.java
    pig/trunk/test/e2e/pig/tests/nightly.conf
    pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Nov  8 19:56:16 2011
@@ -30,6 +30,8 @@ PIG-2328: Add builtin UDFs for building 
 
 PIG-2338: Need signature for EvalFunc (daijy)
 
+PIG-2337: Provide UDF with input schema (xutingz via daijy)
+
 OPTIMIZATIONS
 
 BUG FIXES

Modified: pig/trunk/src/org/apache/pig/EvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/EvalFunc.java?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/EvalFunc.java (original)
+++ pig/trunk/src/org/apache/pig/EvalFunc.java Tue Nov  8 19:56:16 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.util.List;
+import java.util.Properties;
 import java.util.Stack;
 
 import org.apache.commons.logging.Log;
@@ -296,6 +297,7 @@ public abstract class EvalFunc<T>  {
     	return log;
     }
     
+    private Schema inputSchemaInternal=null;
     /**
      * This method will be called by Pig both in the front end and back end to
      * pass a unique signature to the {@link EvalFunc}. The signature can be used
@@ -306,4 +308,20 @@ public abstract class EvalFunc<T>  {
      */
     public void setUDFContextSignature(String signature) {
     }
+    
+    /**
+     * This method is for internal use. It is called by Pig core in both front-end 
+     * and back-end to setup the right input schema for EvalFunc
+     */
+    public void setInputSchema(Schema input){
+    	this.inputSchemaInternal=input;
+    }
+    	
+    /**
+     * This method is intended to be called by the user in {@link EvalFunc} to get the input
+     * schema of the EvalFunc
+     */
+    public Schema getInputSchema(){
+    	return this.inputSchemaInternal;
+    }
 }

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
Tue Nov  8 19:56:16 2011
@@ -109,7 +109,11 @@ public class POUserFunc extends Expressi
 
     private void instantiateFunc(FuncSpec fSpec) {
         this.func = (EvalFunc) PigContext.instantiateFuncFromSpec(fSpec);
-        this.func.setUDFContextSignature(signature);
+        this.setSignature(signature);
+        Properties props = UDFContext.getUDFContext().getUDFProperties(func.getClass());
+    	Schema tmpS=(Schema)props.get("pig.evalfunc.inputschema."+signature);
+    	if(tmpS!=null)
+    		this.func.setInputSchema(tmpS);
         if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
             executor = new MonitoredUDFExecutor(func);
         }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java Tue Nov
 8 19:56:16 2011
@@ -181,6 +181,12 @@ public class UserFuncExpression extends 
             ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
         
         ef.setUDFContextSignature(signature);
+        Properties props = UDFContext.getUDFContext().getUDFProperties(ef.getClass());
+        if(Util.translateSchema(inputSchema)!=null)
+    		props.put("pig.evalfunc.inputschema."+signature, Util.translateSchema(inputSchema));
+        // Store inputSchema into the UDF context
+        ef.setInputSchema(Util.translateSchema(inputSchema));
+        
         Schema udfSchema = ef.outputSchema(Util.translateSchema(inputSchema));
 
         if (udfSchema != null) {

Modified: pig/trunk/src/org/apache/pig/scripting/js/JsFunction.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/js/JsFunction.java?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/js/JsFunction.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/js/JsFunction.java Tue Nov  8 19:56:16 2011
@@ -91,7 +91,6 @@ public class JsFunction extends EvalFunc
     private String functionName;
     private JsScriptEngine jsScriptEngine;
     private Schema outputSchema;
-    private Schema inputSchema;
 
     ///////////////////////
     // Debugging functions
@@ -187,29 +186,13 @@ public class JsFunction extends EvalFunc
 
     }
 
-    private Schema getInputSchema() {
-        if (inputSchema == null) {
-            // input schema must be passed from client to slaves
-            Properties properties = UDFContext.getUDFContext().getUDFProperties(this.getClass(),
new String[]{ functionName });
-            inputSchema = (Schema)properties.get(functionName+".inputSchema");
-        }
-        return inputSchema;
-    }
-
-    private void setInputSchema(Schema inputSchema) {
-        this.inputSchema = inputSchema;
-        // input schema must be passed from client to slaves
-        Properties properties = UDFContext.getUDFContext().getUDFProperties(this.getClass(),
new String[]{ functionName });
-        properties.put(functionName+".inputSchema", inputSchema);
-    }
-
     ///////////////////////////
     // EvalFunc implementation
     ///////////////////////////
 
     @Override
     public Object exec(Tuple tuple) throws IOException {
-        Schema inputSchema = getInputSchema();
+    	Schema inputSchema = this.getInputSchema();
         if (LOG.isDebugEnabled()) {
             LOG.debug( "CALL " + stringify(outputSchema) + " " + functionName + " " + stringify(inputSchema));
         }
@@ -244,7 +227,7 @@ public class JsFunction extends EvalFunc
 
     @Override
     public Schema outputSchema(Schema input) {
-        setInputSchema(input);
+        this.setInputSchema(input);
         return outputSchema;
     }
 

Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue Nov  8 19:56:16 2011
@@ -3970,6 +3970,28 @@ store E into ':OUTPATH:';\, 
                                 e = union b, d;
                                 store e into ':OUTPATH:';?,
                     }
+                ], 
+                
+               },{
+                'name' => 'UDFContextAuto',
+                'tests' => [
+                    {
+                        # See PIG-2337
+                        'num' => 1,
+                        'pig' => q?register :FUNCPATH:/testudf.jar
+                                a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
+                                b = foreach a generate org.apache.pig.test.udf.evalfunc.UDFContextTestAutoUDF(a0);
+                                c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
+                                d = foreach c generate org.apache.pig.test.udf.evalfunc.UDFContextTestAutoUDF(c0);
+                                e = union b, d;
+                                store e into ':OUTPATH:';?,
+                        'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k'
AS (a0);
+                                b = foreach a generate '{a0: bytearray}';
+                                c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
+                                d = foreach c generate '{c0: chararray}';
+                                e = union b, d;
+                                store e into ':OUTPATH:';?,
+                    }
                 ],
             },
         ],

Modified: pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java?rev=1199416&r1=1199415&r2=1199416&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
(original)
+++ pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
Tue Nov  8 19:56:16 2011
@@ -32,7 +32,7 @@ public class UDFContextTestUDF extends E
     @Override
     public String exec(Tuple input) throws IOException {
         Schema sch = (Schema)UDFContext.getUDFContext()
-                .getUDFProperties(this.getClass()).get("pig.evalfunc.signature."+signature);
+                .getUDFProperties(this.getClass()).get("pig.evalfunc.inputschema."+signature);
          return sch.toString();
     }
 
@@ -44,7 +44,7 @@ public class UDFContextTestUDF extends E
     @Override
     public Schema outputSchema(Schema input) {
         Properties props = UDFContext.getUDFContext().getUDFProperties(this.getClass());
-        props.put("pig.evalfunc.signature."+signature, input);
+        props.put("pig.evalfunc.inputschema."+signature, input);
         return new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY));
     }
 }



Mime
View raw message