pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1196816 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ src/org/apache/pig/newplan/logical/expression/ test/e2e/pig/tests/ test/e2e/pig/udfs/java/org/apache/pig/t...
Date Wed, 02 Nov 2011 21:44:50 GMT
Author: daijy
Date: Wed Nov  2 21:44:49 2011
New Revision: 1196816

URL: http://svn.apache.org/viewvc?rev=1196816&view=rev
Log:
PIG-2338: Need signature for EvalFunc

Added:
    pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/EvalFunc.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
    pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
    pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
    pig/trunk/test/e2e/pig/tests/nightly.conf

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Nov  2 21:44:49 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2338: Need signature for EvalFunc (daijy)
+
 OPTIMIZATIONS
 
 BUG FIXES

Modified: pig/trunk/src/org/apache/pig/EvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/EvalFunc.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/EvalFunc.java (original)
+++ pig/trunk/src/org/apache/pig/EvalFunc.java Wed Nov  2 21:44:49 2011
@@ -33,6 +33,8 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.LoadPushDown.RequiredFieldList;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigLogger;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigProgressable;
 
@@ -69,7 +71,7 @@ public abstract class EvalFunc<T>  {
      * should be logged to this via {@link PigLogger#warn}.
      */
     protected PigLogger pigLogger;
-
+    
     private static int nextSchemaId; // for assigning unique ids to UDF columns
     protected String getSchemaName(String name, Schema input) {
         String alias = name + "_";
@@ -293,4 +295,15 @@ public abstract class EvalFunc<T>  {
     public Log getLogger() {
     	return log;
     }
+    
+    /**
+     * This method will be called by Pig both in the front end and back end to
+     * pass a unique signature to the {@link EvalFunc}. The signature can be used
+     * to store into the {@link UDFContext} any information which the 
+     * {@link EvalFunc} needs to store between various method invocations in the
+     * front end and back end.
+     * @param signature a unique signature to identify this EvalFunc
+     */
+    public void setUDFContextSignature(String signature) {
+    }
 }

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
Wed Nov  2 21:44:49 2011
@@ -23,12 +23,14 @@ import java.io.ObjectInputStream;
 import java.lang.reflect.Type;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
 import org.apache.pig.Accumulator;
 import org.apache.pig.Algebraic;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.FuncSpec;
 import org.apache.pig.PigException;
+import org.apache.pig.ResourceSchema;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
@@ -46,6 +48,7 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.impl.plan.NodeIdGenerator;
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.util.UDFContext;
 
 public class POUserFunc extends ExpressionOperator {
 
@@ -66,6 +69,7 @@ public class POUserFunc extends Expressi
 
     private PhysicalOperator referencedOperator = null;
     private boolean isAccumulationDone;
+    private String signature;
 
     public PhysicalOperator getReferencedOperator() {
         return referencedOperator;
@@ -105,6 +109,7 @@ public class POUserFunc extends Expressi
 
     private void instantiateFunc(FuncSpec fSpec) {
         this.func = (EvalFunc) PigContext.instantiateFuncFromSpec(fSpec);
+        this.func.setUDFContextSignature(signature);
         if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
             executor = new MonitoredUDFExecutor(func);
         }
@@ -431,6 +436,7 @@ public class POUserFunc extends Expressi
             NodeIdGenerator.getGenerator().getNextNodeId(mKey.scope)),
             requestedParallelism, null, funcSpec.clone());
         clone.setResultType(resultType);
+        clone.signature = signature;
         return clone;
     }
 
@@ -469,4 +475,11 @@ public class POUserFunc extends Expressi
     public EvalFunc getFunc() {
         return func;
     }
+    
+    public void setSignature(String signature) {
+        this.signature = signature;
+        if (this.func!=null) {
+            this.func.setUDFContextSignature(signature);
+        }
+    }
 }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
(original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
Wed Nov  2 21:44:49 2011
@@ -505,6 +505,7 @@ public class ExpToPhyTranslationVisitor 
             p = new POUserFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
                     .getNextNodeId(DEFAULT_SCOPE)), -1,
                     null, op.getFuncSpec(), (EvalFunc) f);
+            ((POUserFunc)p).setSignature(op.getSignature());
             List<String> cacheFiles = ((EvalFunc)f).getCacheFiles();
             if (cacheFiles != null) {
                 ((POUserFunc)p).setCacheFiles(cacheFiles.toArray(new String[cacheFiles.size()]));

Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java Wed Nov
 2 21:44:49 2011
@@ -21,13 +21,17 @@ package org.apache.pig.newplan.logical.e
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Properties;
+
 import org.apache.pig.EvalFunc;
 import org.apache.pig.FuncSpec;
+import org.apache.pig.ResourceSchema;
 import org.apache.pig.builtin.Nondeterministic;
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.UDFContext;
 import org.apache.pig.newplan.Operator;
 import org.apache.pig.newplan.OperatorPlan;
 import org.apache.pig.newplan.PlanVisitor;
@@ -39,11 +43,16 @@ public class UserFuncExpression extends 
 
     private FuncSpec mFuncSpec;
     private EvalFunc<?> ef = null;
+    private String signature;
+    private static int sigSeq=0;
     
     public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec) {
         super("UserFunc", plan);
         mFuncSpec = funcSpec;
         plan.add(this);
+        if (signature==null) {
+            signature = Integer.toString(sigSeq++);
+        }
     }
     
     
@@ -170,6 +179,8 @@ public class UserFuncExpression extends 
         // This significantly optimize the performance of frontend (PIG-1738)
         if (ef==null)
             ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
+        
+        ef.setUDFContextSignature(signature);
         Schema udfSchema = ef.outputSchema(Util.translateSchema(inputSchema));
 
         if (udfSchema != null) {
@@ -198,6 +209,7 @@ public class UserFuncExpression extends 
                     lgExpPlan,
                     this.getFuncSpec().clone() );
             
+            copy.signature = signature;
             // Deep copy the input expressions.
             List<Operator> inputs = plan.getSuccessors( this );
             if( inputs != null ) {
@@ -232,5 +244,9 @@ public class UserFuncExpression extends 
 
         return msg.toString();
     }
+    
+    public String getSignature() {
+        return signature;
+    }
 
 }

Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Wed Nov  2 21:44:49 2011
@@ -3882,6 +3882,27 @@ store E into ':OUTPATH:';\, 
                                 store D into ':OUTPATH:';?,
                     }
                 ],
+            },{
+                'name' => 'UDFContext',
+                'tests' => [
+                    {
+                        # See PIG-2338
+                        'num' => 1,
+                        'pig' => q?register :FUNCPATH:/testudf.jar
+                                a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
+                                b = foreach a generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(a0);
+                                c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
+                                d = foreach c generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(c0);
+                                e = union b, d;
+                                store e into ':OUTPATH:';?,
+                        'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k'
AS (a0);
+                                b = foreach a generate '{a0: bytearray}';
+                                c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
+                                d = foreach c generate '{c0: chararray}';
+                                e = union b, d;
+                                store e into ':OUTPATH:';?,
+                    }
+                ],
             },
         ],
     },

Added: pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java?rev=1196816&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
(added)
+++ pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
Wed Nov  2 21:44:49 2011
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.udf.evalfunc;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.UDFContext;
+
+public class UDFContextTestUDF extends EvalFunc<String> {
+    String signature;
+    
+    @Override
+    public String exec(Tuple input) throws IOException {
+        Schema sch = (Schema)UDFContext.getUDFContext()
+                .getUDFProperties(this.getClass()).get("pig.evalfunc.signature."+signature);
+         return sch.toString();
+    }
+
+    @Override
+    public void setUDFContextSignature(String signature) {
+        this.signature = signature;
+    }
+        
+    @Override
+    public Schema outputSchema(Schema input) {
+        Properties props = UDFContext.getUDFContext().getUDFProperties(this.getClass());
+        props.put("pig.evalfunc.signature."+signature, input);
+        return new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY));
+    }
+}



Mime
View raw message