pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r662915 - in /incubator/pig/branches/types/src/org/apache/pig/impl: PigContext.java logicalLayer/LOUserFunc.java logicalLayer/LogToPhyTranslationVisitor.java logicalLayer/parser/QueryParser.jjt
Date Tue, 03 Jun 2008 23:26:56 GMT
Author: gates
Date: Tue Jun  3 16:26:55 2008
New Revision: 662915

URL: http://svn.apache.org/viewvc?rev=662915&view=rev
Log:
PIG-158 Santhosh's udf_funcSpec patch.  Now all TestMapReduce tests pass, still seeing failures
in some other end-to-end tests.


Modified:
    incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java?rev=662915&r1=662914&r2=662915&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java Tue Jun  3 16:26:55
2008
@@ -439,7 +439,7 @@
         if (funcSpec != null) {
             className = getClassNameFromSpec(funcSpec);
         }else{
-            className = alias;
+            className = getClassNameFromSpec(alias);
         }
         return resolveClassName(className);
     }
@@ -455,4 +455,12 @@
     public void setExecType(ExecType execType) {
         this.execType = execType;
     }
+
+    public String getFuncSpecFromAlias(String alias) {
+        String funcSpec;
+        if (definedFunctions != null && (funcSpec = definedFunctions.get(alias))!=null)
+            return funcSpec;
+        else
+            return null;
+    }
 }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java?rev=662915&r1=662914&r2=662915&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java Tue
Jun  3 16:26:55 2008
@@ -27,7 +27,7 @@
 public class LOUserFunc extends ExpressionOperator {
     private static final long serialVersionUID = 2L;
 
-    private String mFuncName;
+    private String mFuncSpec;
     private List<ExpressionOperator> mArgs;
 
     /**
@@ -42,16 +42,16 @@
      * @param returnType
      *            return type of this function.
      */
-    public LOUserFunc(LogicalPlan plan, OperatorKey k, String funcName,
+    public LOUserFunc(LogicalPlan plan, OperatorKey k, String funcSpec,
             List<ExpressionOperator> args, byte returnType) {
         super(plan, k, -1);
-        mFuncName = funcName;
+        mFuncSpec = funcSpec;
         mArgs = args;
         mType = returnType;
     }
 
-    public String getFuncName() {
-        return mFuncName;
+    public String getFuncSpec() {
+        return mFuncSpec;
     }
 
     public List<ExpressionOperator> getArguments() {

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java?rev=662915&r1=662914&r2=662915&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java
Tue Jun  3 16:26:55 2008
@@ -656,12 +656,12 @@
 	@Override
 	public void visit(LOUserFunc func) throws VisitorException {
 		String scope = func.getOperatorKey().scope;
-		Object f = PigContext.instantiateFuncFromSpec(func.getFuncName());
+		Object f = PigContext.instantiateFuncFromSpec(func.getFuncSpec());
 		PhysicalOperator p;
 		if(f instanceof EvalFunc) { 
-			p = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(),
null, func.getFuncName(), (EvalFunc)f);
+			p = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(),
null, func.getFuncSpec(), (EvalFunc)f);
 		} else {
-			p = new POUserComparisonFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(),
null, func.getFuncName(), (ComparisonFunc)f);
+			p = new POUserComparisonFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(),
null, func.getFuncSpec(), (ComparisonFunc)f);
 		}
 		p.setResultType(func.getType());
 		currentPlan.add(p);

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=662915&r1=662914&r2=662915&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Tue Jun  3 16:26:55 2008
@@ -672,7 +672,7 @@
 		<USING>  funcName = QualifiedFunction() "(" funcArgs = StringList() ")"
 		{
 			funcSpec = funcName + "(" + funcArgs + ")";
-			log.debug("LoadClause: funcSpec = + funcSpec");
+			log.debug("LoadClause: funcSpec = " + funcSpec);
 		}
 		)?
 	)
@@ -815,7 +815,6 @@
 	ExpressionOperator cond = null; 
 	ExpressionOperator lhs, rhs; 
 	Token t1; 
-	String funcName; 
 	List<ExpressionOperator> args;
     EvalFunc evalFunc = null;
 	log.trace("Entering PUnaryCond");
@@ -920,12 +919,12 @@
 |	LOOKAHEAD(EvalFunction() "(") 
 		(evalFunc=EvalFunction() "(" args=EvalArgs(over,specs,lp,input) ")" 
 			{
-                funcName = evalFunc.getClass().getName();
+                String funcSpec = evalFunc.getClass().getName();
                 Type javaType = evalFunc.getReturnType();
                 byte type = DataType.findType(javaType);
 
                 log.debug("Return type of UDF: " + DataType.findTypeName(type));
-				cond = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type);
+				cond = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec, args, type);
 				lp.add(cond);
 				log.debug("PUnaryCond: Added operator " + cond.getClass().getName() + " " + cond + "
to logical plan " + lp);
 				for(ExpressionOperator exprOp: args) {
@@ -1784,6 +1783,9 @@
 ExpressionOperator FuncEvalSpec(Schema over, Map<String, LogicalOperator> specs, LogicalPlan
lp, LogicalOperator input) : 
 {
 	String funcName = null; 
+	String funcSpec = null; 
+	String funcNameAlias = null; 
+    boolean registeredFunction = false;
 	List<ExpressionOperator> args;
 	ExpressionOperator userFunc;
     LOUserFunc userAliasFunc = null;
@@ -1792,7 +1794,21 @@
 }
 {
 	(
-    LOOKAHEAD({getDefineOp(getToken(1).image) != null}) funcName=QualifiedFunction()
+    LOOKAHEAD({ null != getDefineOp(getToken(1).image) }) funcName=QualifiedFunction()
+|   (
+    LOOKAHEAD({ null != pigContext.getFuncSpecFromAlias(getToken(1).image) }) funcNameAlias=QualifiedFunction()
+    {
+        
+		try{
+			evalFunc = (EvalFunc) pigContext.instantiateFuncFromAlias(funcNameAlias);
+            Type javaType = evalFunc.getReturnType();
+            log.debug("Type: " + javaType);
+            log.debug("funcName: " + funcName + " class name: " + evalFunc.getClass().getName()
+ " return type: " + DataType.findType(javaType));
+		}catch (Exception e){
+			throw new ParseException(e.getMessage());
+		}
+    }
+    )
 |   evalFunc=EvalFunction()
     )
     "(" args=EvalArgs(over,specs,lp,input) ")" 
@@ -1805,19 +1821,31 @@
 
 		if(null != evalFunc) {
             funcName = evalFunc.getClass().getName();
+            if(null != funcNameAlias) {
+                funcSpec = pigContext.getFuncSpecFromAlias(funcNameAlias);
+            } else {
+                funcSpec = funcName;
+            }
             Type javaType = evalFunc.getReturnType();
             byte type = DataType.findType(javaType);
             log.debug("Return type of UDF: " + DataType.findTypeName(type));
-			userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type);
+            log.debug("FuncEvalSpec: funcSpec: " + funcSpec);
+			userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec, args, type);
 		} else {
 			//we have an alias
 		    userAliasFunc = (LOUserFunc)getDefineOp(funcName);
-			int expectedNumArgs = userAliasFunc.getArguments().size();
-			int numArgs = args.size();
-			if(expectedNumArgs != numArgs) {
-				throw new ParseException("Expected " + expectedNumArgs + " argument(s) found " + numArgs
+ " argument(s)");
-			}
-			userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), userAliasFunc.getFuncName(),
userAliasFunc.getArguments(), userAliasFunc.getType());
+            if(null != userAliasFunc) {
+                int expectedNumArgs = userAliasFunc.getArguments().size();
+                int numArgs = args.size();
+                if(expectedNumArgs != numArgs) {
+                    throw new ParseException("Expected " + expectedNumArgs + " argument(s)
found " + numArgs + " argument(s)");
+                }
+                funcSpec = userAliasFunc.getFuncSpec();
+                log.debug("FuncEvalSpec: funcSpec: " + funcSpec);
+                userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec,
args, userAliasFunc.getType());
+            } else {
+                throw new ParseException("Could not locate function declaration for " + funcName);
+            }
 
 		}
 		lp.add(userFunc);
@@ -1872,7 +1900,7 @@
 
 ExpressionOperator FuncDeclareSpec(LogicalPlan lp) : 
 {
-	String funcName; 
+	String funcSpec; 
 	List<ExpressionOperator> args;
 	byte userSpecifiedType = DataType.BYTEARRAY;
     boolean userSpecified = false;
@@ -1882,14 +1910,14 @@
 {
 	[userSpecifiedType = Type() {userSpecified = true;}] evalFunc=EvalFunction() "(" args=FuncDeclareArgs(lp)
")" 
 	{
-        funcName = evalFunc.getClass().getName();
+        funcSpec = evalFunc.getClass().getName();
         Type javaType = evalFunc.getReturnType();
         byte type = DataType.findType(javaType);
         log.debug("Return type of UDF: " + DataType.findTypeName(type));
         if(userSpecified && (userSpecifiedType != type)) {
             throw new ParseException("User specified return type " + DataType.findTypeName(userSpecifiedType)
+ " and deduced return type " + DataType.findTypeName(type) + " are not the same");
         }
-		ExpressionOperator userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName,
args, type);
+		ExpressionOperator userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec,
args, type);
 		log.trace("Exiting FuncDeclareSpec");
 		return userFunc;
 	}



Mime
View raw message