Return-Path: Delivered-To: apmail-incubator-pig-commits-archive@locus.apache.org Received: (qmail 51002 invoked from network); 3 Jun 2008 23:27:17 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 3 Jun 2008 23:27:17 -0000 Received: (qmail 70567 invoked by uid 500); 3 Jun 2008 23:27:20 -0000 Delivered-To: apmail-incubator-pig-commits-archive@incubator.apache.org Received: (qmail 70561 invoked by uid 500); 3 Jun 2008 23:27:20 -0000 Mailing-List: contact pig-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: pig-dev@incubator.apache.org Delivered-To: mailing list pig-commits@incubator.apache.org Received: (qmail 70552 invoked by uid 99); 3 Jun 2008 23:27:20 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Jun 2008 16:27:20 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Jun 2008 23:26:39 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 664C523889C1; Tue, 3 Jun 2008 16:26:56 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r662915 - in /incubator/pig/branches/types/src/org/apache/pig/impl: PigContext.java logicalLayer/LOUserFunc.java logicalLayer/LogToPhyTranslationVisitor.java logicalLayer/parser/QueryParser.jjt Date: Tue, 03 Jun 2008 23:26:56 -0000 To: pig-commits@incubator.apache.org From: gates@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080603232656.664C523889C1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: gates Date: Tue Jun 3 16:26:55 2008 New Revision: 662915 URL: http://svn.apache.org/viewvc?rev=662915&view=rev Log: PIG-158 Santhosh's udf_funcSpec patch. Now all TestMapReduce tests pass, still seeing failures in some other end-to-end tests. Modified: incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Modified: incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java?rev=662915&r1=662914&r2=662915&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/PigContext.java Tue Jun 3 16:26:55 2008 @@ -439,7 +439,7 @@ if (funcSpec != null) { className = getClassNameFromSpec(funcSpec); }else{ - className = alias; + className = getClassNameFromSpec(alias); } return resolveClassName(className); } @@ -455,4 +455,12 @@ public void setExecType(ExecType execType) { this.execType = execType; } + + public String getFuncSpecFromAlias(String alias) { + String funcSpec; + if (definedFunctions != null && (funcSpec = definedFunctions.get(alias))!=null) + return funcSpec; + else + return null; + } } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java?rev=662915&r1=662914&r2=662915&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java Tue Jun 3 16:26:55 2008 @@ -27,7 +27,7 @@ public class LOUserFunc extends ExpressionOperator { private static final long serialVersionUID = 2L; - private String mFuncName; + private String mFuncSpec; private List mArgs; /** @@ -42,16 +42,16 @@ * @param returnType * return type of this function. */ - public LOUserFunc(LogicalPlan plan, OperatorKey k, String funcName, + public LOUserFunc(LogicalPlan plan, OperatorKey k, String funcSpec, List args, byte returnType) { super(plan, k, -1); - mFuncName = funcName; + mFuncSpec = funcSpec; mArgs = args; mType = returnType; } - public String getFuncName() { - return mFuncName; + public String getFuncSpec() { + return mFuncSpec; } public List getArguments() { Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java?rev=662915&r1=662914&r2=662915&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogToPhyTranslationVisitor.java Tue Jun 3 16:26:55 2008 @@ -656,12 +656,12 @@ @Override public void visit(LOUserFunc func) throws VisitorException { String scope = func.getOperatorKey().scope; - Object f = PigContext.instantiateFuncFromSpec(func.getFuncName()); + Object f = PigContext.instantiateFuncFromSpec(func.getFuncSpec()); PhysicalOperator p; if(f instanceof EvalFunc) { - p = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(), null, func.getFuncName(), (EvalFunc)f); + p = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(), null, func.getFuncSpec(), (EvalFunc)f); } else { - p = new POUserComparisonFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(), null, func.getFuncName(), (ComparisonFunc)f); + p = new POUserComparisonFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), func.getRequestedParallelism(), null, func.getFuncSpec(), (ComparisonFunc)f); } p.setResultType(func.getType()); currentPlan.add(p); Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=662915&r1=662914&r2=662915&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Tue Jun 3 16:26:55 2008 @@ -672,7 +672,7 @@ funcName = QualifiedFunction() "(" funcArgs = StringList() ")" { funcSpec = funcName + "(" + funcArgs + ")"; - log.debug("LoadClause: funcSpec = + funcSpec"); + log.debug("LoadClause: funcSpec = " + funcSpec); } )? ) @@ -815,7 +815,6 @@ ExpressionOperator cond = null; ExpressionOperator lhs, rhs; Token t1; - String funcName; List args; EvalFunc evalFunc = null; log.trace("Entering PUnaryCond"); @@ -920,12 +919,12 @@ | LOOKAHEAD(EvalFunction() "(") (evalFunc=EvalFunction() "(" args=EvalArgs(over,specs,lp,input) ")" { - funcName = evalFunc.getClass().getName(); + String funcSpec = evalFunc.getClass().getName(); Type javaType = evalFunc.getReturnType(); byte type = DataType.findType(javaType); log.debug("Return type of UDF: " + DataType.findTypeName(type)); - cond = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type); + cond = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec, args, type); lp.add(cond); log.debug("PUnaryCond: Added operator " + cond.getClass().getName() + " " + cond + " to logical plan " + lp); for(ExpressionOperator exprOp: args) { @@ -1784,6 +1783,9 @@ ExpressionOperator FuncEvalSpec(Schema over, Map specs, LogicalPlan lp, LogicalOperator input) : { String funcName = null; + String funcSpec = null; + String funcNameAlias = null; + boolean registeredFunction = false; List args; ExpressionOperator userFunc; LOUserFunc userAliasFunc = null; @@ -1792,7 +1794,21 @@ } { ( - LOOKAHEAD({getDefineOp(getToken(1).image) != null}) funcName=QualifiedFunction() + LOOKAHEAD({ null != getDefineOp(getToken(1).image) }) funcName=QualifiedFunction() +| ( + LOOKAHEAD({ null != pigContext.getFuncSpecFromAlias(getToken(1).image) }) funcNameAlias=QualifiedFunction() + { + + try{ + evalFunc = (EvalFunc) pigContext.instantiateFuncFromAlias(funcNameAlias); + Type javaType = evalFunc.getReturnType(); + log.debug("Type: " + javaType); + log.debug("funcName: " + funcName + " class name: " + evalFunc.getClass().getName() + " return type: " + DataType.findType(javaType)); + }catch (Exception e){ + throw new ParseException(e.getMessage()); + } + } + ) | evalFunc=EvalFunction() ) "(" args=EvalArgs(over,specs,lp,input) ")" @@ -1805,19 +1821,31 @@ if(null != evalFunc) { funcName = evalFunc.getClass().getName(); + if(null != funcNameAlias) { + funcSpec = pigContext.getFuncSpecFromAlias(funcNameAlias); + } else { + funcSpec = funcName; + } Type javaType = evalFunc.getReturnType(); byte type = DataType.findType(javaType); log.debug("Return type of UDF: " + DataType.findTypeName(type)); - userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type); + log.debug("FuncEvalSpec: funcSpec: " + funcSpec); + userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec, args, type); } else { //we have an alias userAliasFunc = (LOUserFunc)getDefineOp(funcName); - int expectedNumArgs = userAliasFunc.getArguments().size(); - int numArgs = args.size(); - if(expectedNumArgs != numArgs) { - throw new ParseException("Expected " + expectedNumArgs + " argument(s) found " + numArgs + " argument(s)"); - } - userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), userAliasFunc.getFuncName(), userAliasFunc.getArguments(), userAliasFunc.getType()); + if(null != userAliasFunc) { + int expectedNumArgs = userAliasFunc.getArguments().size(); + int numArgs = args.size(); + if(expectedNumArgs != numArgs) { + throw new ParseException("Expected " + expectedNumArgs + " argument(s) found " + numArgs + " argument(s)"); + } + funcSpec = userAliasFunc.getFuncSpec(); + log.debug("FuncEvalSpec: funcSpec: " + funcSpec); + userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec, args, userAliasFunc.getType()); + } else { + throw new ParseException("Could not locate function declaration for " + funcName); + } } lp.add(userFunc); @@ -1872,7 +1900,7 @@ ExpressionOperator FuncDeclareSpec(LogicalPlan lp) : { - String funcName; + String funcSpec; List args; byte userSpecifiedType = DataType.BYTEARRAY; boolean userSpecified = false; @@ -1882,14 +1910,14 @@ { [userSpecifiedType = Type() {userSpecified = true;}] evalFunc=EvalFunction() "(" args=FuncDeclareArgs(lp) ")" { - funcName = evalFunc.getClass().getName(); + funcSpec = evalFunc.getClass().getName(); Type javaType = evalFunc.getReturnType(); byte type = DataType.findType(javaType); log.debug("Return type of UDF: " + DataType.findTypeName(type)); if(userSpecified && (userSpecifiedType != type)) { throw new ParseException("User specified return type " + DataType.findTypeName(userSpecifiedType) + " and deduced return type " + DataType.findTypeName(type) + " are not the same"); } - ExpressionOperator userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type); + ExpressionOperator userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcSpec, args, type); log.trace("Exiting FuncDeclareSpec"); return userFunc; }