Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 60935 invoked from network); 21 Oct 2008 18:32:12 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 21 Oct 2008 18:32:12 -0000 Received: (qmail 68259 invoked by uid 500); 21 Oct 2008 18:32:14 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 68226 invoked by uid 500); 21 Oct 2008 18:32:14 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 68217 invoked by uid 99); 21 Oct 2008 18:32:14 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2008 11:32:14 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2008 18:31:02 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 0A4922388A08; Tue, 21 Oct 2008 11:31:41 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r706708 [4/13] - in /hadoop/core/branches/branch-0.19: ./ src/contrib/hive/ src/contrib/hive/bin/ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/ src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/ src/contrib/hive/conf/... Date: Tue, 21 Oct 2008 18:30:34 -0000 To: core-commits@hadoop.apache.org From: dhruba@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20081021183141.0A4922388A08@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Oct 21 11:29:18 2008 @@ -24,7 +24,10 @@ import java.lang.reflect.Method; import org.antlr.runtime.tree.*; +import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -35,8 +38,10 @@ import org.apache.hadoop.hive.ql.typeinfo.TypeInfo; import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.ql.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.ql.udf.UDFOPPositive; import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.fs.Path; import org.apache.commons.lang.StringUtils; @@ -342,7 +347,12 @@ skipRecursion = true; } break; - + + case HiveParser.TOK_LIMIT: + { + qbp.setDestLimit(ctx_1.dest, new Integer(ast.getChild(0).getText())); + } + break; default: skipRecursion = false; break; @@ -384,8 +394,18 @@ // Pass each where clause to the pruner QBParseInfo qbp = qb.getParseInfo(); for(String clause: qbp.getClauseNames()) { - if (qbp.getWhrForClause(clause) != null) { - pruner.addExpression((CommonTree)qbp.getWhrForClause(clause).getChild(0)); + + CommonTree whexp = (CommonTree)qbp.getWhrForClause(clause); + + if (pruner.getTable().isPartitioned() && + conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict") && + (whexp == null || !pruner.hasPartitionPredicate((CommonTree)whexp.getChild(0)))) { + throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE.getMsg(whexp != null ? whexp : qbp.getSelForClause(clause), + " for Alias " + alias + " Table " + pruner.getTable().getName())); + } + + if (whexp != null) { + pruner.addExpression((CommonTree)whexp.getChild(0)); } } @@ -466,7 +486,7 @@ CommonTree ast = qbp.getDestForClause(name); switch (ast.getToken().getType()) { case HiveParser.TOK_TAB: { - tableSpec ts = new tableSpec(this.db, ast); + tableSpec ts = new tableSpec(this.db, ast, true); if(ts.partSpec == null) { // This is a table @@ -487,6 +507,7 @@ { fname = getTmpFileName(); ctx.setResDir(new Path(fname)); + qb.setIsQuery(true); } qb.getMetaData().setDestForAlias(name, fname, (ast.getToken().getType() == HiveParser.TOK_DIR)); @@ -643,138 +664,50 @@ } } - /** - * Returns the expression for the SerDe field. - * @return null if the tree cannot be represented by a SerDe field. - */ - public static String getSerDeFieldExpression(CommonTree node) { - if (node.getToken().getType() == HiveParser.TOK_COLREF){ - // String tabAlias = node.getChild(0).getText(); - String colName = node.getChild(1).getText(); - return colName; - } - if (node.getChildCount() != 2) { - return null; - } - String left = getSerDeFieldExpression((CommonTree)node.getChild(0)); - if (left == null) return null; - - if (node.getToken().getType() == HiveParser.DOT) { - return left + '.' + node.getChild(1).getText(); - } else if (node.getToken().getType() == HiveParser.LSQUARE){ - return left + '[' + node.getChild(1).getText() + ']'; - } - return null; - } - - /** - * Returns the table name for the SerDe field. - * @return null if the tree cannot be represented by a SerDe field. - */ - public static String getTableName(CommonTree node) { - while (node.getToken().getType() != HiveParser.TOK_COLREF) { - if (node.getChildCount() != 2) return null; - node = (CommonTree) node.getChild(0); - } - return node.getChild(0).getText(); - } @SuppressWarnings("nls") - private OperatorInfoList genFilterPlan(String dest, QB qb, - OperatorInfoList input) throws SemanticException { - - // We can assert here that the input list is of size one - if (input.size() != 1) { - throw new SemanticException("Filter has more than one inputs"); - } + private OperatorInfo genFilterPlan(String dest, QB qb, + OperatorInfo input) throws SemanticException { CommonTree whereExpr = qb.getParseInfo().getWhrForClause(dest); - OperatorInfoList output = (OperatorInfoList)input.clone(); - output.get(0).setOp( + OperatorInfo output = (OperatorInfo)input.clone(); + output.setOp( OperatorFactory.getAndMakeChild( new filterDesc(genExprNodeDesc((CommonTree)whereExpr.getChild(0), qb.getParseInfo().getAlias(), - input.get(0).getRowResolver())), - new RowSchema(output.get(0).getRowResolver().getColumnInfos()), - input.get(0).getOp() + input.getRowResolver())), + new RowSchema(output.getRowResolver().getColumnInfos()), + input.getOp() ) ); - LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + output.get(0).getRowResolver().toString()); + LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + output.getRowResolver().toString()); return output; } @SuppressWarnings("nls") private void genColList(String alias, CommonTree sel, ArrayList col_list, RowResolver input, Integer pos, - RowResolver output, String colAlias) throws SemanticException { + RowResolver output) throws SemanticException { // TODO: Have to put in the support for AS clause - String tabName = ((CommonTree) sel.getChild(0)).getToken().getText(); - ArrayList fieldTypeList = new ArrayList(); - ArrayList fieldList = new ArrayList(); - if (sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) { - // This is the tab.* case - // In this case add all the columns to the fieldList - // from the input schema - for(ColumnInfo colInfo: input.getColumnInfos()) { - String name = colInfo.getInternalName(); - String [] tmp = input.reverseLookup(name); - fieldList.add(name); - fieldTypeList.add(colInfo.getType()); - output.put(alias, tmp[1], new ColumnInfo(pos.toString(), colInfo.getType(), - colInfo.getIsVirtual())); - pos = Integer.valueOf(pos.intValue() + 1); - } - } else { - // For now only allow columns of the form tab.col - if (sel.getChildCount() == 1) { - throw new SemanticException(ErrorMsg.NO_TABLE_ALIAS.getMsg(sel.getChild(0))); - } - - // Lookup the name from the input - ColumnInfo colInfo = input.get(tabName, sel.getChild(1).getText()); - - // TODO: Hack it up for now: Later we have to pass the QB in order to check for the - // table alias instead of relying on input.hasTableAlias - if (colInfo == null && input.getIsExprResolver()) { - throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(sel)); - } else if (!input.hasTableAlias(tabName)) { - throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(sel.getChild(0))); - } else if (colInfo == null) { - throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(sel.getChild(1))); - } - - // Add to the field list - fieldList.add(colInfo.getInternalName()); - fieldTypeList.add(colInfo.getType()); - // Add to the output - if (!StringUtils.isEmpty(alias) && - (output.get(alias, colAlias) != null)) { - throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(sel.getChild(1))); - } - output.put(alias, colAlias, - new ColumnInfo(pos.toString(), colInfo.getType(), colInfo.getIsVirtual())); - pos = Integer.valueOf(pos.intValue() + 1); - } - - // Generate the corresponding expressions - for (int i=0; i col_list = new ArrayList(); RowResolver out_rwsch = new RowResolver(); CommonTree trfm = null; + String alias = qb.getParseInfo().getAlias(); Integer pos = Integer.valueOf(0); // Iterate over the selects @@ -884,10 +813,9 @@ String colAlias = getColAlias(selExpr, "_C" + i); CommonTree sel = (CommonTree)selExpr.getChild(0); - if (sel.getToken().getType() == HiveParser.TOK_COLREF || - sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) { + if (sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) { genColList(qb.getParseInfo().getAlias(), sel, col_list, - input.get(0).getRowResolver(), pos, out_rwsch, colAlias); + input.getRowResolver(), pos, out_rwsch); } else if (sel.getToken().getType() == HiveParser.TOK_TRANSFORM) { if (i > 0) { throw new SemanticException(ErrorMsg.INVALID_TRANSFORM.getMsg(sel)); @@ -896,31 +824,37 @@ CommonTree cols = (CommonTree) trfm.getChild(0); for (int j = 0; j < cols.getChildCount(); ++j) { CommonTree expr = (CommonTree) cols.getChild(j); - if (expr.getToken().getType() == HiveParser.TOK_COLREF || - expr.getToken().getType() == HiveParser.TOK_ALLCOLREF) { - genColList(qb.getParseInfo().getAlias(), expr, - col_list, input.get(0).getRowResolver(), - pos, out_rwsch, - expr.getChild(1).getText()); + if (expr.getToken().getType() == HiveParser.TOK_ALLCOLREF) { + genColList(alias, expr, + col_list, input.getRowResolver(), + pos, out_rwsch); } else { - exprNodeDesc exp = genExprNodeDesc(expr, qb.getParseInfo() - .getAlias(), input.get(0).getRowResolver()); + exprNodeDesc exp = genExprNodeDesc(expr, alias, input.getRowResolver()); col_list.add(exp); - out_rwsch.put(qb.getParseInfo().getAlias(), expr.getText(), + if (!StringUtils.isEmpty(alias) && + (out_rwsch.get(alias, colAlias) != null)) { + throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(expr.getChild(1))); + } + + out_rwsch.put(alias, expr.getText(), new ColumnInfo((Integer.valueOf(pos)).toString(), - String.class, false)); // Everything is a string right now + exp.getTypeInfo())); // Everything is a string right now } } } else { // Case when this is an expression exprNodeDesc exp = genExprNodeDesc(sel, qb.getParseInfo() - .getAlias(), input.get(0).getRowResolver()); + .getAlias(), input.getRowResolver()); col_list.add(exp); + if (!StringUtils.isEmpty(alias) && + (out_rwsch.get(alias, colAlias) != null)) { + throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(sel.getChild(1))); + } // Since the as clause is lacking we just use the text representation // of the expression as the column name - out_rwsch.put(qb.getParseInfo().getAlias(), colAlias, + out_rwsch.put(alias, colAlias, new ColumnInfo((Integer.valueOf(pos)).toString(), - String.class, false)); // Everything is a string right now + exp.getTypeInfo())); // Everything is a string right now } pos = Integer.valueOf(pos.intValue() + 1); } @@ -931,51 +865,23 @@ } } - OperatorInfoList output = (OperatorInfoList) input.clone(); - output.get(0).setOp(OperatorFactory.getAndMakeChild( + OperatorInfo output = (OperatorInfo) input.clone(); + output.setOp(OperatorFactory.getAndMakeChild( new selectDesc(col_list), new RowSchema(out_rwsch.getColumnInfos()), - input.get(0).getOp())); + input.getOp())); - output.get(0).setRowResolver(out_rwsch); + output.setRowResolver(out_rwsch); if (trfm != null) { output = genScriptPlan(trfm, qb, output); } LOG.debug("Created Select Plan for clause: " + dest + " row schema: " - + output.get(0).getRowResolver().toString()); + + output.getRowResolver().toString()); return output; } - private OperatorInfo genGroupByPlanSelectOperator( - QBParseInfo parseInfo, String dest, OperatorInfo groupByOperatorInfo) - throws SemanticException { - - RowResolver groupByOutputRowResolver = groupByOperatorInfo.getRowResolver(); - RowResolver selectOutputRowResolver = new RowResolver(); - ArrayList selectCols = new ArrayList(); - CommonTree selectExpr = parseInfo.getSelForClause(dest); - for (int i = 0; i < selectExpr.getChildCount(); ++i) { - CommonTree sel = (CommonTree) selectExpr.getChild(i).getChild(0); - - // We need to recurse into the expression until we hit a UDAF or keys, - // which are both in groupByOutputToColumns. - exprNodeDesc exp = genExprNodeDesc(sel, parseInfo.getAlias(), - groupByOutputRowResolver); - - selectCols.add(exp); - - selectOutputRowResolver.put(parseInfo.getAlias(), sel.getText(), - new ColumnInfo((Integer.valueOf(i)).toString(), - String.class, false)); // Everything is a class right now - } - - return new OperatorInfo(OperatorFactory.getAndMakeChild(new selectDesc( - selectCols), new RowSchema(selectOutputRowResolver.getColumnInfos()), - groupByOperatorInfo.getOp()), selectOutputRowResolver); - } - @SuppressWarnings("nls") private OperatorInfo genGroupByPlanGroupByOperator( QBParseInfo parseInfo, String dest, OperatorInfo reduceSinkOperatorInfo, @@ -999,8 +905,7 @@ groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); String field = (Integer.valueOf(i)).toString(); groupByOutputRowResolver.put("",grpbyExpr.toStringTree(), - new ColumnInfo(field, exprInfo.getType(), - exprInfo.getIsVirtual())); + new ColumnInfo(field, exprInfo.getType())); } // For each aggregation HashMap aggregationTrees = parseInfo @@ -1037,7 +942,7 @@ value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)); groupByOutputRowResolver.put("",value.toStringTree(), new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(), - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } return new OperatorInfo( @@ -1071,7 +976,7 @@ groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); String field = (Integer.valueOf(i)).toString(); outputRS.put("", text, - new ColumnInfo(field, exprInfo.getType(), exprInfo.getIsVirtual())); + new ColumnInfo(field, exprInfo.getType())); } // For each aggregation @@ -1109,7 +1014,7 @@ value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)); outputRS.put("",value.toStringTree(), new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(), - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } return new OperatorInfo( @@ -1139,7 +1044,7 @@ if (reduceSinkOutputRowResolver.get("", text) == null) { reduceSinkOutputRowResolver.put("", text, new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } else { throw new SemanticException(ErrorMsg.DUPLICATE_GROUPBY_KEY.getMsg(grpbyExpr)); } @@ -1156,7 +1061,7 @@ reduceKeys.add(genExprNodeDesc(parameter, parseInfo.getAlias(), reduceSinkInputRowResolver)); reduceSinkOutputRowResolver.put("", text, new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } } } @@ -1175,13 +1080,14 @@ reduceValues.add(genExprNodeDesc(parameter, parseInfo.getAlias(), reduceSinkInputRowResolver)); reduceSinkOutputRowResolver.put("", text, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } } } return new OperatorInfo( - OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, numPartitionFields), + OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, numPartitionFields, + -1, false), new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), inputOperatorInfo.getOp()), reduceSinkOutputRowResolver @@ -1205,7 +1111,7 @@ assert (outputRS.get("", text) == null); outputRS.put("", text, new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - String.class, false)); + String.class)); } else { // dummy key @@ -1228,8 +1134,8 @@ if (outputRS.get(key, field) == null) { reduceValues.add(new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName())); - outputRS.put(key, field, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), valueInfo.getType(), - valueInfo.getIsVirtual())); + outputRS.put(key, field, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), + valueInfo.getType())); } } } @@ -1246,7 +1152,7 @@ reduceValues.add(grpbyExprNode); outputRS.put("", text, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), - grpbyExprNode.getTypeInfo(), false)); + grpbyExprNode.getTypeInfo())); } } @@ -1263,14 +1169,15 @@ reduceValues.add(pNode); outputRS.put("", text, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), - pNode.getTypeInfo(), false)); + pNode.getTypeInfo())); } } } } return new OperatorInfo( - OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, distinctText == null ? -1 : 1), + OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, + -1, distinctText == null ? -1 : 1, -1, false), new RowSchema(outputRS.getColumnInfos()), input.getOp()), outputRS); } @@ -1305,7 +1212,7 @@ reduceKeys.add(new exprNodeColumnDesc(TypeInfoFactory.getPrimitiveTypeInfo(String.class), field)); reduceSinkOutputRowResolver2.put("", grpbyExpr.toStringTree(), new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + field, - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } // Get partial aggregation results and store in reduceValues ArrayList reduceValues = new ArrayList(); @@ -1314,15 +1221,16 @@ .getAggregationExprsForClause(dest); for (Map.Entry entry : aggregationTrees.entrySet()) { reduceValues.add(new exprNodeColumnDesc(TypeInfoFactory.getPrimitiveTypeInfo(String.class), - (Integer.valueOf(inputField)).toString())); + (Integer.valueOf(inputField)).toString())); inputField++; reduceSinkOutputRowResolver2.put("", ((CommonTree)entry.getValue()).toStringTree(), new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(), - String.class, false)); // Everything is a string right now + String.class)); // Everything is a string right now } return new OperatorInfo( - OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, numPartitionFields), + OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, + numPartitionFields, -1, true), new RowSchema(reduceSinkOutputRowResolver2.getColumnInfos()), groupByOperatorInfo.getOp()), reduceSinkOutputRowResolver2 @@ -1351,8 +1259,7 @@ groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), expression)); String field = (Integer.valueOf(i)).toString(); groupByOutputRowResolver2.put("",grpbyExpr.toStringTree(), - new ColumnInfo(field, exprInfo.getType(), - exprInfo.getIsVirtual())); + new ColumnInfo(field, exprInfo.getType())); } HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); @@ -1374,8 +1281,7 @@ aggregations.add(new aggregationDesc(aggClass, aggParameters, false)); groupByOutputRowResolver2.put("", value.toStringTree(), new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(), - paraExprInfo.getType(), - paraExprInfo.getIsVirtual())); // Everything is a string right now + paraExprInfo.getType())); // Everything is a string right now } return new OperatorInfo( @@ -1402,14 +1308,10 @@ * @throws SemanticException */ @SuppressWarnings({ "unused", "nls" }) - private OperatorInfoList genGroupByPlan1MR(String dest, QB qb, - OperatorInfoList inputList) throws SemanticException { + private OperatorInfo genGroupByPlan1MR(String dest, QB qb, + OperatorInfo input) throws SemanticException { - // We can assert here that the input list is of size one - if (inputList.size() != 1) { - throw new SemanticException("Select has more than one inputs"); - } - OperatorInfo inputOperatorInfo = inputList.get(0); + OperatorInfo inputOperatorInfo = input; QBParseInfo parseInfo = qb.getParseInfo(); // ////// 1. Generate ReduceSinkOperator @@ -1422,14 +1324,7 @@ OperatorInfo groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.COMPLETE); - // ////// 3. Generate SelectOperator - OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo, - dest, groupByOperatorInfo); - - // ////// 4. Create output - OperatorInfoList output = new OperatorInfoList(); - output.add(selectOperatorInfo); - return output; + return groupByOperatorInfo; } /** @@ -1450,14 +1345,10 @@ * @throws SemanticException */ @SuppressWarnings("nls") - private OperatorInfoList genGroupByPlan2MR(String dest, QB qb, - OperatorInfoList inputList) throws SemanticException { + private OperatorInfo genGroupByPlan2MR(String dest, QB qb, + OperatorInfo input) throws SemanticException { - // We can assert here that the input list is of size one - if (inputList.size() != 1) { - throw new SemanticException("Select has more than one inputs"); - } - OperatorInfo inputOperatorInfo = inputList.get(0); + OperatorInfo inputOperatorInfo = input; QBParseInfo parseInfo = qb.getParseInfo(); // ////// 1. Generate ReduceSinkOperator @@ -1483,14 +1374,7 @@ OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR( parseInfo, dest, reduceSinkOperatorInfo2); - // ////// 5. Generate SelectOperator - OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo, - dest, groupByOperatorInfo2); - - // ////// 6. Create output - OperatorInfoList output = new OperatorInfoList(); - output.add(selectOperatorInfo); - return output; + return groupByOperatorInfo2; } /** @@ -1500,14 +1384,10 @@ * shared by all groupbys. */ @SuppressWarnings("nls") - private OperatorInfoList genGroupByPlan3MR(String dest, QB qb, - OperatorInfoList inputList) throws SemanticException { + private OperatorInfo genGroupByPlan3MR(String dest, QB qb, + OperatorInfo input) throws SemanticException { - // We can assert here that the input list is of size one - if (inputList.size() != 1) { - throw new SemanticException("Select has more than one inputs"); - } - OperatorInfo inputOperatorInfo = inputList.get(0); + OperatorInfo inputOperatorInfo = input; QBParseInfo parseInfo = qb.getParseInfo(); // ////// Generate GroupbyOperator @@ -1523,19 +1403,12 @@ OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR( parseInfo, dest, reduceSinkOperatorInfo2); - // ////// Generate SelectOperator - OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo, - dest, groupByOperatorInfo2); - - // ////// Create output - OperatorInfoList output = new OperatorInfoList(); - output.add(selectOperatorInfo); - return output; + return groupByOperatorInfo2; } @SuppressWarnings("nls") - private OperatorInfoList genConversionOps(String dest, QB qb, - OperatorInfoList input) throws SemanticException { + private OperatorInfo genConversionOps(String dest, QB qb, + OperatorInfo input) throws SemanticException { Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest); Table dest_tab = null; @@ -1556,50 +1429,12 @@ } } - /* - // We have the table object here - go over the row resolver - // and check all the types are the same - // Vector srcOpns = input.get(0).getRowResolver().getColumnInfos(); - - Vector insOpns = new Vector(); - try { - StructObjectInspector rowObjectInspector = (StructObjectInspector)dest_tab.getDeserializer().getObjectInspector(); - List fields = rowObjectInspector.getAllStructFieldRefs(); - for (int i=0; i colInfos = inputRR.getColumnInfos(); + + boolean first = true; + for (ColumnInfo colInfo:colInfos) { + String[] nm = inputRR.reverseLookup(colInfo.getInternalName()); + if (!first) + cols = cols.concat(","); + + first = false; + if (nm[0] == null) + cols = cols.concat(nm[1]); + else + cols = cols.concat(nm[0] + "." + nm[1]); + } + this.loadFileWork.add(new loadFileDesc(queryTmpdir, dest_path, - (dest_type.intValue() == QBMetaData.DEST_DFS_FILE))); - break; + (dest_type.intValue() == QBMetaData.DEST_DFS_FILE), cols)); + break; } default: throw new SemanticException("Unknown destination type: " + dest_type); } - OperatorInfoList output = (OperatorInfoList)input.clone(); - output.get(0).setOp( + OperatorInfo output = (OperatorInfo)input.clone(); + output.setOp( OperatorFactory.getAndMakeChild( new fileSinkDesc(queryTmpdir, table_desc), - new RowSchema(output.get(0).getRowResolver().getColumnInfos()), - input.get(0).getOp() + new RowSchema(output.getRowResolver().getColumnInfos()), input.getOp() ) ); LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: " + dest_path + " row schema: " - + output.get(0).getRowResolver().toString()); + + output.getRowResolver().toString()); return output; } @SuppressWarnings("nls") - private OperatorInfoList genReduceSinkPlan(String dest, QB qb, - OperatorInfoList input) throws SemanticException { + private OperatorInfo genLimitPlan(String dest, QB qb, OperatorInfo input, int limit) throws SemanticException { + // A map-only job can be optimized - instead of converting it to a map-reduce job, we can have another map + // job to do the same to avoid the cost of sorting in the map-reduce phase. A better approach would be to + // write into a local file and then have a map-only job. + // Add the limit operator to get the value fields - // We can assert here that the input list is of size one - if (input.size() != 1) { - throw new SemanticException("Select has more than one inputs"); - } + OperatorInfo limitMap = (OperatorInfo)input.clone(); + limitMap.setOp( + OperatorFactory.getAndMakeChild( + new limitDesc(limit), new RowSchema(limitMap.getRowResolver().getColumnInfos()), + input.getOp() + ) + ); + + LOG.debug("Created LimitOperator Plan for clause: " + dest + " row schema: " + + limitMap.getRowResolver().toString()); + + return limitMap; + } + + @SuppressWarnings("nls") + private OperatorInfo genLimitMapRedPlan(String dest, QB qb, OperatorInfo input, int limit, boolean isOuterQuery) throws SemanticException { + // A map-only job can be optimized - instead of converting it to a map-reduce job, we can have another map + // job to do the same to avoid the cost of sorting in the map-reduce phase. A better approach would be to + // write into a local file and then have a map-only job. + // Add the limit operator to get the value fields + OperatorInfo curr = genLimitPlan(dest, qb, input, limit); + + if (isOuterQuery) + return curr; + + // Create a reduceSink operator followed by another limit + curr = genReduceSinkPlan(dest, qb, curr, 1); + return genLimitPlan(dest, qb, curr, limit); + } + + @SuppressWarnings("nls") + private OperatorInfo genReduceSinkPlan(String dest, QB qb, + OperatorInfo input, int numReducers) throws SemanticException { // First generate the expression for the key // The cluster by clause has the aliases for the keys - CommonTree clby = qb.getParseInfo().getClusterByForClause(dest); ArrayList keyCols = new ArrayList(); - int ccount = clby.getChildCount(); - for(int i=0; i valueCols = new ArrayList(); // For the generation of the values expression just get the inputs // signature and generate field expressions for those - for(ColumnInfo colInfo: input.get(0).getRowResolver().getColumnInfos()) { + for(ColumnInfo colInfo: input.getRowResolver().getColumnInfos()) { valueCols.add(new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName())); } - OperatorInfoList interim = (OperatorInfoList)input.clone(); - interim.get(0).setOp( + OperatorInfo interim = (OperatorInfo)input.clone(); + interim.setOp( OperatorFactory.getAndMakeChild( - PlanUtils.getReduceSinkDesc(keyCols, valueCols, keyCols.size()), - new RowSchema(interim.get(0).getRowResolver().getColumnInfos()), - input.get(0).getOp() + PlanUtils.getReduceSinkDesc(keyCols, valueCols, -1, keyCols.size(), numReducers, false), + new RowSchema(interim.getRowResolver().getColumnInfos()), + input.getOp() ) ); // Add the extract operator to get the value fields RowResolver out_rwsch = new RowResolver(); - RowResolver interim_rwsch = interim.get(0).getRowResolver(); + RowResolver interim_rwsch = interim.getRowResolver(); Integer pos = Integer.valueOf(0); for(ColumnInfo colInfo: interim_rwsch.getColumnInfos()) { String [] info = interim_rwsch.reverseLookup(colInfo.getInternalName()); out_rwsch.put(info[0], info[1], - new ColumnInfo(pos.toString(), colInfo.getType(), colInfo.getIsVirtual())); + new ColumnInfo(pos.toString(), colInfo.getType())); pos = Integer.valueOf(pos.intValue() + 1); } - OperatorInfoList output = (OperatorInfoList)interim.clone(); - output.get(0).setOp( + OperatorInfo output = (OperatorInfo)interim.clone(); + output.setOp( OperatorFactory.getAndMakeChild( - new extractDesc( - new exprNodeColumnDesc(String.class, Utilities.ReduceField.VALUE.toString()) - ), + new extractDesc(new exprNodeColumnDesc(String.class, Utilities.ReduceField.VALUE.toString())), new RowSchema(out_rwsch.getColumnInfos()), - interim.get(0).getOp() + interim.getOp() ) ); - output.get(0).setRowResolver(out_rwsch); + output.setRowResolver(out_rwsch); LOG.debug("Created ReduceSink Plan for clause: " + dest + " row schema: " - + output.get(0).getRowResolver().toString()); + + output.getRowResolver().toString()); return output; } @@ -1760,8 +1644,8 @@ ColumnInfo valueInfo = inputRS.get(key, field); keyDesc.add(new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName())); if (outputRS.get(key, field) == null) - outputRS.put(key, field, new ColumnInfo((Integer.valueOf(outputPos++)).toString(), valueInfo.getType(), - valueInfo.getIsVirtual())); + outputRS.put(key, field, new ColumnInfo((Integer.valueOf(outputPos++)).toString(), + valueInfo.getType())); } } @@ -1808,17 +1692,19 @@ outputRS.put(src, field, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), - valueInfo.getType(), valueInfo.getIsVirtual())); + valueInfo.getType())); } } - return new OperatorInfo(OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc( - reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size()), new RowSchema(outputRS.getColumnInfos()), + return new OperatorInfo( + OperatorFactory.getAndMakeChild( + PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(), -1, false), + new RowSchema(outputRS.getColumnInfos()), child.getOp()), outputRS); } private OperatorInfo genJoinOperator(QB qb, QBJoinTree joinTree, - HashMap map) throws SemanticException { + HashMap map) throws SemanticException { QBJoinTree leftChild = joinTree.getJoinSrc(); OperatorInfo joinSrcOp = null; if (leftChild != null) @@ -1831,7 +1717,7 @@ int pos = 0; for (String src : joinTree.getBaseSrc()) { if (src != null) { - OperatorInfo srcOp = map.get(src).get(0); + OperatorInfo srcOp = map.get(src); srcOps[pos] = genJoinReduceSinkChild(qb, joinTree, srcOp, src, pos); pos++; } else { @@ -1881,13 +1767,11 @@ } } - private OperatorInfoList genJoinPlan(QB qb, HashMap map) + private OperatorInfo genJoinPlan(QB qb, HashMap map) throws SemanticException { QBJoinTree joinTree = qb.getQbJoinTree(); OperatorInfo joinOp = genJoinOperator(qb, joinTree, map); - OperatorInfoList output = new OperatorInfoList(); - output.add(joinOp); - return output; + return joinOp; } private QBJoinTree genJoinTree(CommonTree joinParseTree) @@ -2104,19 +1988,19 @@ } @SuppressWarnings("nls") - private OperatorInfoList genBodyPlan(QB qb, OperatorInfoList input) + private OperatorInfo genBodyPlan(QB qb, OperatorInfo input) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); - OperatorInfoList output = new OperatorInfoList(); TreeSet ks = new TreeSet(); ks.addAll(qbp.getClauseNames()); String distinctText = null; CommonTree distn = null; - OperatorInfoList opList = null; + OperatorInfo op = null; boolean grpBy = false; + int numGrpBy = 0; // In case of a multiple group bys, all of them should have the same distinct key for (String dest : ks) { @@ -2124,6 +2008,7 @@ if ((qbp.getAggregationExprsForClause(dest).size() != 0) || (getGroupByForClause(qbp, dest).size() > 0)) { grpBy = true; + numGrpBy++; // If there is a distinctFuncExp, add all parameters to the reduceKeys. if (qbp.getDistinctFuncExprForClause(dest) != null) { @@ -2142,63 +2027,71 @@ // In the first stage, copy the input and all the group by expressions // and aggregate paramaters. This can be optimized in the future to only - // evaluate expressions that occur frequently - if (grpBy) { + // evaluate expressions that occur frequently. For a single groupby, no need to do so + if (grpBy && (numGrpBy > 1)) { OperatorInfo reduceSinkOperatorInfo = - genGroupByPlanReduceSinkOperator(qbp, input.get(0), distn, ks); + genGroupByPlanReduceSinkOperator(qbp, input, distn, ks); // ////// 2. Generate GroupbyOperator OperatorInfo forwardOperatorInfo = genGroupByPlanForwardOperator(qbp, reduceSinkOperatorInfo); - opList = new OperatorInfoList(); - opList.add(forwardOperatorInfo); + op = forwardOperatorInfo; } // Go over all the destination tables + OperatorInfo curr = null; for (String dest : ks) { boolean groupByExpr = false; if (qbp.getAggregationExprsForClause(dest).size() != 0 || getGroupByForClause(qbp, dest).size() > 0) groupByExpr = true; - OperatorInfoList curr = input; - if (groupByExpr) - curr = opList; + curr = input; + if (groupByExpr && (numGrpBy > 1)) + curr = op; if (qbp.getWhrForClause(dest) != null) { curr = genFilterPlan(dest, qb, curr); } if (qbp.getAggregationExprsForClause(dest).size() != 0 - || getGroupByForClause(qbp, dest).size() > 0) - curr = genGroupByPlan3MR(dest, qb, curr); - else - curr = genSelectPlan(dest, qb, curr); - - if (qbp.getClusterByForClause(dest) != null) { - curr = genReduceSinkPlan(dest, qb, curr); - } else if (!qbp.getIsSubQ()) { + || getGroupByForClause(qbp, dest).size() > 0) { + if (numGrpBy > 1) + curr = genGroupByPlan3MR(dest, qb, curr); + else + curr = genGroupByPlan2MR(dest, qb, curr); + } + + curr = genSelectPlan(dest, qb, curr); + Integer limit = qbp.getDestLimit(dest); + + if (qbp.getIsSubQ()) { + if (qbp.getClusterByForClause(dest) != null) + curr = genReduceSinkPlan(dest, qb, curr, -1); + if (limit != null) + curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), false); + } + else + { curr = genConversionOps(dest, qb, curr); + // exact limit can be taken care of by the fetch operator + if (limit != null) { + curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), true); + qb.getParseInfo().setOuterQueryLimit(limit.intValue()); + } curr = genFileSinkPlan(dest, qb, curr); } - - output.addAll(curr); } LOG.debug("Created Body Plan for Query Block " + qb.getId()); - return output; + return curr; } @SuppressWarnings("nls") - private OperatorInfoList genUnionPlan(String unionalias, String leftalias, - OperatorInfoList left, String rightalias, OperatorInfoList right) + private OperatorInfo genUnionPlan(String unionalias, String leftalias, + OperatorInfo leftOp, String rightalias, OperatorInfo rightOp) throws SemanticException { - if (left.size() != 1) { - throw new SemanticException("Select has more than one inputs"); - } - OperatorInfo leftOp = left.get(0); RowResolver leftRR = leftOp.getRowResolver(); - OperatorInfo rightOp = right.get(0); RowResolver rightRR = rightOp.getRowResolver(); HashMap leftmap = leftRR.getFieldMap(leftalias); HashMap rightmap = rightRR.getFieldMap(rightalias); @@ -2236,8 +2129,7 @@ rightOp.getOp().setChildOperators(child); leftOp.getOp().setChildOperators(child); // create operator info list to return - OperatorInfoList unionout = new OperatorInfoList(); - unionout.add(new OperatorInfo(unionforward, unionoutRR)); + OperatorInfo unionout = new OperatorInfo(unionforward, unionoutRR); return unionout; } @@ -2245,7 +2137,7 @@ // ((default_sample_hashfn(cols) & Integer.MAX_VALUE) % denominator) == numerator exprNodeDesc numeratorExpr = new exprNodeConstantDesc( TypeInfoFactory.getPrimitiveTypeInfo(Integer.class), - Integer.valueOf(ts.getNumerator())); + Integer.valueOf(ts.getNumerator() - 1)); exprNodeDesc denominatorExpr = new exprNodeConstantDesc( TypeInfoFactory.getPrimitiveTypeInfo(Integer.class), @@ -2277,7 +2169,7 @@ } @SuppressWarnings("nls") - private OperatorInfoList genTablePlan(String alias, QB qb) + private OperatorInfo genTablePlan(String alias, QB qb) throws SemanticException { Table tab = qb.getMetaData().getSrcForAlias(alias); @@ -2289,7 +2181,7 @@ for (int i=0; i aliasToOpInfo = new HashMap(); + HashMap aliasToOpInfo = new HashMap(); // Recurse over the subqueries to fill the subquery part of the plan for (String alias : qb.getSubqAliases()) { @@ -2417,7 +2308,7 @@ aliasToOpInfo.put(alias, genTablePlan(alias, qb)); } - OperatorInfoList srcOpInfoList = null; + OperatorInfo srcOpInfo = null; // process join if (qb.getParseInfo().getJoinExpr() != null) { @@ -2425,26 +2316,29 @@ QBJoinTree joinTree = genJoinTree(joinExpr); qb.setQbJoinTree(joinTree); mergeJoinTree(qb); - srcOpInfoList = genJoinPlan(qb, aliasToOpInfo); + srcOpInfo = genJoinPlan(qb, aliasToOpInfo); } else // Now if there are more than 1 sources then we have a join case // later we can extend this to the union all case as well - srcOpInfoList = aliasToOpInfo.values() - .iterator().next(); + srcOpInfo = aliasToOpInfo.values().iterator().next(); - OperatorInfoList bodyOpInfoList = genBodyPlan(qb, srcOpInfoList); + OperatorInfo bodyOpInfo = genBodyPlan(qb, srcOpInfo); LOG.debug("Created Plan for Query Block " + qb.getId()); - return bodyOpInfoList; + // is it a top level QB, and can it be optimized ? For eg: select * from T does not need a map-reduce job + QBParseInfo qbp = qb.getParseInfo(); + qbp.setCanOptTopQ(qb.isSelectStarQuery()); + + return bodyOpInfo; } - private Operator getReducer(Operator top) { + private Operator getReduceSink(Operator top) { if (top.getClass() == ReduceSinkOperator.class) { // Get the operator following the reduce sink assert (top.getChildOperators().size() == 1); - return top.getChildOperators().get(0); + return top; } List> childOps = top.getChildOperators(); @@ -2453,7 +2347,7 @@ } for (int i = 0; i < childOps.size(); ++i) { - Operator reducer = getReducer(childOps.get(i)); + Operator reducer = getReduceSink(childOps.get(i)); if (reducer != null) { return reducer; } @@ -2463,13 +2357,41 @@ } @SuppressWarnings("nls") - private void genMapRedTasks() throws SemanticException { + private void genMapRedTasks(QB qb) throws SemanticException { + fetchWork fetch = null; + moveWork mv = null; + Task mvTask = null; + Task fetchTask = null; + + if (qb.getParseInfo().getCanOptTopQ()) { + Iterator> iter = qb.getMetaData().getAliasToTable().entrySet().iterator(); + Table tab = ((Map.Entry)iter.next()).getValue(); + fetch = new fetchWork(tab.getPath(), tab.getDeserializer().getClass(), + tab.getInputFormatClass(), tab.getSchema(), qb.getParseInfo().getOuterQueryLimit()); + + fetchTask = TaskFactory.get(fetch, this.conf); + setFetchTask(fetchTask); + return; + } - // First we generate the move work as this needs to be made dependent on all - // the tasks - // that have a file sink operation - moveWork mv = new moveWork(loadTableWork, loadFileWork); - Task mvTask = TaskFactory.get(mv, this.conf); + // In case of a select, use a fetch task instead of a move task + if (qb.getIsQuery()) { + if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) + throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg()); + String cols = loadFileWork.get(0).getColumns(); + fetch = new fetchWork(new Path(loadFileWork.get(0).getSourceDir()), + MetadataTypedColumnsetSerDe.class, TextInputFormat.class, + Utilities.makeProperties("columns", cols), qb.getParseInfo().getOuterQueryLimit()); + + fetchTask = TaskFactory.get(fetch, this.conf); + setFetchTask(fetchTask); + } + else { + // First we generate the move work as this needs to be made dependent on all + // the tasks that have a file sink operation + mv = new moveWork(loadTableWork, loadFileWork); + mvTask = TaskFactory.get(mv, this.conf); + } // Maintain a map from the top level left most reducer in each of these // trees @@ -2479,12 +2401,24 @@ new HashMap, Task>(); for (String alias_id : this.topOps.keySet()) { Operator topOp = this.topOps.get(alias_id); - Operator reducer = getReducer(topOp); + Operator reduceSink = getReduceSink(topOp); + Operator reducer = null; + if (reduceSink != null) + reducer = reduceSink.getChildOperators().get(0); Task rootTask = opTaskMap.get(reducer); if (rootTask == null) { rootTask = TaskFactory.get(getMapRedWork(), this.conf); opTaskMap.put(reducer, rootTask); ((mapredWork) rootTask.getWork()).setReducer(reducer); + reduceSinkDesc desc = (reduceSink == null) ? null : (reduceSinkDesc)reduceSink.getConf(); + + // The number of reducers may be specified in the plan in some cases, or may need to be inferred + if (desc != null) { + if (desc.getNumReducers() != -1) + ((mapredWork) rootTask.getWork()).setNumReduceTasks(new Integer(desc.getNumReducers())); + else if (desc.getInferNumReducers() == true) + ((mapredWork) rootTask.getWork()).setInferNumReducers(true); + } this.rootTasks.add(rootTask); } genTaskPlan(topOp, rootTask, opTaskMap, mvTask); @@ -2533,7 +2467,7 @@ HashMap, Task> redTaskMap, Task mvTask) { // Check if this is a file sink operator - if (op.getClass() == FileSinkOperator.class) { + if ((op.getClass() == FileSinkOperator.class) && (mvTask != null)) { // If this is a file sink operator then set the move task to be dependent // on the current task currTask.addDependentTask(mvTask); @@ -2577,6 +2511,10 @@ ctask = TaskFactory.get(cplan, this.conf); // Add the reducer cplan.setReducer(reducer); + if (((reduceSinkDesc)child.getConf()).getNumReducers() != -1) + cplan.setNumReduceTasks(new Integer(((reduceSinkDesc)child.getConf()).getNumReducers())); + else + cplan.setInferNumReducers(((reduceSinkDesc)child.getConf()).getInferNumReducers()); redTaskMap.put(reducer, ctask); // Recurse on the reducer @@ -2668,7 +2606,7 @@ @Override @SuppressWarnings("nls") - public void analyze(CommonTree ast, Context ctx) throws SemanticException { + public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { this.ctx = ctx; reset(); @@ -2696,7 +2634,7 @@ // At this point we have the complete operator tree // from which we want to find the reduce operator - genMapRedTasks(); + genMapRedTasks(qb); LOG.info("Completed plan generation"); @@ -2716,6 +2654,7 @@ /** * This function create an ExprNodeDesc for a UDF function given the children (arguments). * It will insert implicit type conversion functions if necessary. + * @throws SemanticException */ public static exprNodeDesc getFuncExprNodeDesc(String udfName, List children) { // Find the corresponding method @@ -2725,7 +2664,33 @@ assert(child != null); TypeInfo childTypeInfo = child.getTypeInfo(); assert(childTypeInfo != null); - argumentClasses.add(childTypeInfo.getPrimitiveClass()); + + // Note: we don't pass the element types of MAP/LIST to UDF. + // That will work for null test and size but not other more complex functionalities like list slice etc. + // For those more complex functionalities, we plan to have a ComplexUDF interface which has an evaluate + // method that accepts a list of objects and a list of objectinspectors. + switch (childTypeInfo.getCategory()) { + case PRIMITIVE: { + argumentClasses.add(childTypeInfo.getPrimitiveClass()); + break; + } + case MAP: { + argumentClasses.add(Map.class); + break; + } + case LIST: { + argumentClasses.add(List.class); + break; + } + case STRUCT: { + argumentClasses.add(Object.class); + break; + } + default: { + // should never happen + assert(false); + } + } } Method udfMethod = FunctionRegistry.getUDFMethod(udfName, false, argumentClasses); if (udfMethod == null) return null; @@ -2740,7 +2705,7 @@ if (desc instanceof exprNodeNullDesc) { exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null); ch.add(newCh); - } else if (pType.isAssignableFrom(desc.getTypeInfo().getPrimitiveClass())) { + } else if (pType.isAssignableFrom(argumentClasses.get(i))) { // no type conversion needed ch.add(desc); } else { @@ -2783,8 +2748,7 @@ // If the current subExpression is pre-calculated, as in Group-By etc. ColumnInfo colInfo = input.get("", expr.toStringTree()); if (colInfo != null) { - desc = new exprNodeColumnDesc( - colInfo.getType(), colInfo.getInternalName()); + desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()); return desc; } @@ -2797,29 +2761,29 @@ int tokType = expr.getType(); switch (tokType) { case HiveParser.TOK_COLREF: { + + // For now only allow columns of the form tab.col + if (expr.getChildCount() == 1) { + throw new SemanticException(ErrorMsg.NO_TABLE_ALIAS.getMsg(expr.getChild(0))); + } - assert(expr.getChildCount() == 2); - String tabAlias = SemanticAnalyzer.getTableName(expr); - String colName = SemanticAnalyzer.getSerDeFieldExpression(expr); + String tabAlias = expr.getChild(0).getText(); + String colName = expr.getChild(1).getText(); if (tabAlias == null || colName == null) { throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr)); } colInfo = input.get(tabAlias, colName); - if (colInfo == null) { - LOG.info("input: " + input.toString() + " expr: " + expr.toStringTree() + " e: " + input.getIsExprResolver()); - } - - // TODO: Hack it up for now: Later we have to pass the QB in order to check for the - // table alias instead of relying on input.hasTableAlias + if (colInfo == null && input.getIsExprResolver()) { throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr)); - } else if (!input.hasTableAlias(expr.getChild(0).getText())) { - throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg((CommonTree)expr.getChild(0))); + } + else if (!input.hasTableAlias(tabAlias)) { + throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0))); } else if (colInfo == null) { - throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg((CommonTree)expr.getChild(1))); + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1))); } - desc = new exprNodeColumnDesc( - colInfo.getType(), colInfo.getInternalName()); + + desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()); break; } @@ -2842,9 +2806,13 @@ return desc; } + static HashMap specialUnaryOperatorTextHashMap; static HashMap specialFunctionTextHashMap; static HashMap conversionFunctionTextHashMap; static { + specialUnaryOperatorTextHashMap = new HashMap(); + specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive"); + specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative"); specialFunctionTextHashMap = new HashMap(); specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull"); specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull"); @@ -2873,16 +2841,23 @@ } public static String getFunctionText(CommonTree expr, boolean isFunction) { - String funcText; + String funcText = null; if (!isFunction) { - // For operator, the function name is the operator text - funcText = expr.getText(); + // For operator, the function name is the operator text, unless it's in our special dictionary + if (expr.getChildCount() == 1) { + funcText = specialUnaryOperatorTextHashMap.get(expr.getType()); + } + if (funcText == null) { + funcText = expr.getText(); + } } else { - // For TOK_FUNCTION, the function name is stored in the first child. + // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our + // special dictionary. assert(expr.getChildCount() >= 1); - funcText = specialFunctionTextHashMap.get(((CommonTree)expr.getChild(0)).getType()); + int funcType = ((CommonTree)expr.getChild(0)).getType(); + funcText = specialFunctionTextHashMap.get(funcType); if (funcText == null) { - funcText = conversionFunctionTextHashMap.get(((CommonTree)expr.getChild(0)).getType()); + funcText = conversionFunctionTextHashMap.get(funcType); } if (funcText == null) { funcText = ((CommonTree)expr.getChild(0)).getText(); @@ -2911,24 +2886,60 @@ exprNodeConstantDesc fieldName = (exprNodeConstantDesc)children.get(1); assert(fieldName.getValue() instanceof String); - // Calculate TypeInfo + // Calculate result TypeInfo String fieldNameString = (String)fieldName.getValue(); - TypeInfo t = object.getTypeInfo().getStructFieldTypeInfo(fieldNameString); + TypeInfo objectTypeInfo = object.getTypeInfo(); - desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString); + // Allow accessing a field of list element structs directly from a list + boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); + if (isList) { + objectTypeInfo = objectTypeInfo.getListElementTypeInfo(); + } + if (objectTypeInfo.getCategory() != Category.STRUCT) { + throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); + } + TypeInfo t = objectTypeInfo.getStructFieldTypeInfo(fieldNameString); + if (isList) { + t = TypeInfoFactory.getListTypeInfo(t); + } + + desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString, isList); } else if (funcText.equals("[")){ // "[]" : LSQUARE/INDEX Expression assert(children.size() == 2); - // Only allow constant integer index for now - if (!(children.get(1) instanceof exprNodeConstantDesc) - || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) { - throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr)); - } - // Calculate TypeInfo - TypeInfo t = children.get(0).getTypeInfo().getListElementTypeInfo(); - desc = new exprNodeIndexDesc(t, children.get(0), children.get(1)); + // Check whether this is a list or a map + TypeInfo myt = children.get(0).getTypeInfo(); + + if (myt.getCategory() == Category.LIST) { + // Only allow constant integer index for now + if (!(children.get(1) instanceof exprNodeConstantDesc) + || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) { + throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr)); + } + + // Calculate TypeInfo + TypeInfo t = myt.getListElementTypeInfo(); + desc = new exprNodeIndexDesc(t, children.get(0), children.get(1)); + } + else if (myt.getCategory() == Category.MAP) { + // Only allow only constant indexes for now + if (!(children.get(1) instanceof exprNodeConstantDesc)) { + throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg(expr)); + } + if (!(((exprNodeConstantDesc)children.get(1)).getValue().getClass() == + myt.getMapKeyTypeInfo().getPrimitiveClass())) { + throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE.getMsg(expr)); + } + // Calculate TypeInfo + TypeInfo t = myt.getMapValueTypeInfo(); + + desc = new exprNodeIndexDesc(t, children.get(0), children.get(1)); + } + else { + throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr)); + } } else { // other operators or functions Class udf = FunctionRegistry.getUDFClass(funcText); @@ -2955,6 +2966,16 @@ } } } + // UDFOPPositive is a no-op. + // However, we still create it, and then remove it here, to make sure we only allow + // "+" for numeric types. + if (desc instanceof exprNodeFuncDesc) { + exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc; + if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) { + assert(funcDesc.getChildren().size() == 1); + desc = funcDesc.getChildren().get(0); + } + } assert(desc != null); return desc; } Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Tue Oct 21 11:29:18 2008 @@ -36,8 +36,11 @@ case HiveParser.TOK_DROPTABLE: case HiveParser.TOK_DESCTABLE: case HiveParser.TOK_ALTERTABLE_ADDCOLS: + case HiveParser.TOK_ALTERTABLE_REPLACECOLS: case HiveParser.TOK_ALTERTABLE_RENAME: - case HiveParser.TOK_SHOWTABLES: + case HiveParser.TOK_ALTERTABLE_DROPPARTS: + case HiveParser.TOK_SHOWTABLES: + case HiveParser.TOK_SHOWPARTITIONS: return new DDLSemanticAnalyzer(conf); case HiveParser.TOK_CREATEFUNCTION: return new FunctionSemanticAnalyzer(conf); Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java Tue Oct 21 11:29:18 2008 @@ -22,11 +22,12 @@ public class DDLWork implements Serializable { private static final long serialVersionUID = 1L; - private createTableDesc createTblDesc; - private dropTableDesc dropTblDesc; - private alterTableDesc alterTblDesc; - private showTablesDesc showTblsDesc; - private descTableDesc descTblDesc; + private createTableDesc createTblDesc; + private dropTableDesc dropTblDesc; + private alterTableDesc alterTblDesc; + private showTablesDesc showTblsDesc; + private showPartitionsDesc showPartsDesc; + private descTableDesc descTblDesc; public DDLWork() { } @@ -66,6 +67,13 @@ } /** + * @param showPartsDesc + */ + public DDLWork(showPartitionsDesc showPartsDesc) { + this.showPartsDesc = showPartsDesc; + } + + /** * @return the createTblDesc */ @explain(displayName="Create Table Operator") @@ -125,6 +133,22 @@ this.showTblsDesc = showTblsDesc; } + + /** + * @return the showPartsDesc + */ + @explain(displayName="Show Partitions Operator") + public showPartitionsDesc getShowPartsDesc() { + return showPartsDesc; + } + + /** + * @param showPartsDesc the showPartsDesc to set + */ + public void setShowPartsDesc(showPartitionsDesc showPartsDesc) { + this.showPartsDesc = showPartsDesc; + } + /** * @return the descTblDesc */ Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Tue Oct 21 11:29:18 2008 @@ -63,21 +63,14 @@ // We will make reduce key and reduce value TableDesc with configurable SerDes - public static reduceSinkDesc getReduceSinkDesc(ArrayList keyCols, - ArrayList valueCols, int numPartitionFields) { + public static reduceSinkDesc getReduceSinkDesc(ArrayList keyCols, + ArrayList valueCols, + int tag, int numPartitionFields, + int numReducers, boolean inferNumReducers) { - return new reduceSinkDesc(keyCols, valueCols, numPartitionFields, - getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())), - getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size()))); - } - - // We will make reduce key and reduce value TableDesc with configurable SerDes - public static reduceSinkDesc getReduceSinkDesc(ArrayList keyCols, - ArrayList valueCols, int tag, int numPartitionFields) { - - return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields, - getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())), - getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size()))); + return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields, numReducers, inferNumReducers, + getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())), + getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size()))); } // We should read the TableDesc from gWork when it is available. Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java Tue Oct 21 11:29:18 2008 @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; -import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -29,7 +28,7 @@ public class alterTableDesc extends ddlDesc implements Serializable { private static final long serialVersionUID = 1L; - public static enum alterTableTypes {RENAME, ADDCOLS}; + public static enum alterTableTypes {RENAME, ADDCOLS, REPLACECOLS}; alterTableTypes op; String oldName; @@ -50,8 +49,8 @@ * @param name name of the table * @param newCols new columns to be added */ - public alterTableDesc(String name, List newCols) { - op = alterTableTypes.ADDCOLS; + public alterTableDesc(String name, List newCols, alterTableTypes alterType) { + this.op = alterType; this.oldName = name; this.newCols = newCols; } @@ -100,6 +99,8 @@ return "rename"; case ADDCOLS: return "add columns"; + case REPLACECOLS: + return "replace columns"; } return "unknown"; @@ -128,4 +129,5 @@ public void setNewCols(List newCols) { this.newCols = newCols; } + } Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java Tue Oct 21 11:29:18 2008 @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.List; +import java.util.Map; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; @@ -43,6 +44,8 @@ String comment; boolean isSequenceFile; String location; + String serName; + Map mapProp; public createTableDesc(String tableName, boolean isExternal, List cols, List partCols, @@ -51,7 +54,7 @@ String collItemDelim, String mapKeyDelim, String lineDelim, String comment, boolean isSequenceFile, - String location) { + String location, String serName, Map mapProp) { this.tableName = tableName; this.isExternal = isExternal; this.bucketCols = bucketCols; @@ -66,6 +69,8 @@ this.mapKeyDelim = mapKeyDelim; this.numBuckets = numBuckets; this.partCols = partCols; + this.serName = serName; + this.mapProp = mapProp; } @explain(displayName="name") @@ -166,7 +171,7 @@ this.comment = comment; } - @explain(displayName="isSequenceFile") + @explain(displayName="isSequenceFile") public boolean isSequenceFile() { return isSequenceFile; } @@ -207,4 +212,35 @@ public void setSortCols(List sortCols) { this.sortCols = sortCols; } + + /** + * @return the serDeName + */ + @explain(displayName="serde name") + public String getSerName() { + return serName; + } + + /** + * @param serName the serName to set + */ + public void setSerName(String serName) { + this.serName = serName; + } + + /** + * @return the serDe properties + */ + @explain(displayName="serde properties") + public Map getMapProp() { + return mapProp; + } + + /** + * @param mapProp the map properties to set + */ + public void setMapProp(Map mapProp) { + this.mapProp = mapProp; + } + } Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java Tue Oct 21 11:29:18 2008 @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.HashMap; + import org.apache.hadoop.fs.Path; @explain(displayName="Describe Table") @@ -26,28 +28,69 @@ { private static final long serialVersionUID = 1L; - String tableName; + String tableName; + HashMap partSpec; Path resFile; + boolean isExt; /** + * @param isExt + * @param partSpec * @param resFile * @param tableName */ - public descTableDesc(Path resFile, String tableName) { + public descTableDesc(Path resFile, String tableName, HashMap partSpec, boolean isExt) { + this.isExt = isExt; + this.partSpec = partSpec; this.resFile = resFile; this.tableName = tableName; } + /** + * @return the isExt + */ + public boolean isExt() { + return isExt; + } + + /** + * @param isExt the isExt to set + */ + public void setExt(boolean isExt) { + this.isExt = isExt; + } + + /** + * @return the tableName + */ @explain(displayName="table") public String getTableName() { return tableName; } + /** + * @param tableName the tableName to set + */ public void setTableName(String tableName) { this.tableName = tableName; } /** + * @return the partSpec + */ + @explain(displayName="partition") + public HashMap getPartSpec() { + return partSpec; + } + + /** + * @param partSpec the partSpec to set + */ + public void setPartSpecs(HashMap partSpec) { + this.partSpec = partSpec; + } + + /** * @return the resFile */ public Path getResFile() { Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java Tue Oct 21 11:29:18 2008 @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.HashMap; +import java.util.List; @explain(displayName="Drop Table") public class dropTableDesc extends ddlDesc implements Serializable @@ -26,12 +28,19 @@ private static final long serialVersionUID = 1L; String tableName; + List> partSpecs; /** * @param tableName */ public dropTableDesc(String tableName) { this.tableName = tableName; + this.partSpecs = null; + } + + public dropTableDesc(String tableName, List> partSpecs) { + this.tableName = tableName; + this.partSpecs = partSpecs; } /** @@ -48,4 +57,18 @@ public void setTableName(String tableName) { this.tableName = tableName; } + + /** + * @return the partSpecs + */ + public List> getPartSpecs() { + return partSpecs; + } + + /** + * @param partSpecs the partSpecs to set + */ + public void setPartSpecs(List> partSpecs) { + this.partSpecs = partSpecs; + } } Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java?rev=706708&r1=706707&r2=706708&view=diff ============================================================================== --- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java (original) +++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java Tue Oct 21 11:29:18 2008 @@ -26,11 +26,13 @@ public class exprNodeColumnDesc extends exprNodeDesc implements Serializable { private static final long serialVersionUID = 1L; private String column; + private boolean isVirtual; public exprNodeColumnDesc() {} public exprNodeColumnDesc(TypeInfo typeInfo, String column) { super(typeInfo); this.column = column; + this.isVirtual = isVirtual; } public exprNodeColumnDesc(Class c, String column) { super(TypeInfoFactory.getPrimitiveTypeInfo(c)); @@ -42,6 +44,7 @@ public void setColumn(String column) { this.column = column; } + public String toString() { return "Column[" + column + "]"; }