hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r1479522 - in /hive/branches/HIVE-4115/ql/src: java/org/apache/hadoop/hive/ql/cube/metadata/ java/org/apache/hadoop/hive/ql/cube/parse/ test/org/apache/hadoop/hive/ql/cube/processors/
Date Mon, 06 May 2013 11:10:40 GMT
Author: amareshwari
Date: Mon May  6 11:10:40 2013
New Revision: 1479522

URL: http://svn.apache.org/r1479522
Log:
Add aggregate resolver

Modified:
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
    hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/Cube.java
Mon May  6 11:10:40 2013
@@ -16,6 +16,10 @@ public final class Cube extends Abstract
   private final Set<CubeMeasure> measures;
   private final Set<CubeDimension> dimensions;
   private static final List<FieldSchema> columns = new ArrayList<FieldSchema>();
+  private final Map<String, CubeMeasure> measureMap;
+  private final Map<String, CubeDimension> dimMap;
+
+
   static {
     columns.add(new FieldSchema("dummy", "string", "dummy column"));
   }
@@ -25,6 +29,17 @@ public final class Cube extends Abstract
     super(name, columns, new HashMap<String, String>());
     this.measures = measures;
     this.dimensions = dimensions;
+
+    measureMap = new HashMap<String, CubeMeasure>();
+    for (CubeMeasure m : measures) {
+      measureMap.put(m.getName(), m);
+    }
+
+    dimMap = new HashMap<String, CubeDimension>();
+    for (CubeDimension dim : dimensions) {
+      dimMap.put(dim.getName(), dim);
+    }
+
     addProperties();
   }
 
@@ -32,6 +47,15 @@ public final class Cube extends Abstract
     super(tbl);
     this.measures = getMeasures(getName(), getProperties());
     this.dimensions = getDimensions(getName(), getProperties());
+    measureMap = new HashMap<String, CubeMeasure>();
+    for (CubeMeasure m : measures) {
+      measureMap.put(m.getName(), m);
+    }
+
+    dimMap = new HashMap<String, CubeDimension>();
+    for (CubeDimension dim : dimensions) {
+      dimMap.put(dim.getName(), dim);
+    }
   }
 
   public Set<CubeMeasure> getMeasures() {
@@ -169,4 +193,12 @@ public final class Cube extends Abstract
     }
     return true;
   }
+
+  public CubeDimension getDimensionByName(String dimension) {
+    return dimMap.get(dimension);
+  }
+
+  public CubeMeasure getMeasureByName(String measure) {
+    return measureMap.get(measure);
+  }
 }

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
Mon May  6 11:10:40 2013
@@ -1,18 +1,322 @@
 package org.apache.hadoop.hive.ql.cube.parse;
 
-import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
+
+import java.util.Arrays;
 
+import org.antlr.runtime.CommonToken;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.cube.metadata.CubeMeasure;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.log4j.Logger;
+/**
+ * <p>Replace select and having columns with default aggregate functions on
+ * them, if default aggregate is defined and if there isn't already an
+ * aggregate function specified on the columns.</p>
+ *
+ * <p>For example, if query is like - <pre>select dim1.name, fact.msr1, fact.msr1
* sin(fact.msr2)/cos(fact.msr2),
+ * sum(fact.msr4) ...</pre>
+ * Then we will replace fact.msr1 with sum(fact.msr1) given that 'sum' has been
+ * set as the default aggregate function for msr1.</p>
+ *
+ * <p>We will also wrap expressions of measures into a default aggregate function.
+ * For example the expression 'fact.msr1 * sin(fact.msr2)/cos(fact.msr2)' will become
+ *  sum(fact.msr1 * sin(fact.msr2)/cos(fact.msr2)), if sum is the default aggregate function.</p>
+ *
+ * <p>Expressions which already contain aggregate sub-expressions will not be changed.</p>
+ *
+ * <p>At this point it's assumed that aliases have been added to all columns.</p>
+ */
 public class AggregateResolver implements ContextRewriter {
+  public static final Logger LOG = Logger.getLogger(AggregateResolver.class);
+
+  private final Configuration conf;
 
   public AggregateResolver(Configuration conf) {
-    // TODO Auto-generated constructor stub
+    this.conf = conf;
   }
 
   @Override
-  public void rewriteContext(CubeQueryContext cubeql) {
-    // TODO
-    // replace select and having columns with default aggregate functions on
-    // them, if default aggregate is defined
+  public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
+    if (cubeql.getCube() == null) {
+      LOG.warn("AggregateResolver called without a cube setup. Returning");
+      return;
+    }
+
+    validateAggregates(cubeql, cubeql.getSelectAST(), false, false, false);
+    validateAggregates(cubeql, cubeql.getHavingAST(), false, false, false);
+
+    String rewritSelect = resolveForSelect(cubeql, cubeql.getSelectTree());
+    System.out.println("New select after aggregate resolver: " + rewritSelect);
+    cubeql.setSelectTree(rewritSelect);
+
+    String rewritHaving = resolveForHaving(cubeql);
+    if (StringUtils.isNotBlank(rewritHaving)) {
+      cubeql.setHavingTree(rewritHaving);
+    }
+    System.out.println("New having after aggregate resolver: " + rewritHaving);
+  }
+
+  private void validateAggregates(CubeQueryContext cubeql, ASTNode node, boolean insideAggregate,
+      boolean insideArithExpr, boolean insideNonAggrFn) throws SemanticException {
+    if (node == null) {
+      return;
+    }
+
+    int nodeType = node.getToken().getType();
+    if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+      // Found a column ref. If this is a measure, it should be inside an aggregate if its
part of
+      // an arithmetic expression or an argument of a non-aggregate function
+      String msrname = getColName(node);
+      if (cubeql.isCubeMeasure(msrname) &&
+          !insideAggregate
+          && (insideArithExpr || insideNonAggrFn)) {
+        throw new SemanticException("Not inside aggregate " + msrname);
+      }
+    } else if (HQLParser.isArithmeticOp(nodeType)) {
+      // Allowed - sum ( msr1 * msr2 + msr3)
+      // Not allowed - msr1 + msr2 * msr3 <- Not inside aggregate
+      // Not allowed - sum(msr1) + msr2 <- Aggregate only on one measure
+      // count of measures within aggregates must be equal to count of measures
+      // if both counts are equal and zero, then this node should be inside aggregate
+      int measuresInAggregates = countMeasuresInAggregates(cubeql, node, false);
+      int measuresInTree = countMeasures(cubeql, node);
+
+      if (measuresInAggregates == measuresInTree) {
+        if (measuresInAggregates == 0 && !insideAggregate) {
+          // (msr1 + msr2)
+          throw new SemanticException("Invalid projection expression: " + HQLParser.getString(node));
+        } else if (insideAggregate) {
+          // sum(sum(msr1) + sum(msr2))
+          throw new SemanticException("Invalid projection expression: " + HQLParser.getString(node));
+        }
+      } else {
+        throw new SemanticException("Invalid projection expression: " + HQLParser.getString(node));
+      }
+    } else {
+      boolean isArithmetic = HQLParser.isArithmeticOp(nodeType);
+      boolean isAggregate = isAggregateAST(node);
+      boolean isNonAggrFn = nodeType == HiveParser.TOK_FUNCTION && !isAggregate;
+      for (int i = 0; i < node.getChildCount(); i++) {
+        validateAggregates(cubeql, (ASTNode) node.getChild(i), isAggregate, isArithmetic,
isNonAggrFn);
+      }
+    }
+  }
+
+  private int countMeasures(CubeQueryContext cubeql, ASTNode node) {
+    int nodeType = node.getToken().getType();
+    if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+      String msrname = getColName(node);
+      if (cubeql.isCubeMeasure(msrname)) {
+        return 1;
+      } else {
+        return 0;
+      }
+    } else {
+      int count = 0;
+      for (int i = 0; i < node.getChildCount(); i++) {
+        count += countMeasures(cubeql, (ASTNode) node.getChild(i));
+      }
+      return count;
+    }
+  }
+
+  private int countMeasuresInAggregates(CubeQueryContext cubeql, ASTNode node, boolean inAggregate)
{
+    int nodeType = node.getToken().getType();
+    if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+      String msrname = getColName(node);
+      if (cubeql.isCubeMeasure(msrname) && inAggregate) {
+        return 1;
+      } else {
+        return 0;
+      }
+    } else {
+      int count = 0;
+      for (int i = 0; i < node.getChildCount(); i++) {
+        boolean isAggr = isAggregateAST((ASTNode)node.getChild(i));
+        count += countMeasuresInAggregates(cubeql, (ASTNode) node.getChild(i), isAggr);
+      }
+      return count;
+    }
+  }
+
+  private String resolveForSelect(CubeQueryContext cubeql, String exprTree) {
+    // Aggregate resolver needs cube to be resolved first
+    assert cubeql.getCube() != null;
+
+    if (StringUtils.isBlank(exprTree)) {
+      return "";
+    }
+
+    System.out.println("resolveAggregatesForExpr - " + exprTree);
+
+    String exprTokens[] = StringUtils.split(exprTree, ",");
+    for (int i = 0; i < exprTokens.length; i++) {
+      String token = exprTokens[i].trim();
+      System.out.println("_NEW_TOKEN_[" + token + "]");
+      String tokenAlias = cubeql.getAlias(token);
+      boolean hasAlias = false;
+      if (StringUtils.isNotBlank(tokenAlias)) {
+        token = token.substring(0, exprTree.lastIndexOf(tokenAlias)).trim();
+        hasAlias = true;
+      }
+
+      if (!cubeql.isAggregateExpr(token)) {
+        if (cubeql.isCubeMeasure(token)) {
+          // Take care of brackets added around col names in HQLParsrer.getString
+          if (token.startsWith("(") && token.endsWith(")") && token.length()
> 2) {
+            token = token.substring(1, token.length() -1);
+          }
+
+          String splits[] = StringUtils.split(token, ".");
+          for (int j = 0; j < splits.length; j++) {
+            splits[j] = splits[j].trim();
+          }
+
+          System.out.println(">> TOKEN:" + token + " SPLITS:" + Arrays.toString(splits));
+          String msrName = (splits.length <= 1) ? splits[0] : splits[1];
+          CubeMeasure measure = cubeql.getCube().getMeasureByName(msrName);
+          if (measure != null) {
+            String msrAggregate = measure.getAggregate();
+
+            if (StringUtils.isNotBlank(msrAggregate)) {
+              System.out.println("#replace msrName:["+msrName+"] "+ " with msrAggregate["+msrAggregate+"]");
+              exprTokens[i] = msrAggregate + "( " + token + ")" + (hasAlias ? " " + tokenAlias
: "");
+              exprTokens[i] = exprTokens[i].toLowerCase();
+              // Add this expression to aggregate expr set so that group by resolver can
skip
+              // over expressions changed during aggregate resolver.
+              cubeql.addAggregateExpr(exprTokens[i]);
+            } else {
+              LOG.warn("Default aggregate not specified. measure:" + msrName + " token:"
+ token);
+            }
+          }
+        }
+      } else {
+        System.out.println("Aggregate already specified: " + token);
+      }
+    }
+
+    return StringUtils.join(exprTokens, ", ");
   }
 
+  // We need to traverse the AST for Having clause.
+  // We need to skip any columns that are inside an aggregate UDAF or inside an arithmetic
expression
+  private String resolveForHaving(CubeQueryContext cubeql) {
+    ASTNode havingTree = cubeql.getHavingAST();
+    String havingTreeStr = cubeql.getHavingTree();
+
+    if (StringUtils.isBlank(havingTreeStr) || havingTree == null) {
+      return null;
+    }
+
+    for (int i = 0; i < havingTree.getChildCount(); i++) {
+      transform(cubeql, havingTree, (ASTNode) havingTree.getChild(i), i);
+    }
+
+    return HQLParser.getString(havingTree);
+  }
+
+  private void transform(CubeQueryContext cubeql, ASTNode parent, ASTNode node, int nodePos)
{
+    if (parent == null || node == null) {
+      return;
+    }
+    int nodeType = node.getToken().getType();
+
+    if (! (isAggregateAST(node) || HQLParser.isArithmeticOp(nodeType))) {
+      if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
+        // Leaf node
+        ASTNode wrapped = wrapAggregate(cubeql, node);
+        if (wrapped != node) {
+          parent.setChild(nodePos, wrapped);
+        }
+      } else {
+        // Dig deeper in non-leaf nodes
+        for (int i = 0; i < node.getChildCount(); i++) {
+          transform(cubeql, node, (ASTNode) node.getChild(i), i);
+        }
+      }
+    }
+  }
+
+  private boolean isAggregateAST(ASTNode node) {
+    int exprTokenType = node.getToken().getType();
+    if (exprTokenType == HiveParser.TOK_FUNCTION
+        || exprTokenType == HiveParser.TOK_FUNCTIONDI
+        || exprTokenType == HiveParser.TOK_FUNCTIONSTAR) {
+      assert (node.getChildCount() != 0);
+      if (node.getChild(0).getType() == HiveParser.Identifier) {
+        String functionName = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0)
+            .getText());
+        if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
+  // Wrap an aggregate function around the node if its a measure, leave it unchanged otherwise
+  private ASTNode wrapAggregate(CubeQueryContext cubeql, ASTNode node) {
+
+    String tabname = null;
+    String colname = null;
+
+    if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) {
+      colname = ((ASTNode) node.getChild(0)).getText();
+    } else {
+      // node in 'alias.column' format
+      ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL,
+          Identifier);
+      ASTNode colIdent = (ASTNode) node.getChild(1);
+
+      colname = colIdent.getText();
+      tabname = tabident.getText();
+    }
+
+    String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
+
+    if (cubeql.isCubeMeasure(msrname)) {
+      CubeMeasure measure = cubeql.getCube().getMeasureByName(colname);
+      String aggregateFn = measure.getAggregate();
+      ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION));
+      fnroot.setParent(node.getParent());
+
+      ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn));
+      fnIdentNode.setParent(fnroot);
+      fnroot.addChild(fnIdentNode);
+
+      node.setParent(fnroot);
+      fnroot.addChild(node);
+
+      return fnroot;
+    } else {
+      return node;
+    }
+  }
+
+  private String getColName(ASTNode node) {
+    String tabname = null;
+    String colname = null;
+    int nodeType = node.getToken().getType();
+    if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
+      colname = ((ASTNode) node.getChild(0)).getText();
+    } else {
+      // node in 'alias.column' format
+      ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL,
+          Identifier);
+      ASTNode colIdent = (ASTNode) node.getChild(1);
+
+      colname = colIdent.getText();
+      tabname = tabident.getText();
+    }
+
+    return StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
+  }
 }

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
Mon May  6 11:10:40 2013
@@ -19,6 +19,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.cube.metadata.AbstractCubeTable;
 import org.apache.hadoop.hive.ql.cube.metadata.Cube;
@@ -95,6 +96,8 @@ public class CubeQueryContext {
   private String groupByTree;
   private ASTNode limitTree;
   private final ASTNode joinTree;
+  private ASTNode havingAST;
+  private ASTNode selectAST;
 
   public CubeQueryContext(ASTNode ast, QB qb, HiveConf conf)
       throws SemanticException {
@@ -109,6 +112,8 @@ public class CubeQueryContext {
     if (qb.getParseInfo().getHavingForClause(clauseName) != null) {
       this.havingTree = HQLParser.getString(qb.getParseInfo().getHavingForClause(
           clauseName)).toLowerCase();
+      this.havingAST = qb.getParseInfo().getHavingForClause(
+          clauseName);
     }
     if (qb.getParseInfo().getOrderByForClause(clauseName) != null) {
       this.orderByTree = HQLParser.getString(qb.getParseInfo()
@@ -121,6 +126,8 @@ public class CubeQueryContext {
     if (qb.getParseInfo().getSelForClause(clauseName) != null) {
       this.selectTree = HQLParser.getString(qb.getParseInfo().getSelForClause(
           clauseName)).toLowerCase();
+      this.selectAST = qb.getParseInfo().getSelForClause(
+          clauseName);
     }
     this.joinTree = qb.getParseInfo().getJoinExpr();
     extractMetaTables();
@@ -550,9 +557,9 @@ public class CubeQueryContext {
     if (joinTree.getJoinCond() != null) {
       builder.append("\n joinConds:");
       for (JoinCond cond: joinTree.getJoinCond()) {
-        builder.append("\n\t left: " + cond.getLeft() + " right: "
-            + cond.getRight() + " type:" + cond.getJoinType()
-            + " preserved:" + cond.getPreserved());
+        builder.append("\n\t left: " + cond.getLeft() + " right: " +
+            cond.getRight() + " type:" + cond.getJoinType() +
+            " preserved:" + cond.getPreserved());
       }
     }
     if (joinTree.getExpressions() != null) {
@@ -759,7 +766,7 @@ public class CubeQueryContext {
           factStorageTable));
     } else {
       if (originalWhereString != null) {
-      whereWithoutTimerange = new StringBuilder(originalWhereString);
+        whereWithoutTimerange = new StringBuilder(originalWhereString);
       } else {
         whereWithoutTimerange = new StringBuilder();
       }
@@ -885,13 +892,23 @@ public class CubeQueryContext {
   }
 
   public boolean isCubeMeasure(String col) {
-    String[] split = col.split("\\.");
+    // Take care of brackets added around col names in HQLParsrer.getString
+    if (col.startsWith("(") && col.endsWith(")") && col.length() > 2)
{
+      col = col.substring(1, col.length() -1);
+    }
+
+    String[] split = StringUtils.split(col, ".");
+    System.out.println("Looking for col [" + col + "] split: " + split.length);
     if (split.length <= 1) {
       return cubeMeasureNames.contains(col);
     } else {
-      if (split[0].equalsIgnoreCase(cube.getName()) ||
-          split[0].equalsIgnoreCase(getAliasForTabName(cube.getName()))) {
-        return cubeMeasureNames.contains(split[1]);
+      String cubeName = split[0].trim();
+      String colName = split[1].trim();
+      if (cubeName.equalsIgnoreCase(cube.getName()) ||
+          cubeName.equalsIgnoreCase(getAliasForTabName(cube.getName()))) {
+        boolean ismeasure = cubeMeasureNames.contains(colName);
+        System.out.println(colName + " IS MEASURE? " + ismeasure);
+        return cubeMeasureNames.contains(colName);
       } else {
         return false;
       }
@@ -925,4 +942,16 @@ public class CubeQueryContext {
     candidateFactTables.remove(fact);
   }
 
+  public void addAggregateExpr(String expr) {
+    aggregateExprs.add(expr);
+  }
+
+  public ASTNode getHavingAST() {
+    return havingAST;
+  }
+
+  public ASTNode getSelectAST() {
+    return selectAST;
+  }
+
 }

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/HQLParser.java
Mon May  6 11:10:40 2013
@@ -19,11 +19,11 @@ import org.apache.hadoop.hive.ql.parse.P
 import org.apache.hadoop.hive.ql.parse.ParseUtils;
 
 public class HQLParser {
-  
+
   public static interface ASTNodeVisitor {
     public void visit(TreeNode node);
   }
-  
+
   public static class TreeNode {
     final TreeNode parent;
     final ASTNode node;
@@ -31,18 +31,20 @@ public class HQLParser {
       this.parent = parent;
       this.node = node;
     }
-    
+
     public TreeNode getParent() {
       return parent;
     }
-    
+
     public ASTNode getNode() {
       return node;
     }
   }
-  
+
   public static final Set<Integer> BINARY_OPERATORS;
-  
+  public static final Set<Integer> ARITHMETIC_OPERATORS;
+
+
   static {
     HashSet<Integer> ops = new HashSet<Integer>();
     ops.add(DOT);
@@ -61,8 +63,21 @@ public class HQLParser {
     ops.add(MOD);
     ops.add(KW_LIKE);
     BINARY_OPERATORS = Collections.unmodifiableSet(ops);
+
+
+    ARITHMETIC_OPERATORS = new HashSet<Integer>();
+    ARITHMETIC_OPERATORS.add(PLUS);
+    ARITHMETIC_OPERATORS.add(MINUS);
+    ARITHMETIC_OPERATORS.add(STAR);
+    ARITHMETIC_OPERATORS.add(DIVIDE);
+    ARITHMETIC_OPERATORS.add(MOD);
+
+  }
+
+  public static boolean isArithmeticOp(int tokenType) {
+    return ARITHMETIC_OPERATORS.contains(tokenType);
   }
-  
+
   public static ASTNode parseHQL(String query) throws ParseException {
     ParseDriver driver = new ParseDriver();
     ASTNode tree = driver.parse(query);
@@ -70,7 +85,7 @@ public class HQLParser {
     printAST(tree);
     return tree;
   }
-  
+
   public static void printAST(ASTNode node) {
     try {
       printAST(getHiveTokenMapping(), node, 0, 0);
@@ -78,7 +93,7 @@ public class HQLParser {
       e.printStackTrace();
     }
   }
-  
+
   /**
    * Debug function for printing query AST to stdout
    * @param node
@@ -89,7 +104,7 @@ public class HQLParser {
     if (node == null || node.isNil()) {
       return;
     }
-    
+
     for (int i = 0; i < level; i++) {
       System.out.print("  ");
     }
@@ -97,10 +112,10 @@ public class HQLParser {
     System.out.print(node.getText() + " [" + tokenMapping.get(
         node.getToken().getType()) + "]");
     System.out.print(" (l"+level + "c" + child + ")");
-    
+
     if (node.getChildCount() > 0) {
       System.out.println(" {");
-      
+
       for (int i = 0; i < node.getChildCount(); i++) {
         Tree tree = node.getChild(i);
         if (tree instanceof ASTNode) {
@@ -110,21 +125,21 @@ public class HQLParser {
         }
         System.out.println();
       }
-      
+
       for (int i = 0; i < level; i++) {
         System.out.print("  ");
       }
-      
+
       System.out.print("}");
-      
+
     } else {
       System.out.print('$');
     }
   }
-  
+
   public static Map<Integer, String> getHiveTokenMapping() throws Exception {
     Map<Integer, String> mapping = new HashMap<Integer, String>();
-    
+
     for (Field f : HiveParser.class.getFields()) {
       if (f.getType() == int.class) {
         Integer tokenId = f.getInt(null);
@@ -132,15 +147,15 @@ public class HQLParser {
         mapping.put(tokenId, token);
       }
     }
-    
+
     return mapping;
   }
-  
+
 
   /**
    * Find a node in the tree rooted at root, given the path of type of tokens
    *  from the root's children to the desired node
-   * 
+   *
    * @param root
    * @param path starts at the level of root's children
    * @return
@@ -149,7 +164,7 @@ public class HQLParser {
     for (int i = 0; i < path.length; i++) {
       int type = path[i];
       boolean hasChildWithType = false;
-      
+
       for (int j = 0; j < root.getChildCount(); j++) {
         ASTNode node = (ASTNode) root.getChild(j);
         if (node.getToken().getType() == type) {
@@ -167,16 +182,16 @@ public class HQLParser {
           continue;
         }
       }
-      
+
       if (!hasChildWithType) {
         // No path from this level
         break;
       }
     }
-    
+
     return null;
   }
-  
+
   /**
    * Breadth first traversal of AST
    * @param root
@@ -186,13 +201,13 @@ public class HQLParser {
     if (root == null) {
       throw new NullPointerException("Root cannot be null");
     }
-    
+
     if (visitor == null) {
       throw new NullPointerException("Visitor cannot be null");
     }
     Queue<TreeNode> queue = new LinkedList<TreeNode>();
     queue.add(new TreeNode(null, root));
-    
+
     while (!queue.isEmpty()) {
       TreeNode node = queue.poll();
       visitor.visit(node);
@@ -202,7 +217,7 @@ public class HQLParser {
       }
     }
   }
-  
+
   /**
    * Recursively reconstruct query string given a query AST
    * @param root
@@ -241,18 +256,18 @@ public class HQLParser {
         if (i != root.getChildCount() -1) {
           buf.append(", ");
         }
-      } 
+      }
     } else {
       for (int i = 0; i < root.getChildCount(); i++) {
         toInfixString((ASTNode) root.getChild(i), buf);
       }
     }
   }
-  
+
   public static void main(String[] args) throws Exception {
-    ASTNode ast = parseHQL("select * from default_table " 
+    ASTNode ast = parseHQL("select * from default_table "
     		);
-    
+
     printAST(getHiveTokenMapping(), ast, 0, 0);
   }
 

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
Mon May  6 11:10:40 2013
@@ -17,7 +17,7 @@ public class LeastDimensionResolver impl
   @Override
   public void rewriteContext(CubeQueryContext cubeql)
       throws SemanticException {
-    if (cubeql.getCube() != null) {
+    if (cubeql.getCube() != null && !cubeql.getCandidateFactTables().isEmpty()) {
       Map<CubeFactTable, Integer> dimWeightMap =
           new HashMap<CubeFactTable, Integer>();
 

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
Mon May  6 11:10:40 2013
@@ -19,7 +19,7 @@ public class LeastPartitionResolver impl
   @Override
   public void rewriteContext(CubeQueryContext cubeql)
       throws SemanticException {
-    if (cubeql.getCube() != null) {
+    if (cubeql.getCube() != null && !cubeql.getCandidateFactTables().isEmpty()) {
       Map<CubeFactTable, Integer> numPartitionsMap =
           new HashMap<CubeFactTable, Integer>();
 

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
Mon May  6 11:10:40 2013
@@ -32,9 +32,10 @@ public class PartitionResolver implement
         CubeFactTable fact = i.next();
         Map<UpdatePeriod, List<String>> partitionColMap =
             new HashMap<UpdatePeriod, List<String>>();
-        factPartitionMap.put(fact, partitionColMap);
         if (!getPartitions(fact, fromDate, toDate, partitionColMap, cubeql)) {
           i.remove();
+        } else {
+          factPartitionMap.put(fact, partitionColMap);
         }
       }
       // set partition cols map in cubeql
@@ -51,7 +52,7 @@ public class PartitionResolver implement
 
     UpdatePeriod interval = fact.maxIntervalInRange(fromDate, toDate);
     if (interval == null) {
-      System.err.println("Could not find a partition for given range:"
+      System.err.println("Could not find partition for given range:"
           + fromDate + "-" + toDate + " in fact:" + fact.getName());
       return false;
     }

Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1479522&r1=1479521&r2=1479522&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Mon May  6 11:10:40 2013
@@ -1,6 +1,7 @@
 package org.apache.hadoop.hive.ql.cube.processors;
 
 import java.text.SimpleDateFormat;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Date;
 
@@ -274,4 +275,40 @@ public class TestCubeDriver {
     System.out.println("cube hql:" + hqlQuery);
   }
 
+  @Test
+  public void testAggregateResolver() throws Exception {
+    conf = new Configuration();
+    driver = new CubeDriver(new HiveConf(new Configuration(), HiveConf.class));
+
+    String timeRange = " where  time_range_in('2013-05-01', '2013-05-03')";
+    System.out.println("#$AGGREGATE_RESOLVER_ TIME_RANGE:" + timeRange);
+    String q1 = "SELECT countryid, testCube.msr2 from testCube " + timeRange;
+    String q2 = "SELECT countryid, testCube.msr2 * testCube.msr2 from testCube " + timeRange;
+    String q3 = "SELECT countryid, sum(testCube.msr2) from testCube " + timeRange;
+    String q4 = "SELECT countryid, sum(testCube.msr2) from testCube "  + timeRange
+        + " having testCube.msr2 > 100";
+    String q5 = "SELECT countryid, testCube.msr2 from testCube " + timeRange
+        + " having testCube.msr2 + testCube.msr2 > 100";
+    String q6 = "SELECT countryid, testCube.msr2 from testCube " + timeRange
+        + " having testCube.msr2 > 100 AND testCube.msr2 < 100";
+    String q7 = "SELECT countryid, sum(testCube.msr2) from testCube " + timeRange
+        + " having (testCube.msr2 > 100) OR (testcube.msr2 < 100 AND SUM(testcube.msr3)
> 1000)";
+
+    String tests[] = {q1, q2, q3, q4, q5, q6, q7};
+
+    int exceptions[] = new int[tests.length];
+    for (int i = 0; i < tests.length; i++) {
+      String hql = null;
+      try {
+        hql = driver.compileCubeQuery(tests[i]);
+      } catch (SemanticException exc) {
+        exceptions[i] = i;
+        exc.printStackTrace();
+      }
+      System.out.println("##----AGGREGATE_RESOLVER_CUBEQL----#" + i + " [" + tests[i] + "
]");
+      System.out.println("##----AGGREGATE_RESOLVER_HQL-----#" + i + " [ " + hql + " ]");
+    }
+    System.out.println("##---AGGREGATE_RESOLVER_ exceptions=" + Arrays.toString(exceptions)
);
+  }
+
 }



Mime
View raw message