lens-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pro...@apache.org
Subject [3/3] incubator-lens git commit: LENS-174: Cube rewriter changes to support multiple expressions
Date Tue, 26 May 2015 12:36:53 GMT
LENS-174: Cube rewriter changes to support multiple expressions


Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/82553db3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/82553db3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/82553db3

Branch: refs/heads/master
Commit: 82553db30abbc981c7e7bbc3788faa7db3270156
Parents: ca30cf8
Author: Amareshwari Sriramadasu <amareshwari@apache.org>
Authored: Tue May 26 18:06:41 2015 +0530
Committer: Rajat Khandelwal <rajatgupta59@gmail.com>
Committed: Tue May 26 18:06:41 2015 +0530

----------------------------------------------------------------------
 .../apache/lens/cube/metadata/Dimension.java    |   6 +-
 .../apache/lens/cube/metadata/ExprColumn.java   |   2 +-
 .../lens/cube/parse/AggregateResolver.java      | 113 ++-
 .../apache/lens/cube/parse/AliasReplacer.java   |  46 +-
 .../apache/lens/cube/parse/CandidateFact.java   |  12 +-
 .../cube/parse/CandidateTablePruneCause.java    |  28 +-
 .../lens/cube/parse/CandidateTableResolver.java |  73 +-
 .../apache/lens/cube/parse/ColumnResolver.java  |  26 +-
 .../lens/cube/parse/CubeQueryContext.java       |  98 ++-
 .../lens/cube/parse/CubeQueryRewriter.java      |  15 +-
 .../cube/parse/DenormalizationResolver.java     |  60 +-
 .../lens/cube/parse/ExpressionResolver.java     | 744 +++++++++++++++++--
 .../apache/lens/cube/parse/FieldValidator.java  |  57 +-
 .../apache/lens/cube/parse/JoinResolver.java    |   4 +-
 .../lens/cube/parse/TimerangeResolver.java      |   1 -
 .../lens/cube/parse/TrackQueriedColumns.java    |  27 +
 .../apache/lens/cube/parse/CubeTestSetup.java   |  46 +-
 .../FieldsCannotBeQueriedTogetherTest.java      | 138 +++-
 .../lens/cube/parse/TestBaseCubeQueries.java    | 112 +--
 .../cube/parse/TestDenormalizationResolver.java |  45 +-
 .../lens/cube/parse/TestExpressionContext.java  | 111 +++
 .../lens/cube/parse/TestExpressionResolver.java | 306 ++++++--
 .../lens/cube/parse/TestJoinResolver.java       |  54 +-
 .../lens/cube/parse/TestQueryMetrics.java       |  14 +-
 lens-cube/src/test/resources/log4j.properties   |   8 +-
 25 files changed, 1726 insertions(+), 420 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
index 1a4b581..9cba3a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
@@ -171,7 +171,11 @@ public class Dimension extends AbstractBaseTable {
   }
 
   public CubeColumn getColumnByName(String column) {
-    return getAttributeByName(column);
+    CubeColumn cubeCol = super.getExpressionByName(column);
+    if (cubeCol == null) {
+      cubeCol = getAttributeByName(column);
+    }
+    return cubeCol;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
index a7f711f..3fc661f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
@@ -142,7 +142,7 @@ public class ExprColumn extends CubeColumn {
       this.endTime = endTime;
     }
 
-    synchronized ASTNode getASTNode() {
+    public synchronized ASTNode getASTNode() {
       if (astNode == null) {
         try {
           if (StringUtils.isNotBlank(expr)) {

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index 76b5729..2ad2b7c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -125,7 +125,7 @@ class AggregateResolver implements ContextRewriter {
   }
 
   private void transform(CubeQueryContext cubeql, ASTNode parent, ASTNode node, int nodePos) throws SemanticException {
-    if (parent == null || node == null) {
+    if (node == null) {
       return;
     }
     int nodeType = node.getToken().getType();
@@ -135,16 +135,18 @@ class AggregateResolver implements ContextRewriter {
         // Leaf node
         ASTNode wrapped = wrapAggregate(cubeql, node);
         if (wrapped != node) {
-          parent.setChild(nodePos, wrapped);
-          // Check if this node has an alias
-          ASTNode sibling = HQLParser.findNodeByPath(parent, Identifier);
-          String expr;
-          if (sibling != null) {
-            expr = HQLParser.getString(parent);
-          } else {
-            expr = HQLParser.getString(wrapped);
+          if (parent != null) {
+            parent.setChild(nodePos, wrapped);
+            // Check if this node has an alias
+            ASTNode sibling = HQLParser.findNodeByPath(parent, Identifier);
+            String expr;
+            if (sibling != null) {
+              expr = HQLParser.getString(parent);
+            } else {
+              expr = HQLParser.getString(wrapped);
+            }
+            cubeql.addAggregateExpr(expr.trim());
           }
-          cubeql.addAggregateExpr(expr.trim());
         }
       } else {
         // Dig deeper in non-leaf nodes
@@ -176,23 +178,31 @@ class AggregateResolver implements ContextRewriter {
     String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
 
     if (cubeql.isCubeMeasure(msrname)) {
-      CubeMeasure measure = cubeql.getCube().getMeasureByName(colname);
-      String aggregateFn = measure.getAggregate();
+      if (cubeql.getQueriedExprs().contains(colname)) {
+        String alias = cubeql.getAliasForTableName(cubeql.getCube().getName());
+        for (ASTNode exprNode : cubeql.getExprCtx().getExpressionContext(colname, alias).getAllASTNodes()) {
+          transform(cubeql, null, exprNode, 0);
+        }
+        return node;
+      } else {
+        CubeMeasure measure = cubeql.getCube().getMeasureByName(colname);
+        String aggregateFn = measure.getAggregate();
 
-      if (StringUtils.isBlank(aggregateFn)) {
-        throw new SemanticException(ErrorMsg.NO_DEFAULT_AGGREGATE, colname);
-      }
-      ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION));
-      fnroot.setParent(node.getParent());
+        if (StringUtils.isBlank(aggregateFn)) {
+          throw new SemanticException(ErrorMsg.NO_DEFAULT_AGGREGATE, colname);
+        }
+        ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION));
+        fnroot.setParent(node.getParent());
 
-      ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn));
-      fnIdentNode.setParent(fnroot);
-      fnroot.addChild(fnIdentNode);
+        ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn));
+        fnIdentNode.setParent(fnroot);
+        fnroot.addChild(fnIdentNode);
 
-      node.setParent(fnroot);
-      fnroot.addChild(node);
+        node.setParent(fnroot);
+        fnroot.addChild(node);
 
-      return fnroot;
+        return fnroot;
+      }
     } else {
       return node;
     }
@@ -208,7 +218,7 @@ class AggregateResolver implements ContextRewriter {
       if (node.getChild(0).getType() == HiveParser.Identifier) {
         function = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0).getText());
       }
-    } else if (isMeasure(cubeql, node)) {
+    } else if (cubeql.isMeasure(node)) {
       // Exit for the recursion
 
       String colname;
@@ -219,16 +229,27 @@ class AggregateResolver implements ContextRewriter {
         ASTNode colIdent = (ASTNode) node.getChild(1);
         colname = colIdent.getText();
       }
-      CubeMeasure measure = cubeql.getCube().getMeasureByName(colname);
-      if (function != null && !function.isEmpty()) {
-        // Get the cube measure object and check if the passed function is the
-        // default one set for this measure
-        return !function.equalsIgnoreCase(measure.getAggregate());
-      } else if (!aggregateResolverDisabled && measure.getAggregate() != null) {
-        // not inside any aggregate, but default aggregate exists
+      colname = colname.toLowerCase();
+      if (cubeql.getQueriedExprs().contains(colname)) {
+        String cubeAlias = cubeql.getAliasForTableName(cubeql.getCube().getName());
+        for (ASTNode exprNode : cubeql.getExprCtx().getExpressionContext(colname, cubeAlias).getAllASTNodes()) {
+          if (hasMeasuresNotInDefaultAggregates(cubeql, exprNode, function, aggregateResolverDisabled)) {
+            return true;
+          }
+        }
         return false;
+      } else {
+        CubeMeasure measure = cubeql.getCube().getMeasureByName(colname);
+        if (function != null && !function.isEmpty()) {
+          // Get the cube measure object and check if the passed function is the
+          // default one set for this measure
+          return !function.equalsIgnoreCase(measure.getAggregate());
+        } else if (!aggregateResolverDisabled && measure.getAggregate() != null) {
+          // not inside any aggregate, but default aggregate exists
+          return false;
+        }
+        return true;
       }
-      return true;
     }
 
     for (int i = 0; i < node.getChildCount(); i++) {
@@ -270,7 +291,7 @@ class AggregateResolver implements ContextRewriter {
     boolean isDistinct = hasDistinct;
     if (exprTokenType == HiveParser.TOK_FUNCTIONDI || exprTokenType == HiveParser.TOK_SELECTDI) {
       isDistinct = true;
-    } else if (isMeasure(cubeql, node) && isDistinct) {
+    } else if (cubeql.isMeasure(node) && isDistinct) {
       // Exit for the recursion
       return true;
     }
@@ -289,7 +310,7 @@ class AggregateResolver implements ContextRewriter {
       return false;
     }
 
-    if (isMeasure(cubeql, node)) {
+    if (cubeql.isMeasure(node)) {
       return true;
     }
 
@@ -302,30 +323,6 @@ class AggregateResolver implements ContextRewriter {
     return false;
   }
 
-  private boolean isMeasure(CubeQueryContext cubeql, ASTNode node) {
-    String tabname = null;
-    String colname;
-    int nodeType = node.getToken().getType();
-    if (!(nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT)) {
-      return false;
-    }
-
-    if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
-      colname = ((ASTNode) node.getChild(0)).getText();
-    } else {
-      // node in 'alias.column' format
-      ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
-      ASTNode colIdent = (ASTNode) node.getChild(1);
-
-      colname = colIdent.getText();
-      tabname = tabident.getText();
-    }
-
-    String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
-
-    return cubeql.isCubeMeasure(msrname);
-  }
-
   static void updateAggregates(ASTNode root, CubeQueryContext cubeql) {
     if (root == null) {
       return;

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
index 9d367c3..4d3443c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
@@ -21,18 +21,14 @@ package org.apache.lens.cube.parse;
 import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
 import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_SELEXPR;
 
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.lens.cube.metadata.AbstractCubeTable;
 import org.apache.lens.cube.metadata.CubeInterface;
 import org.apache.lens.cube.metadata.Dimension;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -50,21 +46,20 @@ import org.antlr.runtime.CommonToken;
  */
 class AliasReplacer implements ContextRewriter {
 
-  private static final Log LOG = LogFactory.getLog(AliasReplacer.class.getName());
-
-  // Mapping of a qualified column name to its table alias
-  private Map<String, String> colToTableAlias;
-
   public AliasReplacer(Configuration conf) {
   }
 
   @Override
   public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
-    colToTableAlias = new HashMap<String, String>();
+    Map<String, String> colToTableAlias = cubeql.getColToTableAlias();
 
     extractTabAliasForCol(cubeql);
     findDimAttributesAndMeasures(cubeql);
 
+    if (colToTableAlias.isEmpty()) {
+      return;
+    }
+
     // Rewrite the all the columns in the query with table alias prefixed.
     // If col1 of table tab1 is accessed, it would be changed as tab1.col1.
     // If tab1 is already aliased say with t1, col1 is changed as t1.col1
@@ -76,10 +71,6 @@ class AliasReplacer implements ContextRewriter {
     // 2: (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key))
     // (TOK_SELEXPR (TOK_FUNCTION count (. (TOK_TABLE_OR_COL src) value))))
     // 3: (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) srckey))))
-    if (colToTableAlias == null) {
-      return;
-    }
-
     replaceAliases(cubeql.getSelectAST(), 0, colToTableAlias);
 
     replaceAliases(cubeql.getHavingAST(), 0, colToTableAlias);
@@ -92,13 +83,11 @@ class AliasReplacer implements ContextRewriter {
 
     replaceAliases(cubeql.getJoinTree(), 0, colToTableAlias);
 
-
     // Update the aggregate expression set
     AggregateResolver.updateAggregates(cubeql.getSelectAST(), cubeql);
     AggregateResolver.updateAggregates(cubeql.getHavingAST(), cubeql);
     // Update alias map as well
     updateAliasMap(cubeql.getSelectAST(), cubeql);
-
   }
 
   /**
@@ -112,22 +101,31 @@ class AliasReplacer implements ContextRewriter {
       Set<String> cubeColsQueried = cubeql.getColumnsQueried(cube.getName());
       Set<String> queriedDimAttrs = new HashSet<String>();
       Set<String> queriedMsrs = new HashSet<String>();
+      Set<String> queriedExprs = new HashSet<String>();
       if (cubeColsQueried != null && !cubeColsQueried.isEmpty()) {
         for (String col : cubeColsQueried) {
           if (cube.getMeasureNames().contains(col)) {
             queriedMsrs.add(col);
           } else if (cube.getDimAttributeNames().contains(col)) {
             queriedDimAttrs.add(col);
+          } else if (cube.getExpressionNames().contains(col)) {
+            queriedExprs.add(col);
           }
         }
       }
       cubeql.addQueriedDimAttrs(queriedDimAttrs);
       cubeql.addQueriedMsrs(queriedMsrs);
+      cubeql.addQueriedExprs(queriedExprs);
     }
   }
 
   private void extractTabAliasForCol(CubeQueryContext cubeql) throws SemanticException {
-    Set<String> columns = cubeql.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE);
+    extractTabAliasForCol(cubeql.getColToTableAlias(), cubeql, cubeql);
+  }
+
+  static void extractTabAliasForCol(Map<String, String> colToTableAlias, CubeQueryContext cubeql,
+    TrackQueriedColumns tqc) throws SemanticException {
+    Set<String> columns = tqc.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE);
     if (columns == null) {
       return;
     }
@@ -136,8 +134,9 @@ class AliasReplacer implements ContextRewriter {
       if (cubeql.getCube() != null) {
         Set<String> cols = cubeql.getCube().getAllFieldNames();
         if (cols.contains(col.toLowerCase())) {
-          colToTableAlias.put(col.toLowerCase(), cubeql.getAliasForTableName(cubeql.getCube().getName()));
-          cubeql.addColumnsQueried((AbstractCubeTable) cubeql.getCube(), col.toLowerCase());
+          String cubeAlias = cubeql.getAliasForTableName(cubeql.getCube().getName());
+          colToTableAlias.put(col.toLowerCase(), cubeAlias);
+          tqc.addColumnsQueried(cubeAlias, col.toLowerCase());
           inCube = true;
         }
       }
@@ -148,8 +147,9 @@ class AliasReplacer implements ContextRewriter {
             if (prevDim != null && !prevDim.equals(dim.getName())) {
               throw new SemanticException(ErrorMsg.AMBIGOUS_DIM_COLUMN, col, prevDim, dim.getName());
             }
-            colToTableAlias.put(col.toLowerCase(), cubeql.getAliasForTableName(dim.getName()));
-            cubeql.addColumnsQueried(dim, col.toLowerCase());
+            String dimAlias = cubeql.getAliasForTableName(dim.getName());
+            colToTableAlias.put(col.toLowerCase(), dimAlias);
+            tqc.addColumnsQueried(dimAlias, col.toLowerCase());
           } else {
             // throw error because column is in both cube and dimension table
             throw new SemanticException(ErrorMsg.AMBIGOUS_CUBE_COLUMN, col, cubeql.getCube().getName(), dim.getName());
@@ -162,7 +162,7 @@ class AliasReplacer implements ContextRewriter {
     }
   }
 
-  private void replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) {
+  static void replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) {
     if (node == null) {
       return;
     }
@@ -211,7 +211,7 @@ class AliasReplacer implements ContextRewriter {
     }
   }
 
-  private void updateAliasMap(ASTNode root, CubeQueryContext cubeql) {
+  static void updateAliasMap(ASTNode root, CubeQueryContext cubeql) {
     if (root == null) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 52bf9aa..63d2508 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -67,6 +67,7 @@ public class CandidateFact implements CandidateTable {
   private ASTNode whereAST;
   private ASTNode groupbyAST;
   private ASTNode havingAST;
+  private ASTNode joinTree;
   private List<TimeRangeNode> timenodes = Lists.newArrayList();
   private final List<Integer> selectIndices = Lists.newArrayList();
   private final List<Integer> dimFieldIndices = Lists.newArrayList();
@@ -140,6 +141,9 @@ public class CandidateFact implements CandidateTable {
   public void copyASTs(CubeQueryContext cubeql) throws SemanticException {
     this.selectAST = HQLParser.copyAST(cubeql.getSelectAST());
     this.whereAST = HQLParser.copyAST(cubeql.getWhereAST());
+    if (cubeql.getJoinTree() != null) {
+      this.joinTree = HQLParser.copyAST(cubeql.getJoinTree());
+    }
     if (cubeql.getGroupByAST() != null) {
       this.groupbyAST = HQLParser.copyAST(cubeql.getGroupByAST());
     }
@@ -180,13 +184,13 @@ public class CandidateFact implements CandidateTable {
    * @throws SemanticException
    */
   public void updateASTs(CubeQueryContext cubeql) throws SemanticException {
-    Set<String> cubeColsQueried = cubeql.getColumnsQueried(cubeql.getCube().getName());
+    Set<String> cubeCols = cubeql.getCube().getAllFieldNames();
 
     // update select AST with selected fields
     int currentChild = 0;
     for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) {
       ASTNode selectExpr = (ASTNode) this.selectAST.getChild(currentChild);
-      Set<String> exprCols = getColsInExpr(cubeColsQueried, selectExpr);
+      Set<String> exprCols = getColsInExpr(cubeCols, selectExpr);
       if (getColumns().containsAll(exprCols)) {
         selectIndices.add(i);
         if (cubeql.getQueriedDimAttrs().containsAll(exprCols)) {
@@ -419,4 +423,8 @@ public class CandidateFact implements CandidateTable {
     }
     return timePartDimensions;
   }
+
+  public ASTNode getJoinTree() {
+    return joinTree;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 8c009b2..5a1f8f9 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -58,6 +58,16 @@ public class CandidateTablePruneCause {
     MORE_PARTITIONS("Picked table has more partitions than minimum"),
     // invalid cube table
     INVALID("Invalid cube table provided in query"),
+    // expression is not evaluable in the candidate
+    EXPRESSION_NOT_EVALUABLE("%s expressions not evaluable") {
+      Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
+        List<String> columns = new ArrayList<String>();
+        for (CandidateTablePruneCause cause : causes) {
+          columns.addAll(cause.getMissingExpressions());
+        }
+        return new String[]{columns.toString()};
+      }
+    },
     // column not valid in cube table
     COLUMN_NOT_VALID("Column not valid in cube table"),
     // column not found in cube table
@@ -206,6 +216,8 @@ public class CandidateTablePruneCause {
   private List<String> missingUpdatePeriods;
   // populated in case of missing columns
   private List<String> missingColumns;
+  // populated in case of expressions not evaluable
+  private List<String> missingExpressions;
   // populated in case of no column part of a join path
   private List<String> joinColumns;
   // the columns that are missing default aggregate. only set in case of MISSING_DEFAULT_AGGREGATE
@@ -227,9 +239,11 @@ public class CandidateTablePruneCause {
     return cause;
   }
 
-  public static CandidateTablePruneCause columnNotFound(Collection<String> missingColumns) {
+  public static CandidateTablePruneCause columnNotFound(Collection<String>... missingColumns) {
     List<String> colList = new ArrayList<String>();
-    colList.addAll(missingColumns);
+    for (Collection<String> missing : missingColumns) {
+      colList.addAll(missing);
+    }
     CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.COLUMN_NOT_FOUND);
     cause.setMissingColumns(colList);
     return cause;
@@ -243,6 +257,16 @@ public class CandidateTablePruneCause {
     return columnNotFound(colList);
   }
 
+  public static CandidateTablePruneCause expressionNotEvaluable(String... exprs) {
+    List<String> colList = new ArrayList<String>();
+    for (String column : exprs) {
+      colList.add(column);
+    }
+    CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE);
+    cause.setMissingExpressions(colList);
+    return cause;
+  }
+
   public static CandidateTablePruneCause missingPartitions(Set<String> nonExistingParts) {
     CandidateTablePruneCause cause =
       new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_PARTITIONS);

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 40561ad..d56fb80 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -248,44 +248,76 @@ class CandidateTableResolver implements ContextRewriter {
             }
           }
         }
+        // go over expressions queried
+        // if expression has no measures, prune facts which cannot evaluate expression
+        // if expression has measures, they should be considered along with other measures and see if the fact can be
+        // part of measure covering set
+        for (String expr : cubeql.getQueriedExprs()) {
+          cubeql.getExprCtx().updateEvaluables(expr, cfact);
+          if (!cubeql.getQueriedExprsWithMeasures().contains(expr) && !cubeql.getExprCtx().isEvaluable(expr, cfact)) {
+            // if expression has no measures, prune facts which cannot evaluate expression
+            LOG.info("Not considering fact table:" + cfact + " as expression " + expr + " is not evaluatable");
+            cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.expressionNotEvaluable(expr));
+            toRemove = true;
+            break;
+          }
+        }
         // check if the candidate fact has atleast one measure queried
-        if (!checkForColumnExists(cfact, queriedMsrs)) {
+        // if expression has measures, they should be considered along with other measures and see if the fact can be
+        // part of measure covering set
+        if (!checkForColumnExists(cfact, queriedMsrs)
+          && (cubeql.getQueriedExprsWithMeasures().isEmpty()
+            || cubeql.getExprCtx().allNotEvaluable(cubeql.getQueriedExprsWithMeasures(), cfact))) {
           LOG.info("Not considering fact table:" + cfact + " as columns " + queriedMsrs + " is not available");
-          cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(queriedMsrs));
+          cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(queriedMsrs,
+            cubeql.getQueriedExprsWithMeasures()));
           toRemove = true;
         }
         if (toRemove) {
           i.remove();
         }
       }
+      Set<String> dimExprs = new HashSet<String>(cubeql.getQueriedExprs());
+      dimExprs.removeAll(cubeql.getQueriedExprsWithMeasures());
+      if (cubeql.getCandidateFacts().size() == 0) {
+        throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN,
+          (!queriedDimAttrs.isEmpty() ? queriedDimAttrs.toString() : "")
+          +  (!dimExprs.isEmpty() ? dimExprs.toString() : ""));
+      }
       // Find out candidate fact table sets which contain all the measures
       // queried
       List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
-      Set<Set<CandidateFact>> cfactset = findCoveringSets(cfacts, queriedMsrs);
+      Set<Set<CandidateFact>> cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
+        cubeql.getQueriedExprsWithMeasures());
       LOG.info("Measure covering fact sets :" + cfactset);
+      String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "")
+        + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : "");
       if (cfactset.isEmpty()) {
-        throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, queriedMsrs.toString());
+        throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
       }
       cubeql.getCandidateFactSets().addAll(cfactset);
-      cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.COLUMN_NOT_FOUND);
+      cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs,
+        cubeql.getQueriedExprsWithMeasures()));
 
       if (cubeql.getCandidateFacts().size() == 0) {
-        throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, queriedDimAttrs.toString());
+        throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
       }
     }
   }
 
-  static Set<Set<CandidateFact>> findCoveringSets(List<CandidateFact> cfactsPassed, Set<String> msrs) {
+  static Set<Set<CandidateFact>> findCoveringSets(CubeQueryContext cubeql, List<CandidateFact> cfactsPassed,
+    Set<String> msrs, Set<String> exprsWithMeasures) {
     Set<Set<CandidateFact>> cfactset = new HashSet<Set<CandidateFact>>();
     List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cfactsPassed);
     for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) {
       CandidateFact cfact = i.next();
       i.remove();
-      if (!checkForColumnExists(cfact, msrs)) {
+      if (!checkForColumnExists(cfact, msrs)
+        && (exprsWithMeasures.isEmpty() || cubeql.getExprCtx().allNotEvaluable(exprsWithMeasures, cfact))) {
         // check if fact contains any of the maeasures
         // if not ignore the fact
         continue;
-      } else if (cfact.getColumns().containsAll(msrs)) {
+      } else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) {
         // return single set
         Set<CandidateFact> one = new LinkedHashSet<CandidateFact>();
         one.add(cfact);
@@ -293,15 +325,17 @@ class CandidateTableResolver implements ContextRewriter {
       } else {
         // find the remaining measures in other facts
         Set<String> remainingMsrs = new HashSet<String>(msrs);
+        Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures);
         remainingMsrs.removeAll(cfact.getColumns());
-        Set<Set<CandidateFact>> coveringSets = findCoveringSets(cfacts, remainingMsrs);
+        remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact));
+        Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs);
         if (!coveringSets.isEmpty()) {
           for (Set<CandidateFact> set : coveringSets) {
             set.add(cfact);
             cfactset.add(set);
           }
         } else {
-          LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs);
+          LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs);
         }
       }
     }
@@ -503,9 +537,20 @@ class CandidateTableResolver implements ContextRewriter {
           if (cubeql.getColumnsQueried(dim.getName()) != null) {
             for (String col : cubeql.getColumnsQueried(dim.getName())) {
               if (!cdim.getColumns().contains(col.toLowerCase())) {
-                // check if it available as reference, if not remove the
-                // candidate
-                if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, dim.getName())) {
+                // check if the column is an expression
+                if (cdim.getBaseTable().getExpressionNames().contains(col)) {
+                  cubeql.getExprCtx().updateEvaluables(col, cdim);
+                  // check if the expression is evaluatable
+                  if (!cubeql.getExprCtx().isEvaluable(col, cdim)) {
+                    LOG.info("Not considering dimtable:" + cdim + " as expression " + col + " is not evaluatable");
+                    cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.expressionNotEvaluable(
+                      col));
+                    i.remove();
+                    break;
+                  }
+                } else if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, dim.getName())) {
+                  // check if it available as reference, if not remove the
+                  // candidate
                   LOG.info("Not considering dimtable:" + cdim + " as column " + col + " is not available");
                   cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.columnNotFound(col));
                   i.remove();

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
index 1aa33db..0849381 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
@@ -28,7 +28,6 @@ import org.apache.lens.cube.parse.HQLParser.TreeNode;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
@@ -60,27 +59,24 @@ class ColumnResolver implements ContextRewriter {
     }
     getColsForSelectTree(cubeql);
     getColsForWhereTree(cubeql);
-    getColsForTree(cubeql, cubeql.getJoinTree());
-    getColsForTree(cubeql, cubeql.getGroupByAST());
-    getColsForTree(cubeql, cubeql.getHavingAST());
-    getColsForTree(cubeql, cubeql.getOrderByAST());
+    getColsForTree(cubeql, cubeql.getJoinTree(), cubeql);
+    getColsForTree(cubeql, cubeql.getGroupByAST(), cubeql);
+    getColsForTree(cubeql, cubeql.getHavingAST(), cubeql);
+    getColsForTree(cubeql, cubeql.getOrderByAST(), cubeql);
 
     // Update join dimension tables
     for (String table : cubeql.getTblAliasToColumns().keySet()) {
-      try {
-        if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table)) {
-          if (!cubeql.addQueriedTable(table)) {
-            throw new SemanticException(ErrorMsg.NEITHER_CUBE_NOR_DIMENSION);
-          }
+      if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table)) {
+        if (!cubeql.addQueriedTable(table)) {
+          throw new SemanticException(ErrorMsg.NEITHER_CUBE_NOR_DIMENSION);
         }
-      } catch (HiveException e) {
-        throw new SemanticException(e);
       }
     }
   }
 
   // finds columns in AST passed.
-  private void getColsForTree(final CubeQueryContext cubeql, ASTNode tree) throws SemanticException {
+  static void getColsForTree(final CubeQueryContext cubeql, ASTNode tree, final TrackQueriedColumns tqc)
+    throws SemanticException {
     if (tree == null) {
       return;
     }
@@ -104,7 +100,7 @@ class ColumnResolver implements ContextRewriter {
             // column is an existing alias
             return;
           }
-          cubeql.addColumnsQueried(CubeQueryContext.DEFAULT_TABLE, column);
+          tqc.addColumnsQueried(CubeQueryContext.DEFAULT_TABLE, column);
         } else if (node.getToken().getType() == DOT) {
           // This is for the case where column name is prefixed by table name
           // or table alias
@@ -116,7 +112,7 @@ class ColumnResolver implements ContextRewriter {
           String column = colIdent.getText().toLowerCase();
           String table = tabident.getText().toLowerCase();
 
-          cubeql.addColumnsQueried(table, column);
+          tqc.addColumnsQueried(table, column);
         }
       }
     });

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 3964c1a..0409af3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -19,6 +19,9 @@
 
 package org.apache.lens.cube.parse;
 
+import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
+
 import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE;
 
 import java.io.ByteArrayOutputStream;
@@ -45,7 +48,7 @@ import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
 
-public class CubeQueryContext {
+public class CubeQueryContext implements TrackQueriedColumns {
   public static final String TIME_RANGE_FUNC = "time_range_in";
   public static final String NOW = "now";
   public static final String DEFAULT_TABLE = "_default_";
@@ -78,6 +81,16 @@ public class CubeQueryContext {
   @Getter
   private final Set<String> queriedMsrs = new HashSet<String>();
 
+  @Getter
+  private final Set<String> queriedExprs = new HashSet<String>();
+
+  @Getter
+  private final Set<String> queriedExprsWithMeasures = new HashSet<String>();
+
+  @Getter
+  // Mapping of a qualified column name to its table alias
+  private final Map<String, String> colToTableAlias = new HashMap<String, String>();
+
   @Getter()
   private final Set<Set<CandidateFact>> candidateFactSets = new HashSet<Set<CandidateFact>>();
 
@@ -134,6 +147,9 @@ public class CubeQueryContext {
   private JoinResolver.AutoJoinContext autoJoinCtx;
   @Getter
   @Setter
+  private ExpressionResolver.ExpressionResolverContext exprCtx;
+  @Getter
+  @Setter
   private DenormalizationResolver.DenormalizationContext deNormCtx;
   @Getter
   private PruneCauses<CubeFactTable> factPruningMsgs =
@@ -341,7 +357,7 @@ public class CubeQueryContext {
     boolean isRequiredInJoinChain = false;
   }
 
-  public void addOptionalDimTable(String alias, String col, CandidateTable candidate, boolean isRequiredInJoin)
+  public void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String... cols)
     throws SemanticException {
     alias = alias.toLowerCase();
     try {
@@ -354,8 +370,10 @@ public class CubeQueryContext {
         optDim = new OptionalDimCtx();
         optionalDimensions.put(dim, optDim);
       }
-      if (col != null && candidate != null) {
-        optDim.colQueried.add(col);
+      if (cols != null && candidate != null) {
+        for (String col : cols) {
+          optDim.colQueried.add(col);
+        }
         optDim.requiredForCandidates.add(candidate);
       }
       if (!optDim.isRequiredInJoinChain) {
@@ -413,7 +431,7 @@ public class CubeQueryContext {
         return cubeTblEntry.getKey();
       }
     }
-    return tableName;
+    return tableName.toLowerCase();
   }
 
   public void print() {
@@ -784,6 +802,22 @@ public class CubeQueryContext {
       }
     }
 
+    // pick dimension tables required during expression expansion for the picked fact and dimensions
+    Set<Dimension> exprDimTables = new HashSet<Dimension>();
+    if (cfacts != null) {
+      for (CandidateFact cfact : cfacts) {
+        Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1);
+        exprDimTables.addAll(factExprDimTables);
+        if (cfacts.size() > 1) {
+          factDimMap.get(cfact).addAll(factExprDimTables);
+        }
+      }
+    } else {
+      // dim only query
+      exprDimTables.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false));
+    }
+    dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimTables));
+
     // pick denorm tables for the picked fact and dimensions
     Set<Dimension> denormTables = new HashSet<Dimension>();
     if (cfacts != null) {
@@ -891,25 +925,57 @@ public class CubeQueryContext {
 
     String[] split = StringUtils.split(col, ".");
     if (split.length <= 1) {
-      return cube.getMeasureNames().contains(col.trim().toLowerCase());
+      col = col.trim().toLowerCase();
+      if (queriedExprs.contains(col)) {
+        return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).isHasMeasures();
+      } else {
+        return cube.getMeasureNames().contains(col);
+      }
     } else {
-      String cubeName = split[0].trim();
-      String colName = split[1].trim();
-      if (cubeName.equalsIgnoreCase(cube.getName())
-        || cubeName.equalsIgnoreCase(getAliasForTableName(cube.getName()))) {
-        return cube.getMeasureNames().contains(colName.toLowerCase());
+      String cubeName = split[0].trim().toLowerCase();
+      String colName = split[1].trim().toLowerCase();
+      if (cubeName.equalsIgnoreCase(cube.getName()) || cubeName.equals(getAliasForTableName(cube.getName()))) {
+        if (queriedExprs.contains(colName)) {
+          return exprCtx.getExpressionContext(colName, cubeName).isHasMeasures();
+        } else {
+          return cube.getMeasureNames().contains(colName.toLowerCase());
+        }
       } else {
         return false;
       }
     }
   }
 
+  boolean isMeasure(ASTNode node) {
+    String tabname = null;
+    String colname;
+    int nodeType = node.getToken().getType();
+    if (!(nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT)) {
+      return false;
+    }
+
+    if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
+      colname = ((ASTNode) node.getChild(0)).getText();
+    } else {
+      // node in 'alias.column' format
+      ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
+      ASTNode colIdent = (ASTNode) node.getChild(1);
+
+      colname = colIdent.getText();
+      tabname = tabident.getText();
+    }
+
+    String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
+
+    return isCubeMeasure(msrname);
+  }
+
   public boolean isAggregateExpr(String expr) {
     return aggregateExprs.contains(expr == null ? null : expr.toLowerCase());
   }
 
   public boolean hasAggregates() {
-    return !aggregateExprs.isEmpty();
+    return !aggregateExprs.isEmpty() || getExprCtx().hasAggregates();
   }
 
   public String getAlias(String expr) {
@@ -1035,6 +1101,14 @@ public class CubeQueryContext {
     queriedMsrs.addAll(msrs);
   }
 
+  public void addQueriedExprs(Set<String> exprs) {
+    queriedExprs.addAll(exprs);
+  }
+
+  public void addQueriedExprsWithMeasures(Set<String> exprs) {
+    queriedExprsWithMeasures.addAll(exprs);
+  }
+
   /**
    * Prune candidate fact sets with respect to available candidate facts.
    * <p/>

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index b7a92e7..57c91e7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -132,15 +132,15 @@ public class CubeQueryRewriter {
    * removed.
    */
   private void setupRewriters() {
-    // Rewrite base trees with expressions expanded
-    rewriters.add(new ExpressionResolver(conf));
     // Resolve columns - the column alias and table alias
     rewriters.add(new ColumnResolver(conf));
     // Rewrite base trees (groupby, having, orderby, limit) using aliases
     rewriters.add(new AliasReplacer(conf));
+    ExpressionResolver exprResolver = new ExpressionResolver(conf);
     DenormalizationResolver denormResolver = new DenormalizationResolver(conf);
     CandidateTableResolver candidateTblResolver = new CandidateTableResolver(conf);
     StorageTableResolver storageTableResolver = new StorageTableResolver(conf);
+    rewriters.add(exprResolver);
     // De-normalized columns resolved
     rewriters.add(denormResolver);
     // Resolve candidate fact tables and dimension tables for columns queried
@@ -160,18 +160,21 @@ public class CubeQueryRewriter {
     // Phase 1: resolve fact tables.
     rewriters.add(storageTableResolver);
     if (lightFactFirst) {
+      // Prune candidate tables for which denorm column references do not exist
+      rewriters.add(denormResolver);
+      // Prune candidate facts without any valid expressions
+      rewriters.add(exprResolver);
       rewriters.add(new LightestFactResolver(conf));
     }
     // Phase 2: resolve fact table partitions.
     rewriters.add(storageTableResolver);
     rewriters.add(new MaxCoveringFactResolver(conf));
-    if (!lightFactFirst) {
-      rewriters.add(new LightestFactResolver(conf));
-    }
     // Phase 3: resolve dimension tables and partitions.
     rewriters.add(storageTableResolver);
-    // Check for candidate tables using de-normalized columns
+    // Prune candidate tables for which denorm column references do not exist
     rewriters.add(denormResolver);
+    // Prune candidate facts without any valid expressions
+    rewriters.add(exprResolver);
     rewriters.add(new LeastPartitionResolver(conf));
     if (!lightFactFirst) {
       rewriters.add(new LightestFactResolver(conf));

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index e0f7bea..1a7b6b4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -25,6 +25,8 @@ import java.util.*;
 
 import org.apache.lens.cube.metadata.*;
 import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
+import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
+import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -150,11 +152,11 @@ public class DenormalizationResolver implements ContextRewriter {
             refCols.add(refer);
             // Add to optional tables
             if (refer.col.isChainedColumn()) {
-              cubeql.addOptionalDimTable(refer.col.getChainName(), refer.col.getRefColumn(), table, false);
+              cubeql.addOptionalDimTable(refer.col.getChainName(), table, false, refer.col.getRefColumn());
 
             } else {
               for (TableReference reference : refer.col.getReferences()) {
-                cubeql.addOptionalDimTable(reference.getDestTable(), reference.getDestColumn(), table, false);
+                cubeql.addOptionalDimTable(reference.getDestTable(), table, false, reference.getDestColumn());
               }
             }
             return true;
@@ -317,6 +319,32 @@ public class DenormalizationResolver implements ContextRewriter {
     }
   }
 
+  private void addRefColsQueried(CubeQueryContext cubeql, TrackQueriedColumns tqc, DenormalizationContext denormCtx) {
+    for (Map.Entry<String, Set<String>> entry : tqc.getTblAliasToColumns().entrySet()) {
+      // skip default alias
+      if (entry.getKey() == CubeQueryContext.DEFAULT_TABLE) {
+        continue;
+      }
+      // skip join chain aliases
+      if (cubeql.getJoinchains().keySet().contains(entry.getKey().toLowerCase())) {
+        continue;
+      }
+      AbstractCubeTable tbl = cubeql.getCubeTableForAlias(entry.getKey());
+      Set<String> columns = entry.getValue();
+      for (String column : columns) {
+        CubeColumn col;
+        if (tbl instanceof CubeInterface) {
+          col = ((CubeInterface) tbl).getColumnByName(column);
+        } else {
+          col = ((Dimension) tbl).getColumnByName(column);
+        }
+        if (col instanceof ReferencedDimAtrribute) {
+          // considering all referenced dimensions to be denormalized columns
+          denormCtx.addReferencedCol(column, new ReferencedQueriedColumn((ReferencedDimAtrribute) col, tbl));
+        }
+      }
+    }
+  }
   /**
    * Find all de-normalized columns, if these columns are not directly available in candidate tables, query will be
    * replaced with the corresponding table reference
@@ -328,27 +356,13 @@ public class DenormalizationResolver implements ContextRewriter {
       // Adds all the reference dimensions as eligible for denorm fields
       denormCtx = new DenormalizationContext(cubeql);
       cubeql.setDeNormCtx(denormCtx);
-      for (Map.Entry<String, Set<String>> entry : cubeql.getTblAliasToColumns().entrySet()) {
-        // skip default alias
-        if (entry.getKey() == CubeQueryContext.DEFAULT_TABLE) {
-          continue;
-        }
-        // skip join chain aliases
-        if (cubeql.getJoinchains().keySet().contains(entry.getKey().toLowerCase())) {
-          continue;
-        }
-        AbstractCubeTable tbl = cubeql.getCubeTableForAlias(entry.getKey());
-        Set<String> columns = entry.getValue();
-        for (String column : columns) {
-          CubeColumn col;
-          if (tbl instanceof CubeInterface) {
-            col = ((CubeInterface) tbl).getColumnByName(column);
-          } else {
-            col = ((Dimension) tbl).getColumnByName(column);
-          }
-          if (col instanceof ReferencedDimAtrribute) {
-            // considering all referenced dimensions to be denormalized columns
-            denormCtx.addReferencedCol(column, new ReferencedQueriedColumn((ReferencedDimAtrribute) col, tbl));
+      // add ref columns in cube
+      addRefColsQueried(cubeql, cubeql, denormCtx);
+      // add ref columns from expressions
+      for (Set<ExpressionContext> ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) {
+        for (ExpressionContext ec : ecSet) {
+          for (ExprSpecContext esc : ec.getAllExprs()) {
+            addRefColsQueried(cubeql, esc, denormCtx);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/82553db3/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 5355049..f2a7039 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -19,129 +19,692 @@
 
 package org.apache.lens.cube.parse;
 
-import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.DOT;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
+
+import java.util.*;
 
 import org.apache.lens.cube.metadata.AbstractBaseTable;
-import org.apache.lens.cube.metadata.DerivedCube;
+import org.apache.lens.cube.metadata.AbstractCubeTable;
+import org.apache.lens.cube.metadata.CubeColumn;
+import org.apache.lens.cube.metadata.CubeInterface;
 import org.apache.lens.cube.metadata.Dimension;
 import org.apache.lens.cube.metadata.ExprColumn;
+import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
 import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor;
 import org.apache.lens.cube.parse.HQLParser.TreeNode;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.HiveParser;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 import org.antlr.runtime.CommonToken;
 
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+
 /**
  * Replaces expression with its AST in all query ASTs
  */
+@Slf4j
 class ExpressionResolver implements ContextRewriter {
 
   public ExpressionResolver(Configuration conf) {
   }
 
-  @Override
-  public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
-    resolveClause(cubeql, cubeql.getSelectAST());
-    resolveClause(cubeql, cubeql.getWhereAST());
-    resolveClause(cubeql, cubeql.getJoinTree());
-    resolveClause(cubeql, cubeql.getGroupByAST());
-    resolveClause(cubeql, cubeql.getHavingAST());
-    resolveClause(cubeql, cubeql.getOrderByAST());
-
-    AggregateResolver.updateAggregates(cubeql.getSelectAST(), cubeql);
-    AggregateResolver.updateAggregates(cubeql.getHavingAST(), cubeql);
-  }
+  static class ExpressionContext {
+    @Getter
+    private final ExprColumn exprCol;
+    @Getter
+    private final AbstractBaseTable srcTable;
+    @Getter
+    private final String srcAlias;
+    @Getter
+    private Set<ExprSpecContext> allExprs = new LinkedHashSet<ExprSpecContext>();
+    private Set<CandidateTable> directlyAvailableIn = new HashSet<CandidateTable>();
+    private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions =
+      new HashMap<CandidateTable, Set<ExprSpecContext>>();
+    @Getter
+    private boolean hasMeasures = false;
 
-  private void resolveClause(final CubeQueryContext cubeql, ASTNode clause) throws SemanticException {
-    if (clause == null) {
-      return;
+    ExpressionContext(CubeQueryContext cubeql, ExprColumn exprCol, AbstractBaseTable srcTable, String srcAlias)
+      throws SemanticException {
+      this.srcTable = srcTable;
+      this.exprCol = exprCol;
+      this.srcAlias = srcAlias;
+      for (ExprSpec es : exprCol.getExpressionSpecs()) {
+        allExprs.add(new ExprSpecContext(es, cubeql));
+      }
+      resolveColumnsAndAlias(cubeql);
+    }
+    private void resolveColumnsAndAlias(CubeQueryContext cubeql) throws SemanticException {
+      for (ExprSpecContext esc : allExprs) {
+        esc.resolveColumns(cubeql);
+        esc.replaceAliasInAST(cubeql, cubeql.getColToTableAlias());
+      }
+      for (ExprSpecContext esc : allExprs) {
+        for (String table : esc.getTblAliasToColumns().keySet()) {
+          try {
+            if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
+              cubeql.addOptionalDimTable(table, null,
+                false, esc.getTblAliasToColumns().get(table).toArray(new String[0]));
+              esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
+            }
+          } catch (HiveException e) {
+            throw new SemanticException(e);
+          }
+        }
+      }
+      resolveColumnsAndReplaceAlias(cubeql, allExprs, cubeql.getColToTableAlias());
     }
 
-    // Traverse the tree and resolve expression columns
-    HQLParser.bft(clause, new ASTNodeVisitor() {
-      @Override
-      public void visit(TreeNode visited) throws SemanticException {
-        ASTNode node = visited.getNode();
-        int childcount = node.getChildCount();
-        for (int i = 0; i < childcount; i++) {
-          ASTNode current = (ASTNode) node.getChild(i);
-          if (current.getToken().getType() == TOK_TABLE_OR_COL && (node != null && node.getToken().getType() != DOT)) {
-            // Take child ident.totext
-            ASTNode ident = (ASTNode) current.getChild(0);
-            String column = ident.getText().toLowerCase();
-            ASTNode childExpr = getExprAST(cubeql, column);
-            if (childExpr != null) {
-              node.setChild(i, replaceAlias(childExpr, cubeql));
+    private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs,
+      Map<String, String> colToTableAlias) throws SemanticException {
+      Set<ExprSpecContext> nestedExpressions = new LinkedHashSet<ExprSpecContext>();
+      for (ExprSpecContext esc : exprs) {
+        for (Map.Entry<String, Set<String>> entry : esc.tblAliasToColumns.entrySet()) {
+          if (entry.getKey().equals(CubeQueryContext.DEFAULT_TABLE)) {
+            continue;
+          }
+          AbstractBaseTable baseTable = (AbstractBaseTable)cubeql.getCubeTableForAlias(entry.getKey());
+         // if (baseTable == null) {
+         //   continue;
+         // }
+          Set<String> exprCols = new HashSet<String>();
+          for (String col : entry.getValue()) {
+            // col is an expression
+            if (baseTable.getExpressionNames().contains(col)) {
+              exprCols.add(col);
             }
-          } else if (current.getToken().getType() == DOT) {
-            // This is for the case where column name is prefixed by table name
-            // or table alias
-            // For example 'select fact.id, dim2.id ...'
-            // Right child is the column name, left child.ident is table name
-            ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
-            ASTNode colIdent = (ASTNode) current.getChild(1);
+          }
+          // get all combinations of expression replaced with inner exprs AST.
+          addAllNestedExpressions(cubeql, esc, baseTable, nestedExpressions, exprCols);
+        }
+      }
+      for (ExprSpecContext esc : nestedExpressions) {
+        esc.resolveColumns(cubeql);
+        esc.replaceAliasInAST(cubeql, colToTableAlias);
+        for (String table : esc.getTblAliasToColumns().keySet()) {
+          try {
+            if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
+              cubeql.addOptionalDimTable(table, null, true,
+                esc.getTblAliasToColumns().get(table).toArray(new String[0]));
+              esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
+            }
+          } catch (HiveException e) {
+            throw new SemanticException(e);
+          }
+        }
+      }
+      exprs.addAll(nestedExpressions);
+    }
 
-            String column = colIdent.getText().toLowerCase();
+    private void addAllNestedExpressions(CubeQueryContext cubeql, ExprSpecContext baseEsc, AbstractBaseTable baseTable,
+      Set<ExprSpecContext> nestedExpressions, Set<String> exprCols) throws SemanticException {
+      for (String col : exprCols) {
+        Set<ExprSpecContext> replacedExpressions = new LinkedHashSet<ExprSpecContext>();
+        for (ExprSpec es : baseTable.getExpressionByName(col).getExpressionSpecs()) {
+          ASTNode finalAST = HQLParser.copyAST(baseEsc.getFinalAST());
+          replaceColumnInAST(finalAST, col, es.getASTNode());
+          ExprSpecContext replacedESC = new ExprSpecContext(baseEsc, es, finalAST, cubeql);
+          nestedExpressions.add(replacedESC);
+          replacedExpressions.add(replacedESC);
+        }
+        Set<String> remaining = new LinkedHashSet<String>(exprCols);
+        remaining.remove(col);
+        for (ExprSpecContext replacedESC : replacedExpressions) {
+          addAllNestedExpressions(cubeql, replacedESC, baseTable, nestedExpressions, remaining);
+        }
+      }
+    }
 
-            ASTNode childExpr = getExprAST(cubeql, tabident.getText().toLowerCase(), column);
-            if (childExpr != null) {
-              node.setChild(i, replaceAlias(childExpr, cubeql));
+    void addDirectlyAvailable(CandidateTable cTable) {
+      directlyAvailableIn.add(cTable);
+    }
+    void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContext esc) throws SemanticException {
+      Set<ExprSpecContext> evalSet = evaluableExpressions.get(cTable);
+      if (evalSet == null) {
+        evalSet = new LinkedHashSet<ExprSpecContext>();
+        evaluableExpressions.put(cTable, evalSet);
+        // add optional dimensions involved in expressions
+        for (String table : esc.getTblAliasToColumns().keySet()) {
+          try {
+            if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
+              cubeql.addOptionalDimTable(table, cTable,
+                false, esc.getTblAliasToColumns().get(table).toArray(new String[0]));
+              esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
             }
+          } catch (HiveException e) {
+            throw new SemanticException(e);
           }
         }
       }
-
-    });
+      evalSet.add(esc);
+    }
+    Set<ASTNode> getAllASTNodes() {
+      Set<ASTNode> allAST = new HashSet<ASTNode>();
+      for (ExprSpecContext esc : allExprs) {
+        allAST.add(esc.finalAST);
+      }
+      return allAST;
+    }
   }
 
-  private ASTNode getExprAST(final CubeQueryContext cubeql, String table, String column) throws SemanticException {
-    if (cubeql.getQueriedTable(table) == null) {
-      if (!cubeql.addQueriedTable(table)) {
-        throw new SemanticException(ErrorMsg.NEITHER_CUBE_NOR_DIMENSION);
+  static class ExprSpecContext implements TrackQueriedColumns {
+    @Getter
+    private Set<ExprSpec> exprSpecs = new LinkedHashSet<ExprSpec>();
+    @Getter
+    private ASTNode finalAST;
+    private Set<Dimension> exprDims = new HashSet<Dimension>();
+    // for each expression store alias to columns queried
+    @Getter
+    private Map<String, Set<String>> tblAliasToColumns = new HashMap<String, Set<String>>();
+
+    ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws SemanticException {
+      // replaces table names in expression with aliases in the query
+      finalAST = replaceAlias(exprSpec.getASTNode(), cubeql);
+      exprSpecs.add(exprSpec);
+    }
+    public ExprSpecContext(ExprSpecContext nested, ExprSpec current, ASTNode node,
+      CubeQueryContext cubeql) throws SemanticException {
+      exprSpecs.addAll(nested.exprSpecs);
+      exprSpecs.add(current);
+      finalAST = replaceAlias(node, cubeql);
+    }
+    public void replaceAliasInAST(CubeQueryContext cubeql, Map<String, String> colToTableAlias)
+      throws SemanticException {
+      AliasReplacer.extractTabAliasForCol(colToTableAlias, cubeql, this);
+      AliasReplacer.replaceAliases(finalAST, 0, colToTableAlias);
+    }
+    public void addColumnsQueried(String alias, String column) {
+      Set<String> cols = tblAliasToColumns.get(alias.toLowerCase());
+      if (cols == null) {
+        cols = new HashSet<String>();
+        tblAliasToColumns.put(alias.toLowerCase(), cols);
       }
+      cols.add(column);
     }
-    if (!(cubeql.getQueriedTable(table) instanceof AbstractBaseTable)) {
+
+    void resolveColumns(CubeQueryContext cubeql) throws SemanticException {
+      // finds all columns and table aliases in the expression
+      ColumnResolver.getColsForTree(cubeql, finalAST, this);
+    }
+
+    Date getStartTime() {
+      Set<Date> startTimes = new HashSet<Date>();
+      for (ExprSpec es : exprSpecs) {
+        if (es.getStartTime() != null) {
+          startTimes.add(es.getStartTime());
+        }
+      }
+      if (!startTimes.isEmpty()) {
+        return Collections.max(startTimes);
+      }
       return null;
     }
-    if (((AbstractBaseTable) cubeql.getQueriedTable(table)).getExpressionByName(column) == null) {
+
+    Date getEndTime() {
+      Set<Date> endTimes = new HashSet<Date>();
+      for (ExprSpec es : exprSpecs) {
+        if (es.getEndTime() != null) {
+          endTimes.add(es.getEndTime());
+        }
+      }
+      if (!endTimes.isEmpty()) {
+        return Collections.min(endTimes);
+      }
       return null;
     }
-    return ((AbstractBaseTable) cubeql.getQueriedTable(table)).getExpressionByName(column).getAst();
+
+    public boolean isValidInTimeRange(final TimeRange range) {
+      return isValidFrom(range.getFromDate()) && isValidTill(range.getToDate());
+    }
+
+    public boolean isValidFrom(@NonNull final Date date) {
+      return (getStartTime() == null) ? true : date.equals(getStartTime()) || date.after(getStartTime());
+    }
+
+    public boolean isValidTill(@NonNull final Date date) {
+      return (getEndTime() == null) ? true : date.equals(getEndTime()) || date.before(getEndTime());
+    }
+
+    public String toString() {
+      return HQLParser.getString(finalAST);
+    }
+  }
+
+  @AllArgsConstructor
+  @ToString
+  private static class PickedExpression {
+    private String srcAlias;
+    private ExprSpecContext pickedCtx;
   }
 
-  private ASTNode getExprAST(final CubeQueryContext cubeql, final String column) throws SemanticException {
-    ExprColumn expr = null;
-    AbstractBaseTable table = null;
-    if (cubeql.getCube() != null && !(cubeql.getCube() instanceof DerivedCube)) {
-      // no expression resolver for derived cubes
-      if (cubeql.getCube().getExpressionNames().contains(column.toLowerCase())) {
-        expr = cubeql.getCube().getExpressionByName(column);
-        table = (AbstractBaseTable) cubeql.getCube();
+  static class ExpressionResolverContext {
+    @Getter
+    private Map<String, Set<ExpressionContext>> allExprsQueried = new HashMap<String, Set<ExpressionContext>>();
+    private Map<String, Set<PickedExpression>> pickedExpressions = new HashMap<String, Set<PickedExpression>>();
+    private final CubeQueryContext cubeql;
+
+    ExpressionResolverContext(CubeQueryContext cubeql) {
+      this.cubeql = cubeql;
+    }
+    void addExpressionQueried(ExpressionContext expr) {
+      String exprCol = expr.getExprCol().getName().toLowerCase();
+      Set<ExpressionContext> ecSet = allExprsQueried.get(exprCol);
+      if (ecSet == null) {
+        ecSet = new LinkedHashSet<ExpressionContext>();
+        allExprsQueried.put(exprCol, ecSet);
+      }
+      ecSet.add(expr);
+    }
+
+    boolean isQueriedExpression(String column) {
+      return allExprsQueried.containsKey(column);
+    }
+
+    boolean hasAggregates() {
+      for (Set<ExpressionContext> ecSet : allExprsQueried.values()) {
+        for (ExpressionContext ec : ecSet) {
+          for (ExprSpecContext esc : ec.allExprs) {
+            if (HQLParser.isAggregateAST(esc.finalAST)) {
+              return true;
+            }
+          }
+        }
+      }
+      return false;
+    }
+
+    ExpressionContext getExpressionContext(String expr, String alias) {
+      for (ExpressionContext ec : allExprsQueried.get(expr)) {
+        if (ec.getSrcAlias().equals(alias)) {
+          return ec;
+        }
+      }
+      throw new IllegalArgumentException("no expression available for " + expr + " alias:" + alias);
+    }
+
+    public boolean hasMeasures(String expr, CubeInterface cube) {
+      String alias = cubeql.getAliasForTableName(cube.getName());
+      ExpressionContext ec = getExpressionContext(expr, alias);
+      boolean hasMeasures = false;
+      for (ExprSpecContext esc : ec.allExprs) {
+        if (esc.getTblAliasToColumns().get(alias) != null) {
+          for (String cubeCol : esc.getTblAliasToColumns().get(alias)) {
+            if (cube.getMeasureByName(cubeCol) != null) {
+              hasMeasures = true;
+              break;
+            }
+          }
+        }
       }
+      ec.hasMeasures = hasMeasures;
+      return hasMeasures;
     }
-    if (cubeql.getDimensions() != null) {
-      for (Dimension dim : cubeql.getDimensions()) {
-        if (dim.getExpressionNames().contains(column.toLowerCase())) {
-          if (expr != null) {
-            throw new SemanticException(ErrorMsg.AMBIGOUS_DIM_COLUMN, table.getName(), dim.getName());
+
+    //updates all expression specs which are evaluable
+    public void updateEvaluables(String expr, CandidateTable cTable)
+      throws SemanticException {
+      String alias = cubeql.getAliasForTableName(cTable.getBaseTable().getName());
+      ExpressionContext ec = getExpressionContext(expr, alias);
+      if (cTable.getColumns().contains(expr)) {
+        // expression is directly materialized in candidate table
+        ec.addDirectlyAvailable(cTable);
+      }
+      for (ExprSpecContext esc : ec.allExprs) {
+        if (esc.getTblAliasToColumns().get(alias) == null) {
+          ec.addEvaluable(cubeql, cTable, esc);
+        } else {
+          Set<String> columns = esc.getTblAliasToColumns().get(alias);
+          boolean isEvaluable = true;
+          for (String col : columns) {
+            if (!cTable.getColumns().contains(col.toLowerCase())) {
+              if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) {
+                // check if it is available as reference, if not expression is not evaluable
+                isEvaluable = false;
+                break;
+              }
+            }
+          }
+          if (isEvaluable) {
+            ec.addEvaluable(cubeql, cTable, esc);
           }
-          expr = dim.getExpressionByName(column);
-          table = dim;
         }
       }
     }
-    if (expr == null) {
+
+    // checks if expr is evaluable
+    public boolean isEvaluable(String expr, CandidateTable cTable) {
+      ExpressionContext ec = getExpressionContext(expr, cubeql.getAliasForTableName(cTable.getBaseTable().getName()));
+      if (ec.directlyAvailableIn.contains(cTable)) {
+        return true;
+      }
+      if (ec.evaluableExpressions.get(cTable) == null) {
+        return false;
+      }
+      return !ec.evaluableExpressions.get(cTable).isEmpty();
+    }
+
+    /**
+     *
+     * @param exprs
+     * @return
+     */
+    public boolean allNotEvaluable(Set<String> exprs, CandidateTable cTable) {
+      for (String expr : exprs) {
+        if (!isEvaluable(expr, cTable)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    public Collection<String> coveringExpressions(Set<String> exprs, CandidateTable cTable) {
+      Set<String> coveringSet = new HashSet<String>();
+      for (String expr : exprs) {
+        if (isEvaluable(expr, cTable)) {
+          coveringSet.add(expr);
+        }
+      }
+      return coveringSet;
+    }
+
+    /**
+     * Returns true if all passed expressions are evaluable
+     *
+     * @param cTable
+     * @param exprs
+     * @return
+     */
+    public boolean allEvaluable(CandidateTable cTable, Set<String> exprs) {
+      for (String expr : exprs) {
+        if (!isEvaluable(expr, cTable)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public Set<Dimension> rewriteExprCtx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery,
+      boolean replaceFact) throws SemanticException {
+      Set<Dimension> exprDims = new HashSet<Dimension>();
+      if (!allExprsQueried.isEmpty()) {
+        // pick expressions for fact
+        if (cfact != null) {
+          pickExpressionsForTable(cfact);
+        }
+        // pick expressions for dimensions
+        if (dimsToQuery != null && !dimsToQuery.isEmpty()) {
+          for (CandidateDim cdim : dimsToQuery.values()) {
+            pickExpressionsForTable(cdim);
+          }
+        }
+        // Replace picked expressions in all the base trees
+        replacePickedExpressions(cfact, replaceFact);
+        for (Set<PickedExpression> peSet : pickedExpressions.values()) {
+          for (PickedExpression pe : peSet) {
+            exprDims.addAll(pe.pickedCtx.exprDims);
+          }
+        }
+      }
+      pickedExpressions.clear();
+      return exprDims;
+    }
+
+    private void replacePickedExpressions(CandidateFact cfact, boolean replaceFact)
+      throws SemanticException {
+      if (replaceFact) {
+        replaceAST(cubeql, cfact.getSelectAST());
+        replaceAST(cubeql, cfact.getWhereAST());
+        replaceAST(cubeql, cfact.getJoinTree());
+        replaceAST(cubeql, cfact.getGroupByAST());
+        replaceAST(cubeql, cfact.getHavingAST());
+      } else {
+        replaceAST(cubeql, cubeql.getSelectAST());
+        replaceAST(cubeql, cubeql.getWhereAST());
+        replaceAST(cubeql, cubeql.getJoinTree());
+        replaceAST(cubeql, cubeql.getGroupByAST());
+        replaceAST(cubeql, cubeql.getHavingAST());
+      }
+      replaceAST(cubeql, cubeql.getOrderByAST());
+    }
+
+    private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws SemanticException {
+      if (node == null) {
+        return;
+      }
+      // Traverse the tree and resolve expression columns
+      HQLParser.bft(node, new ASTNodeVisitor() {
+        @Override
+        public void visit(TreeNode visited) throws SemanticException {
+          ASTNode node = visited.getNode();
+          int childcount = node.getChildCount();
+          for (int i = 0; i < childcount; i++) {
+            ASTNode current = (ASTNode) node.getChild(i);
+            if (current.getToken().getType() == DOT) {
+              // This is for the case where column name is prefixed by table name
+              // or table alias
+              // For example 'select fact.id, dim2.id ...'
+              // Right child is the column name, left child.ident is table name
+              ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
+              ASTNode colIdent = (ASTNode) current.getChild(1);
+              String column = colIdent.getText().toLowerCase();
+
+              if (pickedExpressions.containsKey(column)) {
+                PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase());
+                if (expr != null) {
+                  node.setChild(i, replaceAlias(expr.pickedCtx.finalAST, cubeql));
+                }
+              }
+            }
+          }
+        }
+      });
+    }
+
+    private PickedExpression getPickedExpression(String column, String alias) {
+      Set<PickedExpression> peSet = pickedExpressions.get(column);
+      if (peSet != null && !peSet.isEmpty()) {
+        for (PickedExpression picked : peSet) {
+          if (picked.srcAlias.equals(alias)) {
+            return picked;
+          }
+        }
+      }
       return null;
     }
-    return expr.getAst();
+
+    private void pickExpressionsForTable(CandidateTable cTable) {
+      for (Map.Entry<String, Set<ExpressionContext>> ecEntry : allExprsQueried.entrySet()) {
+        Set<ExpressionContext> ecSet = ecEntry.getValue();
+        for (ExpressionContext ec : ecSet) {
+          if (ec.getSrcTable().getName().equals(cTable.getBaseTable().getName())) {
+            if (!ec.directlyAvailableIn.contains(cTable)) {
+              if (ec.evaluableExpressions.get(cTable) != null && !ec.evaluableExpressions.get(cTable).isEmpty()) {
+                // pick first evaluable expression
+                Set<PickedExpression> peSet = pickedExpressions.get(ecEntry.getKey());
+                if (peSet == null) {
+                  peSet = new HashSet<PickedExpression>();
+                  pickedExpressions.put(ecEntry.getKey(), peSet);
+                }
+                peSet.add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next()));
+              }
+            }
+          }
+        }
+      }
+    }
+
+    void pruneExpressions() {
+      for (Set<ExpressionContext> ecSet : allExprsQueried.values()) {
+        for (ExpressionContext ec : ecSet) {
+          Set<ExprSpecContext> removedEsc = new HashSet<ExprSpecContext>();
+          for(Iterator<ExprSpecContext> iterator = ec.getAllExprs().iterator(); iterator.hasNext();) {
+            ExprSpecContext esc = iterator.next();
+            boolean removed = false;
+            // Go over expression dims and remove expression involving dimensions for which candidate tables are
+            // not there
+            for (Dimension exprDim : esc.exprDims) {
+              if (cubeql.getCandidateDims().get(exprDim) == null || cubeql.getCandidateDims().get(exprDim).isEmpty()) {
+                log.info("Removing expression {} as {} it does not have any candidate tables", esc, exprDim);
+                iterator.remove();
+                removedEsc.add(esc);
+                removed = true;
+                break;
+              }
+            }
+            if (removed) {
+              continue;
+            }
+            //remove expressions which are not valid in the timerange queried
+            // If an expression is defined as
+            // ex = a + b // from t1 to t2;
+            // ex = c + d // from t2 to t3
+            // With range queried, invalid expressions will be removed
+            // If range is including more than one expression, queries can be unioned as an improvement at later time.
+            // But for now, they are not eligible expressions
+            for (TimeRange range : cubeql.getTimeRanges()) {
+              if (!esc.isValidInTimeRange(range)) {
+                log.info("Removing expression " + esc + " as it is not valid in timerange queried");
+                iterator.remove();
+                removedEsc.add(esc);
+                removed = true;
+                break;
+              }
+            }
+            if (removed) {
+              continue;
+            }
+            // Go over expressions and remove expression containing unavailable columns in timerange
+            // In the example above,
+            // if ex = a +b ; and a is not available in timerange queried, it will be removed.
+            for (TimeRange range : cubeql.getTimeRanges()) {
+              boolean toRemove = false;
+              for (Map.Entry<String, Set<String>> entry : esc.getTblAliasToColumns().entrySet()) {
+                if (CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(entry.getKey())) {
+                  continue;
+                }
+                AbstractBaseTable baseTable = (AbstractBaseTable) cubeql.getCubeTableForAlias(entry.getKey());
+                for (String col : entry.getValue()) {
+                  if (!baseTable.getColumnByName(col).isColumnAvailableInTimeRange(range)) {
+                    toRemove = true;
+                    break;
+                  }
+                }
+                if (toRemove) {
+                  break;
+                }
+              }
+              if (toRemove) {
+                log.info("Removing expression " + esc + " as its columns are unavailable in timerange queried");
+                iterator.remove();
+                removedEsc.add(esc);
+                removed = true;
+                break;
+              }
+            }
+          }
+          for (Set<ExprSpecContext> evalSet : ec.evaluableExpressions.values()) {
+            evalSet.removeAll(removedEsc);
+          }
+        }
+      }
+    }
+  }
+
+  @Override
+  public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
+    ExpressionResolverContext exprCtx = cubeql.getExprCtx();
+    if (exprCtx == null) {
+      exprCtx = new ExpressionResolverContext(cubeql);
+      cubeql.setExprCtx(exprCtx);
+      for (Map.Entry<String, Set<String>> entry : cubeql.getTblAliasToColumns().entrySet()) {
+        String alias = entry.getKey();
+        // skip default alias
+        if (alias == CubeQueryContext.DEFAULT_TABLE) {
+          continue;
+        }
+        AbstractCubeTable tbl = cubeql.getCubeTableForAlias(alias);
+        Set<String> columns = entry.getValue();
+        for (String column : columns) {
+          CubeColumn col;
+          if (tbl instanceof CubeInterface) {
+            col = ((CubeInterface) tbl).getColumnByName(column);
+          } else {
+            col = ((Dimension) tbl).getColumnByName(column);
+          }
+          if (col instanceof ExprColumn) {
+            exprCtx.addExpressionQueried(new ExpressionContext(cubeql, (ExprColumn)col, (AbstractBaseTable)tbl, alias));
+          }
+        }
+      }
+      Set<String> exprsWithMeasures = new HashSet<String>();
+      for (String expr : cubeql.getQueriedExprs()) {
+        if (cubeql.getExprCtx().hasMeasures(expr, cubeql.getCube())) {
+          // expression has measures
+          exprsWithMeasures.add(expr);
+        }
+      }
+      cubeql.addQueriedExprsWithMeasures(exprsWithMeasures);
+
+    } else {
+      // prune invalid expressions
+      cubeql.getExprCtx().pruneExpressions();
+      // prune candidate facts without any valid expressions
+      if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) {
+        for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) {
+          CandidateFact cfact = i.next();
+          for (Map.Entry<String, Set<ExpressionContext>> ecEntry : exprCtx.allExprsQueried.entrySet()) {
+            Set<ExpressionContext> ecSet = ecEntry.getValue();
+            for (ExpressionContext ec : ecSet) {
+              if (ec.getSrcTable().getName().equals(cfact.getBaseTable().getName())) {
+                if (!ec.directlyAvailableIn.contains(cfact)
+                  && (ec.evaluableExpressions.get(cfact) == null
+                  || ec.evaluableExpressions.get(cfact).isEmpty())) {
+                  log.info("Not considering fact table:{} as {} is not evaluable", cfact, ec.exprCol.getName());
+                  cubeql.addFactPruningMsgs(cfact.fact,
+                    CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
+                  i.remove();
+                }
+              }
+            }
+          }
+        }
+      }
+      // prune candidate dims without any valid expressions
+      if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) {
+        for (Dimension dim : cubeql.getDimensions()) {
+          for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) {
+            CandidateDim cdim = i.next();
+            for (Map.Entry<String, Set<ExpressionContext>> ecEntry : exprCtx.allExprsQueried.entrySet()) {
+              Set<ExpressionContext> ecSet = ecEntry.getValue();
+              for (ExpressionContext ec : ecSet) {
+                if (ec.getSrcTable().getName().equals(cdim.getBaseTable().getName())) {
+                  if (!ec.directlyAvailableIn.contains(cdim)
+                    && (ec.evaluableExpressions.get(cdim) == null
+                    || ec.evaluableExpressions.get(cdim).isEmpty())) {
+                    log.info("Not considering dim table:{} as {} is not evaluable", cdim, ec.exprCol.getName());
+                    cubeql.addDimPruningMsgs(dim, cdim.dimtable,
+                      CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
+                    i.remove();
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
   }
 
-  private ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws SemanticException {
+  private static ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws SemanticException {
     ASTNode finalAST = HQLParser.copyAST(expr);
     HQLParser.bft(finalAST, new ASTNodeVisitor() {
       @Override
@@ -166,4 +729,43 @@ class ExpressionResolver implements ContextRewriter {
     });
     return finalAST;
   }
+
+  private static void replaceColumnInAST(ASTNode expr, final String toReplace, final ASTNode columnAST)
+    throws SemanticException {
+    if (expr == null) {
+      return;
+    }
+    // Traverse the tree and resolve expression columns
+    HQLParser.bft(expr, new ASTNodeVisitor() {
+      @Override
+      public void visit(TreeNode visited) throws SemanticException {
+        ASTNode node = visited.getNode();
+        int childcount = node.getChildCount();
+        for (int i = 0; i < childcount; i++) {
+          ASTNode current = (ASTNode) node.getChild(i);
+          if (current.getToken().getType() == TOK_TABLE_OR_COL && (node != null && node.getToken().getType() != DOT)) {
+            // Take child ident.totext
+            ASTNode ident = (ASTNode) current.getChild(0);
+            String column = ident.getText().toLowerCase();
+            if (toReplace.equals(column)) {
+              node.setChild(i, HQLParser.copyAST(columnAST));
+            }
+          } else if (current.getToken().getType() == DOT) {
+            // This is for the case where column name is prefixed by table name
+            // or table alias
+            // For example 'select fact.id, dim2.id ...'
+            // Right child is the column name, left child.ident is table name
+            ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
+            ASTNode colIdent = (ASTNode) current.getChild(1);
+
+            String column = colIdent.getText().toLowerCase();
+
+            if (toReplace.equals(column)) {
+              node.setChild(i, HQLParser.copyAST(columnAST));
+            }
+          }
+        }
+      }
+    });
+  }
 }


Mime
View raw message