hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r1469787 - in /hive/branches/HIVE-4115/ql/src: java/org/apache/hadoop/hive/ql/cube/parse/ test/org/apache/hadoop/hive/ql/cube/processors/
Date Fri, 19 Apr 2013 11:39:46 GMT
Author: amareshwari
Date: Fri Apr 19 11:39:45 2013
New Revision: 1469787

URL: http://svn.apache.org/r1469787
Log:
Merge CubeQueryContextWithStorage with CubeQueryContext

Removed:
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContextWithStorage.java
Modified:
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
    hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java?rev=1469787&r1=1469786&r2=1469787&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AliasReplacer.java
Fri Apr 19 11:39:45 2013
@@ -1,5 +1,8 @@
 package org.apache.hadoop.hive.ql.cube.parse;
 
+import java.util.List;
+import java.util.Map;
+
 import org.apache.hadoop.conf.Configuration;
 
 public class AliasReplacer implements ContextRewriter {
@@ -22,5 +25,7 @@ public class AliasReplacer implements Co
     // 2: (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key))
     // (TOK_SELEXPR (TOK_FUNCTION count (. (TOK_TABLE_OR_COL src) value))))
     // 3: (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) srckey))))
+
+    Map<String, List<String>> tblToColumns = cubeql.getTblToColumns();
   }
 }

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1469787&r1=1469786&r2=1469787&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
Fri Apr 19 11:39:45 2013
@@ -1,24 +1,30 @@
 package org.apache.hadoop.hive.ql.cube.parse;
 
+import static org.apache.hadoop.hive.ql.parse.HiveParser.DOT;
 import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
 import static org.apache.hadoop.hive.ql.parse.HiveParser.KW_AND;
 import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FUNCTION;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
 
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.cube.metadata.AbstractCubeTable;
 import org.apache.hadoop.hive.ql.cube.metadata.Cube;
 import org.apache.hadoop.hive.ql.cube.metadata.CubeDimensionTable;
 import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
 import org.apache.hadoop.hive.ql.cube.metadata.CubeMetastoreClient;
 import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
+import org.apache.hadoop.hive.ql.cube.parse.HQLParser.ASTNodeVisitor;
+import org.apache.hadoop.hive.ql.cube.parse.HQLParser.TreeNode;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.JoinCond;
@@ -49,11 +55,37 @@ public class CubeQueryContext {
   protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap
=
       new HashMap<CubeFactTable, Map<UpdatePeriod, List<String>>>();
 
+  private List<String> supportedStorages;
+  private boolean allStoragesSupported;
+  private final Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factStorageMap
=
+      new HashMap<CubeFactTable, Map<UpdatePeriod,List<String>>>();
+  private final Map<CubeDimensionTable, List<String>> dimStorageMap =
+      new HashMap<CubeDimensionTable, List<String>>();
+  private final Map<String, String> storageTableToWhereClause =
+      new HashMap<String, String>();
+  private final Map<AbstractCubeTable, String> storageTableToQuery =
+      new HashMap<AbstractCubeTable, String>();
+  private ASTNode fromTree;
+  private ASTNode whereTree;
+  private ASTNode havingTree;
+  private ASTNode orderByTree;
+  private ASTNode selectTree;
+  private ASTNode groupbyTree;
+  private ASTNode limitTree;
+  private ASTNode joinTree;
+
   public CubeQueryContext(ASTNode ast, QB qb, HiveConf conf)
       throws SemanticException {
     this.ast = ast;
     this.qb = qb;
     this.conf = conf;
+    this.clauseName = getClause();
+    this.whereTree = qb.getParseInfo().getWhrForClause(clauseName);
+    this.havingTree = qb.getParseInfo().getHavingForClause(clauseName);
+    this.orderByTree = qb.getParseInfo().getOrderByForClause(clauseName);
+    this.groupbyTree = qb.getParseInfo().getGroupByForClause(clauseName);
+    this.selectTree = qb.getParseInfo().getSelForClause(clauseName);
+    this.joinTree = qb.getParseInfo().getJoinExpr();
     extractMetaTables();
     extractTimeRange();
   }
@@ -171,13 +203,11 @@ public class CubeQueryContext {
     System.out.println("timeTo:" + timeTo);
   }
 
-/*  private void extractColumns() {
+  private void extractColumns() {
     //columnAliases = new ArrayList<String>();
-    tblToColumns = new LinkedHashMap<String, List<String>>();
-    ASTNode selectTree = qb.getParseInfo().getSelForClause(clauseName);
 
     // Check if its 'select *  from...'
-    if (selectTree.getChildCount() == 1) {
+   /* if (selectTree.getChildCount() == 1) {
       ASTNode star = HQLParser.findNodeByPath(selectTree, TOK_SELEXPR,
           TOK_ALLCOLREF);
       if (star == null) {
@@ -191,7 +221,7 @@ public class CubeQueryContext {
           selectAllColumns = true;
         }
       }
-    }
+    } */
 
     // Traverse select, where, groupby, having and orderby trees to get column
     // names
@@ -245,19 +275,12 @@ public class CubeQueryContext {
             if (!colList.contains(column)) {
               colList.add(column);
             }
-          } else if (node.getToken().getType() == TOK_SELEXPR) {
-            // Extract column aliases for the result set, only applies to select
-            // trees
-            ASTNode alias = HQLParser.findNodeByPath(node, Identifier);
-            if (alias != null) {
-              columnAliases.add(alias.getText());
-            }
           }
         }
       });
     }
   }
-*/
+
   public String getFromDateRaw() {
     return fromDateRaw;
   }
@@ -455,27 +478,27 @@ public class CubeQueryContext {
   }
 
   public ASTNode getSelectTree() {
-    return qb.getParseInfo().getSelForClause(getClause());
+    return selectTree;
   }
 
   public ASTNode getWhereTree() {
-    return qb.getParseInfo().getWhrForClause(getClause());
+    return whereTree;
   }
 
   public ASTNode getGroupbyTree() {
-    return qb.getParseInfo().getGroupByForClause(getClause());
+    return groupbyTree;
   }
 
   public ASTNode getHavingTree() {
-    return qb.getParseInfo().getHavingForClause(getClause());
+    return havingTree;
   }
 
   public ASTNode getJoinTree() {
-    return qb.getParseInfo().getJoinExpr();
+    return joinTree;
   }
 
   public ASTNode getOrderbyTree() {
-    return qb.getParseInfo().getOrderByForClause(getClause());
+    return orderByTree;
   }
 
   public ASTNode getFromTree() {
@@ -485,9 +508,6 @@ public class CubeQueryContext {
   public Integer getLimitValue() {
     return qb.getParseInfo().getDestLimit(getClause());
   }
-  public String toHQL() throws SemanticException {
-    return null;
-  }
 
   public Map<CubeFactTable, Map<UpdatePeriod, List<String>>> getFactPartitionMap()
{
     return factPartitionMap;
@@ -497,4 +517,171 @@ public class CubeQueryContext {
       Map<UpdatePeriod, List<String>>> factPartitionMap) {
     this.factPartitionMap.putAll(factPartitionMap);
   }
+
+  public List<String> getSupportedStorages() {
+    return supportedStorages;
+  }
+
+  public void setSupportedStorages(List<String> supportedStorages) {
+    this.supportedStorages = supportedStorages;
+    this.allStoragesSupported = (supportedStorages == null);
+  }
+
+  private final String baseQueryFormat = "SELECT %s FROM %s";
+
+  String getQueryFormat() {
+    StringBuilder queryFormat = new StringBuilder();
+    queryFormat.append(baseQueryFormat);
+    if (getWhereTree() != null || hasPartitions()) {
+      queryFormat.append(" WHERE %s");
+    }
+    if (getGroupbyTree() != null) {
+      queryFormat.append(" GROUP BY %s");
+    }
+    if (getHavingTree() != null) {
+      queryFormat.append(" HAVING %s");
+    }
+    if (getOrderbyTree() != null) {
+      queryFormat.append(" ORDER BY %s");
+    }
+    if (getLimitValue() != null) {
+      queryFormat.append(" LIMIT %s");
+    }
+    return queryFormat.toString();
+  }
+
+  private Object[] getQueryTreeStrings(String factStorageTable) {
+    List<String> qstrs = new ArrayList<String>();
+    qstrs.add(HQLParser.getString(getSelectTree()));
+    String fromString = HQLParser.getString(getFromTree()).toLowerCase();
+    String whereString = getWhereTree(factStorageTable);
+    for (Map.Entry<AbstractCubeTable, String> entry :
+        storageTableToQuery.entrySet()) {
+      String src = entry.getKey().getName().toLowerCase();
+      System.out.println("From string:" + fromString + " src:" + src + " value:" + entry.getValue());
+      fromString = fromString.replaceAll(src, entry.getValue() + " " + src);
+    }
+    qstrs.add(fromString);
+    if (whereString != null) {
+      qstrs.add(whereString);
+    }
+    if (getGroupbyTree() != null) {
+      qstrs.add(HQLParser.getString(getGroupbyTree()));
+    }
+    if (getHavingTree() != null) {
+      qstrs.add(HQLParser.getString(getHavingTree()));
+    }
+    if (getOrderbyTree() != null) {
+      qstrs.add(HQLParser.getString(getOrderbyTree()));
+    }
+    if (getLimitValue() != null) {
+      qstrs.add(String.valueOf(getLimitValue()));
+    }
+    return qstrs.toArray(new String[0]);
+  }
+
+  private String toHQL(String tableName) {
+    String qfmt = getQueryFormat();
+    System.out.println("qfmt:" + qfmt);
+    return String.format(qfmt, getQueryTreeStrings(tableName));
+  }
+
+  public String getWhereTree(String factStorageTable) {
+    String originalWhereString = HQLParser.getString(getWhereTree());
+    String whereWithoutTimerange;
+    if (factStorageTable != null) {
+      whereWithoutTimerange = originalWhereString.substring(0,
+          originalWhereString.indexOf(CubeQueryContext.TIME_RANGE_FUNC));
+    } else {
+      whereWithoutTimerange = originalWhereString;
+    }
+    // add where clause for all dimensions
+    for (CubeDimensionTable dim : dimensions) {
+      String storageTable = dimStorageMap.get(dim).get(0);
+      storageTableToQuery.put(dim, storageTable);
+      whereWithoutTimerange += storageTableToWhereClause.get(storageTable);
+    }
+    if (factStorageTable != null) {
+      // add where clause for fact;
+      return whereWithoutTimerange + storageTableToWhereClause.get(
+          factStorageTable);
+    } else {
+      return whereWithoutTimerange;
+    }
+  }
+
+  public String toHQL() throws SemanticException {
+    CubeFactTable fact = null;
+    if (hasCubeInQuery()) {
+      if (candidateFactTables.size() > 0) {
+        fact = candidateFactTables.iterator().next();
+      }
+    }
+    if (fact == null && !hasDimensionInQuery()) {
+      throw new SemanticException("No valid fact table available");
+    }
+
+    if (fact != null) {
+      Map<UpdatePeriod, List<String>> storageTableMap = factStorageMap.get(fact);
+      Map<UpdatePeriod, List<String>> partColMap = factPartitionMap.get(fact);
+
+      StringBuilder query = new StringBuilder();
+      Iterator<UpdatePeriod> it = partColMap.keySet().iterator();
+      while (it.hasNext()) {
+        UpdatePeriod updatePeriod = it.next();
+        String storageTable = storageTableMap.get(updatePeriod).get(0);
+        storageTableToQuery.put(getCube(), storageTable);
+        query.append(toHQL(storageTable));
+        if (it.hasNext()) {
+          query.append(" UNION ");
+        }
+      }
+      return query.toString();
+    } else {
+      return toHQL(null);
+    }
+  }
+
+  public Map<CubeFactTable, Map<UpdatePeriod, List<String>>> getFactStorageMap()
+  {
+    return factStorageMap;
+  }
+
+  public void setFactStorageMap(Map<CubeFactTable,
+      Map<UpdatePeriod, List<String>>> factStorageMap) {
+    this.factStorageMap.putAll(factStorageMap);
+  }
+
+  public void setDimStorageMap(
+      Map<CubeDimensionTable, List<String>> dimStorageMap) {
+    this.dimStorageMap.putAll(dimStorageMap);
+  }
+
+  public Map<CubeDimensionTable, List<String>> getDimStorageMap() {
+    return this.dimStorageMap;
+  }
+
+  public Map<String, String> getStorageTableToWhereClause() {
+    return storageTableToWhereClause;
+  }
+
+  public void setStorageTableToWhereClause(Map<String, String> whereClauseMap) {
+    storageTableToWhereClause.putAll(whereClauseMap);
+  }
+
+  public boolean hasPartitions() {
+    return !storageTableToWhereClause.isEmpty();
+  }
+
+  public boolean isStorageSupported(String storage) {
+    if (!allStoragesSupported) {
+      return supportedStorages.contains(storage);
+    }
+    return true;
+  }
+
+  public Map<String, List<String>> getTblToColumns() {
+    return tblToColumns;
+  }
+
 }

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java?rev=1469787&r1=1469786&r2=1469787&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
Fri Apr 19 11:39:45 2013
@@ -44,21 +44,19 @@ public class CubeQueryRewriter {
 
   public CubeQueryContext rewritePhase1(ASTNode astnode)
       throws SemanticException, ParseException {
-    CubeQueryContext ctx;
       CubeSemanticAnalyzer analyzer =  new CubeSemanticAnalyzer(
           new HiveConf(conf, HiveConf.class));
       analyzer.analyzeInternal(astnode);
-      ctx = analyzer.getQueryContext();
+      CubeQueryContext ctx = analyzer.getQueryContext();
       rewrite(phase1Rewriters, ctx);
     return ctx;
   }
 
   public CubeQueryContext rewritePhase2(CubeQueryContext cubeql,
       List<String> storages) throws SemanticException {
-    CubeQueryContextWithStorage ctx = new CubeQueryContextWithStorage(
-        (CubeQueryContext)cubeql, storages);
-    rewrite(phase2Rewriters, ctx);
-    return ctx;
+    cubeql.setSupportedStorages(storages);
+    rewrite(phase2Rewriters, cubeql);
+    return cubeql;
   }
 
 

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java?rev=1469787&r1=1469786&r2=1469787&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
Fri Apr 19 11:39:45 2013
@@ -21,8 +21,6 @@ public class StorageTableResolver implem
   @Override
   public void rewriteContext(CubeQueryContext cubeql)
       throws SemanticException {
-    CubeQueryContextWithStorage cubeqlStorage =
-        (CubeQueryContextWithStorage) cubeql;
     Map<String, String> storageTableToWhereClause =
         new HashMap<String, String>();
 
@@ -42,7 +40,7 @@ public class StorageTableResolver implem
         storageTableMap.put(updatePeriod, storageTables);
         List<String> parts = partitionColMap.get(updatePeriod);
         for (String storage : fact.getStorages()) {
-          if (cubeqlStorage.isStorageSupported(storage)) {
+          if (cubeql.isStorageSupported(storage)) {
             String tableName = MetastoreUtil.getFactStorageTableName(
                 fact.getName(), updatePeriod, Storage.getPrefix(storage));
             storageTables.add(tableName);
@@ -54,7 +52,7 @@ public class StorageTableResolver implem
         }
       }
     }
-    cubeqlStorage.setFactStorageMap(factStorageMap);
+    cubeql.setFactStorageMap(factStorageMap);
 
     //resolve dimension tables
     Map<CubeDimensionTable, List<String>> dimStorageMap =
@@ -63,7 +61,7 @@ public class StorageTableResolver implem
         List<String> storageTables = new ArrayList<String>();
         dimStorageMap.put(dim, storageTables);
         for (String storage : dim.getStorages()) {
-          if (cubeqlStorage.isStorageSupported(storage)) {
+          if (cubeql.isStorageSupported(storage)) {
             String tableName = MetastoreUtil.getDimStorageTableName(
                 dim.getName(), Storage.getPrefix(storage));
             storageTables.add(tableName);
@@ -76,8 +74,8 @@ public class StorageTableResolver implem
           }
         }
       }
-    cubeqlStorage.setDimStorageMap(dimStorageMap);
-    cubeqlStorage.setStorageTableToWhereClause(storageTableToWhereClause);
+    cubeql.setDimStorageMap(dimStorageMap);
+    cubeql.setStorageTableToWhereClause(storageTableToWhereClause);
   }
 
   private String getWherePartClause(String tableName, List<String> parts) {

Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1469787&r1=1469786&r2=1469787&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Fri Apr 19 11:39:45 2013
@@ -14,7 +14,7 @@ import org.junit.Test;
 
 public class TestCubeDriver {
 
-  private final Configuration conf = new Configuration();
+  private Configuration conf;
   private CubeDriver driver;
 
   @BeforeClass
@@ -39,6 +39,7 @@ public class TestCubeDriver {
 
   @Test
   public void testQueryWithNow() throws Exception {
+    conf = new Configuration();
     driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
     Throwable th = null;
     try {
@@ -53,6 +54,7 @@ public class TestCubeDriver {
 
   @Test
   public void testCubeWhereQuery() throws Exception {
+    conf = new Configuration();
     driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
     Calendar cal = Calendar.getInstance();
     Date now = cal.getTime();
@@ -70,6 +72,7 @@ public class TestCubeDriver {
 
   @Test
   public void testCubeWhereQueryForMonth() throws Exception {
+    conf = new Configuration();
     driver = new CubeDriver(new HiveConf(new Configuration(), HiveConf.class));
     Calendar cal = Calendar.getInstance();
     Date now = cal.getTime();
@@ -94,6 +97,7 @@ public class TestCubeDriver {
 
   @Test
   public void testDimensionQueryWithMultipleStorages() throws Exception {
+    conf = new Configuration();
     driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
     String hqlQuery = driver.compileCubeQuery("select name, stateid from citytable");
     System.out.println("cube hql:" + hqlQuery);
@@ -113,15 +117,23 @@ public class TestCubeDriver {
     driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
     hqlQuery = driver.compileCubeQuery("select name, stateid from citytable");
     System.out.println("cube hql:" + hqlQuery);
-
   }
 
   @Test
-  public void testLimitQuery() throws Exception {
-    driver = new CubeDriver(new HiveConf(new Configuration(), HiveConf.class));
+  public void testLimitQueryOnDimension() throws Exception {
+    conf = new Configuration();
+    driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
     String hqlQuery = driver.compileCubeQuery("select name, stateid from citytable limit
100");
     System.out.println("cube hql:" + hqlQuery);
     //Assert.assertEquals(queries[1], cubeql.toHQL());
+    conf.set(HiveConf.ConfVars.HIVE_DRIVER_SUPPORTED_STORAGES.toString(), "C2");
+    driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
+    hqlQuery = driver.compileCubeQuery("select name, stateid from citytable limit 100");
+    System.out.println("cube hql:" + hqlQuery);
+    conf.set(HiveConf.ConfVars.HIVE_DRIVER_SUPPORTED_STORAGES.toString(), "C1");
+    driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
+    hqlQuery = driver.compileCubeQuery("select name, stateid from citytable limit 100");
+    System.out.println("cube hql:" + hqlQuery);
   }
 
 }



Mime
View raw message