hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r1478649 - in /hive/branches/HIVE-4115/ql/src: java/org/apache/hadoop/hive/ql/cube/parse/ test/org/apache/hadoop/hive/ql/cube/parse/ test/org/apache/hadoop/hive/ql/cube/processors/
Date Fri, 03 May 2013 05:52:37 GMT
Author: amareshwari
Date: Fri May  3 05:52:37 2013
New Revision: 1478649

URL: http://svn.apache.org/r1478649
Log:
Add least partition resolver

Modified:
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
    hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
    hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
    hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
Fri May  3 05:52:37 2013
@@ -66,8 +66,6 @@ public class CubeQueryContext {
   private final Map<String, List<String>> tblAliasToColumns = new HashMap<String,
List<String>>();
   private final Set<String> cubeColumnsQueried = new HashSet<String>();
   private final Map<String, String> columnToTabAlias = new HashMap<String, String>();
-  protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap
=
-      new HashMap<CubeFactTable, Map<UpdatePeriod, List<String>>>();
   private final Map<CubeQueryExpr, Set<String>> exprToCols = new HashMap<CubeQueryExpr,
Set<String>>();
   private final Map<CubeQueryExpr, Set<String>> queryExprToExprs = new HashMap<CubeQueryExpr,
Set<String>>();
   private final Map<String, String> exprToAlias = new HashMap<String, String>();
@@ -75,6 +73,8 @@ public class CubeQueryContext {
   private final Set<String> aggregateExprs = new HashSet<String>();
 
   // storage specific
+  protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap
=
+      new HashMap<CubeFactTable, Map<UpdatePeriod, List<String>>>();
   private List<String> supportedStorages;
   private boolean allStoragesSupported;
   private final Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factStorageMap
=

Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
Fri May  3 05:52:37 2013
@@ -1,19 +1,54 @@
 package org.apache.hadoop.hive.ql.cube.parse;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
+import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 public class LeastPartitionResolver implements ContextRewriter {
 
   public LeastPartitionResolver(Configuration conf) {
-    // TODO Auto-generated constructor stub
   }
 
   @Override
   public void rewriteContext(CubeQueryContext cubeql)
       throws SemanticException {
-    // TODO Auto-generated method stub
-
+    if (cubeql.getCube() != null) {
+      Map<CubeFactTable, Integer> numPartitionsMap =
+          new HashMap<CubeFactTable, Integer>();
+
+      for (CubeFactTable fact : cubeql.getCandidateFactTables()) {
+         numPartitionsMap.put(fact, getTotalPartitions(
+             cubeql.getFactPartitionMap().get(fact)));
+      }
+
+      int minPartitions = Collections.min(numPartitionsMap.values());
+
+      for (Iterator<CubeFactTable> i =
+          cubeql.getCandidateFactTables().iterator(); i.hasNext();) {
+        CubeFactTable fact = i.next();
+        if (numPartitionsMap.get(fact) > minPartitions) {
+          System.out.println("Removing fact:" + fact +
+              " from candidate fact tables as it requires more partitions to" +
+              " be queried:" +  numPartitionsMap.get(fact) + " minimum:"
+              + minPartitions);
+          i.remove();
+        }
+      }
+    }
   }
 
+  private int getTotalPartitions(Map<UpdatePeriod, List<String>> map) {
+    int count = 0;
+    for (List<String> parts : map.values()) {
+      count += parts.size();
+    }
+    return count;
+  }
 }

Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
(original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
Fri May  3 05:52:37 2013
@@ -112,6 +112,32 @@ public class CubeTestSetup {
         storageAggregatePeriods);
   }
 
+  private void createCubeFactOnlyHourly(CubeMetastoreClient client) throws HiveException
{
+    String factName = "testFact2";
+    List<FieldSchema> factColumns = new ArrayList<FieldSchema>(
+        cubeMeasures.size());
+    for (CubeMeasure measure : cubeMeasures) {
+      factColumns.add(measure.getColumn());
+    }
+
+    // add dimensions of the cube
+    factColumns.add(new FieldSchema("zipcode","int", "zip"));
+    factColumns.add(new FieldSchema("cityid","int", "city id"));
+
+    Map<Storage, List<UpdatePeriod>> storageAggregatePeriods =
+        new HashMap<Storage, List<UpdatePeriod>>();
+    List<UpdatePeriod> updates  = new ArrayList<UpdatePeriod>();
+    updates.add(UpdatePeriod.HOURLY);
+    Storage hdfsStorage = new HDFSStorage("C1",
+        TextInputFormat.class.getCanonicalName(),
+        HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+    storageAggregatePeriods.put(hdfsStorage, updates);
+
+    // create cube fact
+    client.createCubeFactTable(cubeName, factName, factColumns,
+        storageAggregatePeriods);
+  }
+
   private void createCubeFactMonthly(CubeMetastoreClient client) throws HiveException {
     String factName = "testFactMonthly";
     List<FieldSchema> factColumns = new ArrayList<FieldSchema>(
@@ -169,6 +195,7 @@ public class CubeTestSetup {
         new HiveConf(this.getClass()));
     createCube(client);
     createCubeFact(client);
+    createCubeFactOnlyHourly(client);
     createDimWithTwoStorages(client);
     createCubeFactMonthly(client);
   }

Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
(original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Fri May  3 05:52:37 2013
@@ -89,7 +89,9 @@ public class TestCubeDriver {
     conf = new Configuration();
     driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
     System.out.println("Test from:" + getDateUptoHours(twodaysBack) + " to:" + getDateUptoHours(now));
-    //String expected = "select SUM(testCube.msr2) from "
+    //String expected = " sum( testcube.msr2 ) FROM  C1_testfact_HOURLY testcube  WHERE "
+ whereClause(HOURLY) + " UNION " +
+    // SELECT sum( testcube.msr2 ) FROM  C1_testfact_DAILY testcube  WHERE + whereClause(DAILY)
+
     String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
         " where time_range_in('" + getDateUptoHours(twodaysBack)
         + "','" + getDateUptoHours(now) + "')");



Mime
View raw message