lens-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject [1/3] lens git commit: LENS-795 : Look ahead is failing with partition not found if any process time partitions are absent
Date Thu, 08 Oct 2015 09:45:50 GMT
Repository: lens
Updated Branches:
  refs/heads/master 2ecef112f -> 7080b76c6


LENS-795 : Look ahead is failing with partition not found if any process time partitions are
absent


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d7f09ede
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d7f09ede
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d7f09ede

Branch: refs/heads/master
Commit: d7f09ede37e1f2d75e3e3d62a2e3a0a2375575cc
Parents: e5b793c
Author: Rajat Khandelwal <prongs@apache.org>
Authored: Wed Sep 30 19:17:26 2015 +0530
Committer: Raju Bairishetti <raju.bairishetti@im0933-x0.corp.inmobi.com>
Committed: Wed Sep 30 19:17:26 2015 +0530

----------------------------------------------------------------------
 .../PartitionRangesForPartitionColumns.java      |  6 ++++--
 .../lens/cube/parse/StorageTableResolver.java    | 19 +++++++++++++++----
 2 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/d7f09ede/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
index b389a8f..fa3f6ef 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
@@ -41,10 +41,12 @@ public class PartitionRangesForPartitionColumns extends HashMap<String,
RangesPa
       .rangeUpto(partition.next().withUpdatePeriod(UpdatePeriod.values()[0])));
   }
 
-  public Set<String> toSet() {
+  public Set<String> toSet(Set<String> partColsQueried) {
     Set<String> ret = Sets.newHashSet();
     for (Map.Entry<String, RangesPartitionTimeline> entry : entrySet()) {
-      ret.add(entry.getKey() + ":" + entry.getValue().getRanges());
+      if (partColsQueried.contains(entry.getKey())) {
+        ret.add(entry.getKey() + ":" + entry.getValue().getRanges());
+      }
     }
     return ret;
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/d7f09ede/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 68ab5ab..f67fc26 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -363,7 +363,9 @@ class StorageTableResolver implements ContextRewriter {
       PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns();
       boolean noPartsForRange = false;
       Set<String> unsupportedTimeDims = Sets.newHashSet();
+      Set<String> partColsQueried = Sets.newHashSet();
       for (TimeRange range : cubeql.getTimeRanges()) {
+        partColsQueried.add(range.getPartitionColumn());
         StringBuilder extraWhereClause = new StringBuilder();
         Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses,
missingParts);
         // If no partitions were found, then we'll fallback.
@@ -389,6 +391,7 @@ class StorageTableResolver implements ContextRewriter {
           if (fallBackRange == null) {
             break;
           }
+          partColsQueried.add(fallBackRange.getPartitionColumn());
           rangeParts = getPartitions(cfact.fact, fallBackRange, skipStorageCauses, missingParts);
           extraWhereClause.append(sep)
             .append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()),
timeDim));
@@ -433,7 +436,7 @@ class StorageTableResolver implements ContextRewriter {
         i.remove();
         continue;
       }
-      Set<String> nonExistingParts = missingParts.toSet();
+      Set<String> nonExistingParts = missingParts.toSet(partColsQueried);
       if (!nonExistingParts.isEmpty()) {
         addNonExistingParts(cfact.fact.getName(), nonExistingParts);
       }
@@ -637,8 +640,10 @@ class StorageTableResolver implements ContextRewriter {
                   for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) {
                     FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(),
pPart,
                       partWhereClauseFormat);
-                    updateFactPartitionStorageTablesFrom(fact, innerPart, pPart.getStorageTables());
-                    partitions.add(innerPart);
+                    updateFactPartitionStorageTablesFrom(fact, innerPart, pPart);
+                    if (innerPart.isFound()) {
+                      partitions.add(innerPart);
+                    }
                   }
                   log.debug("added all sub partitions blindly in pPart: {}", pPart);
                 }
@@ -678,7 +683,7 @@ class StorageTableResolver implements ContextRewriter {
         updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions);
   }
 
-  void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
+  private void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
     FactPartition part, Set<String> storageTableNames) throws LensException, HiveException,
ParseException {
     for (String storageTableName : storageTableNames) {
       if (client.factPartitionExists(fact, part, storageTableName)) {
@@ -687,4 +692,10 @@ class StorageTableResolver implements ContextRewriter {
       }
     }
   }
+
+  private void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
+    FactPartition part, FactPartition pPart) throws LensException, HiveException, ParseException
{
+    updateFactPartitionStorageTablesFrom(fact, part, pPart.getStorageTables());
+    part.setFound(part.isFound() && pPart.isFound());
+  }
 }


Mime
View raw message