lens-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject [33/51] [abbrv] lens git commit: LENS-1391 : Fix pending review comments on MaxCoveringFactResolver and LeastPartitionResolver
Date Sun, 21 May 2017 11:43:29 GMT
LENS-1391 : Fix pending review comments on MaxCoveringFactResolver and LeastPartitionResolver


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c4cd6d84
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c4cd6d84
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c4cd6d84

Branch: refs/heads/current-release-line
Commit: c4cd6d84ec533d84c24717591c890972f5d858cb
Parents: f8a3879
Author: Lavkesh Lahngir <lavkesh@linux.com>
Authored: Wed Apr 19 11:05:57 2017 +0530
Committer: Amareshwari Sriramadasu <amareshwari@apache.org>
Committed: Wed Apr 19 11:05:57 2017 +0530

----------------------------------------------------------------------
 .../lens/cli/commands/LensSchemaCommands.java   | 12 +---------
 .../cube/parse/CandidateTablePruneCause.java    |  7 ++++++
 .../lens/cube/parse/LeastPartitionResolver.java | 15 +++----------
 .../cube/parse/MaxCoveringFactResolver.java     | 23 ++++++--------------
 .../lens/cube/parse/StorageCandidate.java       |  5 +++--
 .../server/query/QueryAPIErrorResponseTest.java |  2 +-
 6 files changed, 22 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index befe4e6..d3f9142 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -20,19 +20,11 @@ package org.apache.lens.cli.commands;
 
 import java.io.File;
 import java.io.FilenameFilter;
-import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.lens.api.metastore.SchemaTraverser;
-import org.apache.lens.api.metastore.XBaseCube;
-import org.apache.lens.api.metastore.XDerivedCube;
-import org.apache.lens.api.metastore.XDimension;
-import org.apache.lens.api.metastore.XDimensionTable;
-import org.apache.lens.api.metastore.XFactTable;
-import org.apache.lens.api.metastore.XSegmentation;
-import org.apache.lens.api.metastore.XStorage;
+import org.apache.lens.api.metastore.*;
 import org.apache.lens.cli.commands.annotations.UserDocumentation;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -42,9 +34,7 @@ import org.springframework.shell.core.annotation.CliCommand;
 import org.springframework.shell.core.annotation.CliOption;
 import org.springframework.shell.support.logging.HandlerUtils;
 import org.springframework.stereotype.Component;
-import org.springframework.util.Assert;
 
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 @Component

http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 1c0d356..29af419 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -235,6 +235,7 @@ public class CandidateTablePruneCause {
   // if a time dim is not supported by the fact. Would be set if and only if
   // the fact is not partitioned by part col of the time dim and time dim is not a dim attribute
   private Set<String> unsupportedTimeDims;
+  private MaxCoveringFactResolver.TimeCovered maxTimeCovered;
   // time covered
   // ranges in which fact is invalid
   private List<TimeRange> invalidRanges;
@@ -295,6 +296,12 @@ public class CandidateTablePruneCause {
     return cause;
   }
 
+  public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered)
{
+    CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA);
+    cause.setMaxTimeCovered(timeCovered);
+    return cause;
+  }
+
   public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection<String>
colSet) {
     CandidateTablePruneCause cause =
       new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH);

http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index a9bd164..5adbc23 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -28,7 +28,7 @@ import org.apache.lens.server.api.error.LensException;
 import lombok.extern.slf4j.Slf4j;
 
 /**
- * Prune candidate fact sets which require more partitions than minimum parts.
+ * Prune candidates which require more partitions than minimum parts.
  */
 @Slf4j
 class LeastPartitionResolver implements ContextRewriter {
@@ -41,9 +41,9 @@ class LeastPartitionResolver implements ContextRewriter {
       //The number of partitions being calculated is not the actual number of partitions,
       // they are number of time values now instead of partitions.
       // This seems fine, as the less number of time values actually represent the rollups
on time. And with
-      // MaxCoveringFactResolver facts with less partitions which are not covering the range
would be removed.
+      // MaxCoveringFactResolver candidates with less partitions which are not covering the
range would be removed.
       for (Candidate candidate : cubeql.getCandidates()) {
-        factPartCount.put(candidate, getPartCount(candidate));
+        factPartCount.put(candidate, candidate.getParticipatingPartitions().size());
       }
 
       double minPartitions = Collections.min(factPartCount.values());
@@ -60,13 +60,4 @@ class LeastPartitionResolver implements ContextRewriter {
       }
     }
   }
-
-  private int getPartCount(Candidate candidate) {
-    int parts = 0;
-    for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) {
-      parts += sc.getNumQueriedParts();
-    }
-    return parts;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
index 34180d1..86e5634 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
@@ -23,7 +23,6 @@ import java.util.Iterator;
 import java.util.Map;
 
 import org.apache.lens.cube.metadata.FactPartition;
-import org.apache.lens.cube.metadata.UpdatePeriod;
 import org.apache.lens.cube.metadata.timeline.RangesPartitionTimeline;
 import org.apache.lens.server.api.error.LensException;
 
@@ -47,7 +46,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
   public void rewriteContext(CubeQueryContext cubeql) {
     if (failOnPartialData) {
       // if fail on partial data is true, by the time this resolver starts,
-      // all candidate fact sets are covering full time range. We can avoid
+      // all candidates are covering full time range. We can avoid
       // redundant computation.
       return;
     }
@@ -62,7 +61,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
   }
 
   private void resolveByTimeCovered(CubeQueryContext cubeql) {
-    // For each part column, which candidate fact sets are covering how much amount.
+    // For each part column, which candidates are covering how much amount.
     // Later, we'll maximize coverage for each queried part column.
     Map<String, Map<Candidate, Long>> partCountsPerPartCol = Maps.newHashMap();
     for (Candidate cand : cubeql.getCandidates()) {
@@ -73,7 +72,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
         partCountsPerPartCol.get(entry.getKey()).put(cand, entry.getValue());
       }
     }
-    // for each queried partition, prune fact sets that are covering less range than max
+    // for each queried partition, prune candidate that are covering less range than max
     for (String partColQueried : cubeql.getPartitionColumnsQueried()) {
       if (partCountsPerPartCol.get(partColQueried) != null) {
         long maxTimeCovered = Collections.max(partCountsPerPartCol.get(partColQueried).values());
@@ -89,8 +88,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
             log.info("Not considering Candidate:{} from Candidate set as it covers less time
than the max"
               + " for partition column: {} which is: {}", candidate, partColQueried, timeCovered);
             iter.remove();
-            cubeql.addCandidatePruningMsg(candidate,
-              new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.LESS_DATA));
+            cubeql.addCandidatePruningMsg(candidate, CandidateTablePruneCause.lessData(timeCovered));
           }
         }
       }
@@ -112,7 +110,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
       return;
     }
 
-    // We prune those candidate fact set, whose dataCompletenessFactor is less than maxDataCompletenessFactor
+    // We prune those candidate whose dataCompletenessFactor is less than maxDataCompletenessFactor
     Iterator<Candidate> iter = cubeql.getCandidates().iterator();
     while (iter.hasNext()) {
       Candidate cand = iter.next();
@@ -121,8 +119,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
         log.info("Not considering Candidate :{} from the list as the dataCompletenessFactor
for this:{} is "
           + "less than the max:{}", cand, dataCompletenessFactor, maxDataCompletenessFactor);
         iter.remove();
-        cubeql.addCandidatePruningMsg(cand,
-          new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.INCOMPLETE_PARTITION));
+        cubeql.addCandidatePruningMsg(cand, CandidateTablePruneCause.incompletePartitions(null));
       }
     }
   }
@@ -145,19 +142,13 @@ class MaxCoveringFactResolver implements ContextRewriter {
   }
 
   /**
-   * Returns time covered by fact set for each part column.
+   * Returns time covered by candidate for each part column.
    *
    * @param cand
    * @return
    */
   private Map<String, Long> getTimeCoveredForEachPartCol(Candidate cand) {
     Map<String, Long> ret = Maps.newHashMap();
-    UpdatePeriod smallest = UpdatePeriod.values()[UpdatePeriod.values().length - 1];
-    for (FactPartition part : cand.getParticipatingPartitions()) {
-      if (part.getPeriod().compareTo(smallest) < 0) {
-        smallest = part.getPeriod();
-      }
-    }
     PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns();
     for (FactPartition part : cand.getParticipatingPartitions()) {
       if (part.isFound()) {

http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index d95cf27..95e3c95 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -152,8 +152,6 @@ public class StorageCandidate implements Candidate, CandidateTable {
    */
   @Getter
   private Set<String> nonExistingPartitions = new HashSet<>();
-  @Getter
-  private int numQueriedParts = 0;
 
   /**
    * This will be true if this storage candidate has multiple storage tables (one per update
period)
@@ -162,6 +160,9 @@ public class StorageCandidate implements Candidate, CandidateTable {
   @Getter
   private boolean isStorageTblsAtUpdatePeriodLevel;
 
+  @Getter
+  private int numQueriedParts = 0;
+
   public StorageCandidate(StorageCandidate sc) throws LensException {
     this(sc.getCube(), sc.getFact(), sc.getStorageName(), sc.getCubeql());
     this.validUpdatePeriods.addAll(sc.getValidUpdatePeriods());

http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 064da01..5409d21 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -283,7 +283,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
 
       //Create a StorageTable
       XStorageTables tables = new XStorageTables();
-      tables.getStorageTable().add(createStorageTblElement(testStorage,"DAILY"));
+      tables.getStorageTable().add(createStorageTblElement(testStorage, "DAILY"));
       xFactTable.setStorageTables(tables);
 
       createFactFailFast(target, sessionId, xFactTable, mt);


Mime
View raw message