impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [08/50] [abbrv] incubator-impala git commit: IMPALA-4592: Improve error msg for non-deterministic predicates.
Date Thu, 15 Dec 2016 23:00:34 GMT
IMPALA-4592: Improve error msg for non-deterministic predicates.

Impala cannot correctly evaluate or assign some non-deterministic
predicates. This patch improves the error message shown when
trying to evaluate such unsupported predicates for the purpose
of partition pruning.

Change-Id: I94765f62bde94f4faa7fc5c26d928099ca1496d1
Reviewed-on: http://gerrit.cloudera.org:8080/5386
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6098ac71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6098ac71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6098ac71

Branch: refs/heads/hadoop-next
Commit: 6098ac7162742c11350de708188ce6c3f7ce11a7
Parents: 5188f87
Author: Alex Behm <alex.behm@cloudera.com>
Authored: Tue Dec 6 14:30:43 2016 -0800
Committer: Internal Jenkins <cloudera-hudson@gerrit.cloudera.org>
Committed: Wed Dec 7 06:27:51 2016 +0000

----------------------------------------------------------------------
 .../org/apache/impala/analysis/PartitionSet.java  |  5 +++--
 .../impala/planner/HdfsPartitionFilter.java       | 18 +++++++++++-------
 .../impala/planner/HdfsPartitionPruner.java       |  6 +++---
 .../queries/PlannerTest/hdfs.test                 | 11 +++++++++++
 4 files changed, 28 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
index 6de2d93..3ba2ad2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
@@ -25,7 +25,7 @@ import org.apache.impala.catalog.Column;
 import org.apache.impala.catalog.HdfsPartition;
 import org.apache.impala.catalog.Table;
 import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.ImpalaException;
 import org.apache.impala.common.Reference;
 import org.apache.impala.planner.HdfsPartitionPruner;
 import org.apache.impala.thrift.TPartitionKeyValue;
@@ -87,7 +87,8 @@ public class PartitionSet extends PartitionSpecBase {
     try {
       HdfsPartitionPruner pruner = new HdfsPartitionPruner(desc);
       partitions_ = pruner.prunePartitions(analyzer, transformedConjuncts, true);
-    } catch (InternalException e) {
+    } catch (ImpalaException e) {
+      if (e instanceof AnalysisException) throw (AnalysisException) e;
       throw new AnalysisException("Partition expr evaluation failed in the backend.", e);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
index 7368358..3c0fb15 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionFilter.java
@@ -21,9 +21,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import org.apache.impala.analysis.Analyzer;
 import org.apache.impala.analysis.Expr;
 import org.apache.impala.analysis.ExprSubstitutionMap;
@@ -33,10 +30,14 @@ import org.apache.impala.analysis.SlotRef;
 import org.apache.impala.catalog.Column;
 import org.apache.impala.catalog.HdfsPartition;
 import org.apache.impala.catalog.HdfsTable;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.NotImplementedException;
 import org.apache.impala.service.FeSupport;
 import org.apache.impala.thrift.TColumnValue;
 import org.apache.impala.thrift.TResultRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -83,7 +84,7 @@ public class HdfsPartitionFilter {
    * that pass the filter.
    */
   public HashSet<Long> getMatchingPartitionIds(ArrayList<HdfsPartition> partitions,
-      Analyzer analyzer) throws InternalException {
+      Analyzer analyzer) throws ImpalaException {
     HashSet<Long> result = new HashSet<Long>();
     // List of predicates to evaluate
     ArrayList<Expr> predicates = new ArrayList<Expr>(partitions.size());
@@ -110,7 +111,7 @@ public class HdfsPartitionFilter {
    * for the partition cols with the respective partition-key values.
    */
   private Expr buildPartitionPredicate(HdfsPartition partition, Analyzer analyzer)
-      throws InternalException {
+      throws ImpalaException {
     // construct smap
     ExprSubstitutionMap sMap = new ExprSubstitutionMap();
     for (int i = 0; i < refdKeys_.size(); ++i) {
@@ -123,7 +124,10 @@ public class HdfsPartitionFilter {
       LOG.trace("buildPartitionPredicate: " + literalPredicate.toSql() + " " +
           literalPredicate.debugString());
     }
-    Preconditions.checkState(literalPredicate.isConstant());
+    if (!literalPredicate.isConstant()) {
+      throw new NotImplementedException(
+          "Unsupported non-deterministic predicate: " + predicate_.toSql());
+    }
     return literalPredicate;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
index b8eb7ed..046240f 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
@@ -42,7 +42,7 @@ import org.apache.impala.analysis.TupleDescriptor;
 import org.apache.impala.catalog.HdfsPartition;
 import org.apache.impala.catalog.HdfsTable;
 import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.ImpalaException;
 import org.apache.impala.rewrite.BetweenToCompoundRule;
 import org.apache.impala.rewrite.ExprRewriter;
 import org.slf4j.Logger;
@@ -94,7 +94,7 @@ public class HdfsPartitionPruner {
    */
   public List<HdfsPartition> prunePartitions(
       Analyzer analyzer, List<Expr> conjuncts, boolean allowEmpty)
-      throws InternalException, AnalysisException {
+      throws ImpalaException {
     // Start with creating a collection of partition filters for the applicable conjuncts.
     List<HdfsPartitionFilter> partitionFilters = Lists.newArrayList();
     // Conjuncts that can be evaluated from the partition key values.
@@ -438,7 +438,7 @@ public class HdfsPartitionPruner {
    * filters that could not be evaluated from the partition key values.
    */
   private void evalPartitionFiltersInBe(List<HdfsPartitionFilter> filters,
-      HashSet<Long> matchingPartitionIds, Analyzer analyzer) throws InternalException
{
+      HashSet<Long> matchingPartitionIds, Analyzer analyzer) throws ImpalaException
{
     Map<Long, HdfsPartition> partitionMap = tbl_.getPartitionMap();
     // Set of partition ids that pass a filter
     HashSet<Long> matchingIds = Sets.newHashSet();

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6098ac71/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
index 606425b..83f1b46 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
@@ -1019,3 +1019,14 @@ PLAN-ROOT SINK
 00:SCAN HDFS [functional.alltypes]
    partitions=0/24 files=0 size=0B
 ====
+# IMPALA-4592: Test that we bail on evaluating non-deterministic predicates when trying
+# to prune partitions.
+select * from functional.alltypes where rand() > 100
+---- PLAN
+not implemented: Unsupported non-deterministic predicate: rand() > 100
+====
+# IMPALA-4592: Same as above but the predicate references a partition column.
+select * from functional.alltypes where rand() > year
+---- PLAN
+not implemented: Unsupported non-deterministic predicate: rand() > year
+====


Mime
View raw message