quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jianq...@apache.org
Subject incubator-quickstep git commit: Use BitVector as LIPFilter implementation when applicable
Date Wed, 22 Feb 2017 20:05:27 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/use-exact-filter [created] 2b2d7ba19


Use BitVector as LIPFilter implementation when applicable


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2b2d7ba1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2b2d7ba1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2b2d7ba1

Branch: refs/heads/use-exact-filter
Commit: 2b2d7ba1970ade47b1170cd7974cb2fc53f7ba71
Parents: 1572762
Author: Jianqiao Zhu <jianqiao@cs.wisc.edu>
Authored: Wed Feb 22 14:06:55 2017 -0600
Committer: Jianqiao Zhu <jianqiao@cs.wisc.edu>
Committed: Wed Feb 22 14:06:55 2017 -0600

----------------------------------------------------------------------
 query_optimizer/rules/AttachLIPFilters.cpp | 74 +++++++++++++++++++++++--
 query_optimizer/rules/AttachLIPFilters.hpp |  9 +++
 query_optimizer/rules/CMakeLists.txt       |  1 +
 3 files changed, 79 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2b2d7ba1/query_optimizer/rules/AttachLIPFilters.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/AttachLIPFilters.cpp b/query_optimizer/rules/AttachLIPFilters.cpp
index 4b6ac59..9a13b48 100644
--- a/query_optimizer/rules/AttachLIPFilters.cpp
+++ b/query_optimizer/rules/AttachLIPFilters.cpp
@@ -20,6 +20,7 @@
 #include "query_optimizer/rules/AttachLIPFilters.hpp"
 
 #include <algorithm>
+#include <cstdint>
 #include <map>
 #include <set>
 #include <unordered_set>
@@ -37,6 +38,7 @@
 #include "query_optimizer/physical/PhysicalType.hpp"
 #include "query_optimizer/physical/Selection.hpp"
 #include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "types/TypeID.hpp"
 #include "types/TypedValue.hpp"
 #include "utility/lip_filter/LIPFilter.hpp"
 
@@ -126,11 +128,40 @@ void AttachLIPFilters::attachLIPFilters(
         const E::ExprId source_attr_id = pair.second->source_attribute->id();
         if (already_filtered_attributes->find(source_attr_id)
                 == already_filtered_attributes->end()) {
-          lip_filter_configuration_->addBuildInfo(
-              P::SingleIdentityHashFilterBuildInfo::Create(
-                  pair.second->source_attribute,
-                  std::max(64uL, pair.second->estimated_cardinality * 8u)),
-              pair.second->source);
+          bool use_exact_filter = false;
+          std::int64_t min_cpp_value;
+          std::int64_t max_cpp_value;
+          const bool has_exact_min_max_stats =
+              findExactMinMaxValuesForAttributeHelper(pair.second->source,
+                                                      pair.second->source_attribute,
+                                                      &min_cpp_value,
+                                                      &max_cpp_value);
+          if (has_exact_min_max_stats) {
+            const std::int64_t value_range = max_cpp_value - min_cpp_value;
+            DCHECK_GE(value_range, 0);
+            // TODO(jianqiao): Add this threshold as a gflag (together with
+            // InjectJoinFilters::kMaxFilterSize).
+            if (value_range <= 1000000000L) {
+              use_exact_filter = true;
+            }
+          }
+
+          if (use_exact_filter) {
+            lip_filter_configuration_->addBuildInfo(
+                P::BitVectorExactFilterBuildInfo::Create(
+                    pair.second->source_attribute,
+                    min_cpp_value,
+                    max_cpp_value,
+                    false),
+                pair.second->source);
+          } else {
+            lip_filter_configuration_->addBuildInfo(
+                P::SingleIdentityHashFilterBuildInfo::Create(
+                    pair.second->source_attribute,
+                    std::max(64uL, pair.second->estimated_cardinality * 8u)),
+                pair.second->source);
+          }
+
           lip_filter_configuration_->addProbeInfo(
               P::LIPFilterProbeInfo::Create(
                   pair.first,
@@ -258,5 +289,38 @@ const std::vector<AttachLIPFilters::LIPFilterInfoPtr>& AttachLIPFilters
   return probe_side_info_.at(node);
 }
 
+bool AttachLIPFilters::findExactMinMaxValuesForAttributeHelper(
+    const physical::PhysicalPtr &physical_plan,
+    const expressions::AttributeReferencePtr &attribute,
+    std::int64_t *min_cpp_value,
+    std::int64_t *max_cpp_value) const {
+  bool min_value_is_exact;
+  bool max_value_is_exact;
+
+  const TypedValue min_value =
+      cost_model_->findMinValueStat(physical_plan, attribute, &min_value_is_exact);
+  const TypedValue max_value =
+      cost_model_->findMaxValueStat(physical_plan, attribute, &max_value_is_exact);
+  if (min_value.isNull() || max_value.isNull() ||
+      (!min_value_is_exact) || (!max_value_is_exact)) {
+    return false;
+  }
+
+  switch (attribute->getValueType().getTypeID()) {
+    case TypeID::kInt: {
+      *min_cpp_value = min_value.getLiteral<int>();
+      *max_cpp_value = max_value.getLiteral<int>();
+      return true;
+    }
+    case TypeID::kLong: {
+      *min_cpp_value = min_value.getLiteral<std::int64_t>();
+      *max_cpp_value = max_value.getLiteral<std::int64_t>();
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+
 }  // namespace optimizer
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2b2d7ba1/query_optimizer/rules/AttachLIPFilters.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/AttachLIPFilters.hpp b/query_optimizer/rules/AttachLIPFilters.hpp
index b8cfc39..36cb010 100644
--- a/query_optimizer/rules/AttachLIPFilters.hpp
+++ b/query_optimizer/rules/AttachLIPFilters.hpp
@@ -21,6 +21,7 @@
 #define QUICKSTEP_QUERY_OPTIMIZER_RULES_ATTACH_LIP_FILTERS_HPP_
 
 #include <cstddef>
+#include <cstdint>
 #include <map>
 #include <memory>
 #include <set>
@@ -135,6 +136,14 @@ class AttachLIPFilters : public Rule<physical::Physical> {
 
   const std::vector<LIPFilterInfoPtr>& getProbeSideInfo(const NodeList &path);
 
+  // TODO(jianqiao): refactor this method as it is a duplication of
+  // InjectJoinFilters::findExactMinMaxValuesForAttributeHelper().
+  bool findExactMinMaxValuesForAttributeHelper(
+      const physical::PhysicalPtr &physical_plan,
+      const expressions::AttributeReferencePtr &attribute,
+      std::int64_t *min_cpp_value,
+      std::int64_t *max_cpp_value) const;
+
   std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_;
   std::map<physical::PhysicalPtr, std::vector<LIPFilterInfoPtr>> build_side_info_;
   std::map<physical::PhysicalPtr, std::vector<LIPFilterInfoPtr>> probe_side_info_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2b2d7ba1/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 427500d..6847951 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -59,6 +59,7 @@ target_link_libraries(quickstep_queryoptimizer_rules_AttachLIPFilters
                       quickstep_queryoptimizer_physical_Selection
                       quickstep_queryoptimizer_physical_TopLevelPlan
                       quickstep_queryoptimizer_rules_Rule
+                      quickstep_types_TypeID
                       quickstep_types_TypedValue
                       quickstep_utility_Macros
                       quickstep_utility_lipfilter_LIPFilter)


Mime
View raw message