Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B2935200C18 for ; Sat, 28 Jan 2017 00:41:37 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id B113F160B5C; Fri, 27 Jan 2017 23:41:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3D95A160B5B for ; Sat, 28 Jan 2017 00:41:36 +0100 (CET) Received: (qmail 93261 invoked by uid 500); 27 Jan 2017 23:41:35 -0000 Mailing-List: contact commits-help@quickstep.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@quickstep.incubator.apache.org Delivered-To: mailing list commits@quickstep.incubator.apache.org Received: (qmail 93252 invoked by uid 99); 27 Jan 2017 23:41:35 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 23:41:35 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 06523C0AD7 for ; Fri, 27 Jan 2017 23:41:35 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -5.719 X-Spam-Level: X-Spam-Status: No, score=-5.719 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RCVD_IN_SORBS_SPAM=0.5, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id DaNjqrNs9RYn for ; Fri, 27 Jan 2017 23:41:32 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id DC54F5FBCE for ; Fri, 27 Jan 2017 23:41:31 +0000 (UTC) Received: (qmail 92794 invoked by uid 99); 27 Jan 2017 23:40:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 23:40:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 60BD7DFC61; Fri, 27 Jan 2017 23:40:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianqiao@apache.org To: commits@quickstep.incubator.apache.org Message-Id: <7f8e9dda772042ae8250413a245471cf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-quickstep git commit: Updates Date: Fri, 27 Jan 2017 23:40:41 +0000 (UTC) archived-at: Fri, 27 Jan 2017 23:41:37 -0000 Repository: incubator-quickstep Updated Branches: refs/heads/exact-filter 8b70d662e -> 84e95cb29 Updates Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/84e95cb2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/84e95cb2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/84e95cb2 Branch: refs/heads/exact-filter Commit: 84e95cb2963ce7fbe7be08ae1029feba74a5dcbd Parents: 8b70d66 Author: Jianqiao Zhu Authored: Fri Jan 27 17:40:29 2017 -0600 Committer: Jianqiao Zhu Committed: Fri Jan 27 17:40:29 2017 -0600 ---------------------------------------------------------------------- query_optimizer/LIPFilterGenerator.cpp | 49 ++-- query_optimizer/LIPFilterGenerator.hpp | 4 +- .../physical/LIPFilterConfiguration.hpp | 280 +++++++++++++------ query_optimizer/rules/AttachLIPFilters.cpp | 18 +- query_optimizer/rules/InjectJoinFilters.cpp | 29 +- utility/PlanVisualizer.cpp | 4 +- utility/lip_filter/BitVectorExactFilter.hpp | 43 +-- utility/lip_filter/LIPFilter.proto | 7 +- utility/lip_filter/LIPFilterFactory.cpp | 30 +- 9 files changed, 303 insertions(+), 161 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/query_optimizer/LIPFilterGenerator.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/LIPFilterGenerator.cpp b/query_optimizer/LIPFilterGenerator.cpp index ef984d4..2ce2ea8 100644 --- a/query_optimizer/LIPFilterGenerator.cpp +++ b/query_optimizer/LIPFilterGenerator.cpp @@ -20,11 +20,13 @@ #include "query_optimizer/LIPFilterGenerator.hpp" #include +#include #include #include #include "catalog/CatalogAttribute.hpp" #include "query_execution/QueryContext.pb.h" +#include "query_optimizer/physical/LIPFilterConfiguration.hpp" #include "relational_operators/RelationalOperator.hpp" #include "types/Type.hpp" #include "utility/lip_filter/LIPFilter.hpp" @@ -47,7 +49,7 @@ void LIPFilterGenerator::registerAttributeMap( if (build_it != build_info_map.end()) { auto &map_entry = attribute_map_[node]; for (const auto &info : build_it->second) { - E::ExprId attr_id = info.build_attribute->id(); + E::ExprId attr_id = info->build_attribute()->id(); map_entry.emplace(attr_id, attribute_substitution_map.at(attr_id)); } } @@ -57,7 +59,7 @@ void LIPFilterGenerator::registerAttributeMap( if (probe_it != probe_info_map.end()) { auto &map_entry = attribute_map_[node]; for (const auto &info : probe_it->second) { - E::ExprId attr_id = info.probe_attribute->id(); + E::ExprId attr_id = info->probe_attribute()->id(); map_entry.emplace(attr_id, attribute_substitution_map.at(attr_id)); } } @@ -107,7 +109,7 @@ void LIPFilterGenerator::deployBuilderInternal( serialization::QueryContext *query_context_proto, const physical::PhysicalPtr &builder_node, const QueryPlan::DAGNodeIndex builder_operator_index, - const std::vector &build_info_vec) { + const std::vector &build_info_vec) { auto *lip_filter_deployment_info_proto = getLIPFilterDeploymentProto(builder_operator_index, query_context_proto); @@ -116,31 +118,37 @@ void LIPFilterGenerator::deployBuilderInternal( // Add the LIPFilter information into query context. const QueryContext::lip_filter_id lip_filter_id = query_context_proto->lip_filters_size(); serialization::LIPFilter *lip_filter_proto = query_context_proto->add_lip_filters(); - const CatalogAttribute *target_attr = builder_attribute_map.at(info.build_attribute->id()); + const CatalogAttribute *target_attr = + builder_attribute_map.at(info->build_attribute()->id()); const Type &attr_type = target_attr->getType(); - switch (info.filter_type) { + switch (info->filter_type()) { case LIPFilterType::kSingleIdentityHashFilter: { DCHECK(!attr_type.isVariableLength()); - DCHECK(!info.is_anti_filter); + const P::SingleIdentityHashFilterBuildInfo &sihf_info = + *std::static_pointer_cast(info); lip_filter_proto->set_lip_filter_type( serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER); - lip_filter_proto->SetExtension( - serialization::SingleIdentityHashFilter::filter_cardinality, info.filter_cardinality); - lip_filter_proto->SetExtension( - serialization::SingleIdentityHashFilter::attribute_size, attr_type.minimumByteLength()); + lip_filter_proto->SetExtension(serialization::SingleIdentityHashFilter::filter_cardinality, + sihf_info.filter_cardinality()); + lip_filter_proto->SetExtension(serialization::SingleIdentityHashFilter::attribute_size, + attr_type.minimumByteLength()); break; } case LIPFilterType::kBitVectorExactFilter: { DCHECK(!attr_type.isVariableLength()); + const P::BitVectorExactFilterBuildInfo &bvef_info = + *std::static_pointer_cast(info); lip_filter_proto->set_lip_filter_type( serialization::LIPFilterType::BIT_VECTOR_EXACT_FILTER); - lip_filter_proto->SetExtension( - serialization::BitVectorExactFilter::filter_cardinality, info.filter_cardinality); - lip_filter_proto->SetExtension( - serialization::BitVectorExactFilter::attribute_size, attr_type.minimumByteLength()); - lip_filter_proto->SetExtension( - serialization::BitVectorExactFilter::is_anti_filter, info.is_anti_filter); + lip_filter_proto->SetExtension(serialization::BitVectorExactFilter::min_value, + bvef_info.min_value()); + lip_filter_proto->SetExtension(serialization::BitVectorExactFilter::max_value, + bvef_info.max_value()); + lip_filter_proto->SetExtension(serialization::BitVectorExactFilter::attribute_size, + attr_type.minimumByteLength()); + lip_filter_proto->SetExtension(serialization::BitVectorExactFilter::is_anti_filter, + bvef_info.is_anti_filter()); break; } default: @@ -150,7 +158,7 @@ void LIPFilterGenerator::deployBuilderInternal( // Register the builder information which is needed later by the probers. lip_filter_builder_map_.emplace( - std::make_pair(info.build_attribute->id(), builder_node), + std::make_pair(info->build_attribute()->id(), builder_node), std::make_pair(lip_filter_id, builder_operator_index)); // Add the builder deployment information into query context. @@ -166,7 +174,7 @@ void LIPFilterGenerator::deployProberInteral( serialization::QueryContext *query_context_proto, const physical::PhysicalPtr &prober_node, const QueryPlan::DAGNodeIndex prober_operator_index, - const std::vector &probe_info_vec) { + const std::vector &probe_info_vec) { auto *lip_filter_deployment_info_proto = getLIPFilterDeploymentProto(prober_operator_index, query_context_proto); @@ -175,8 +183,9 @@ void LIPFilterGenerator::deployProberInteral( // Find the corresponding builder for the to-be-probed LIPFilter. const auto &builder_info = lip_filter_builder_map_.at( - std::make_pair(info.build_attribute->id(), info.builder)); - const CatalogAttribute *target_attr = prober_attribute_map.at(info.probe_attribute->id()); + std::make_pair(info->build_attribute()->id(), info->builder())); + const CatalogAttribute *target_attr = + prober_attribute_map.at(info->probe_attribute()->id()); // Add the prober deployment information into query context. auto *lip_filter_entry_proto = lip_filter_deployment_info_proto->add_probe_entries(); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/query_optimizer/LIPFilterGenerator.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/LIPFilterGenerator.hpp b/query_optimizer/LIPFilterGenerator.hpp index f6d931e..750499d 100644 --- a/query_optimizer/LIPFilterGenerator.hpp +++ b/query_optimizer/LIPFilterGenerator.hpp @@ -181,13 +181,13 @@ class LIPFilterGenerator { serialization::QueryContext *query_context_proto, const physical::PhysicalPtr &builder_node, const QueryPlan::DAGNodeIndex builder_operator_index, - const std::vector &build_info_vec); + const std::vector &build_info_vec); void deployProberInteral(QueryPlan *execution_plan, serialization::QueryContext *query_context_proto, const physical::PhysicalPtr &prober_node, const QueryPlan::DAGNodeIndex prober_operator_index, - const std::vector &probe_info_vec); + const std::vector &probe_info_vec); serialization::LIPFilterDeployment* getLIPFilterDeploymentProto( const QueryPlan::DAGNodeIndex op_index, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/query_optimizer/physical/LIPFilterConfiguration.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/physical/LIPFilterConfiguration.hpp b/query_optimizer/physical/LIPFilterConfiguration.hpp index b4ae548..dd56734 100644 --- a/query_optimizer/physical/LIPFilterConfiguration.hpp +++ b/query_optimizer/physical/LIPFilterConfiguration.hpp @@ -21,6 +21,7 @@ #define QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_LIP_FILTER_CONFIGURATION_HPP_ #include +#include #include #include #include @@ -40,75 +41,211 @@ namespace physical { class Physical; typedef std::shared_ptr PhysicalPtr; +class LIPFilterBuildInfo; +typedef std::shared_ptr LIPFilterBuildInfoPtr; + +class LIPFilterProbeInfo; +typedef std::shared_ptr LIPFilterProbeInfoPtr; + /** * @brief Optimizer information for a LIP filter builder. */ -struct LIPFilterBuildInfo { +class LIPFilterBuildInfo { + public: + /** + * @return The LIPFilter's type. + */ + LIPFilterType filter_type() const { + return filter_type_; + } + + /** + * @return The LIPFilter's build attribute. + */ + const expressions::AttributeReferencePtr& build_attribute() const { + return build_attribute_; + } + + protected: /** * @brief Constructor. * - * @param build_attribute_in The attribute to build the LIP filter with. - * @param filter_cardinality_in The LIP filter's cardinality. * @param filter_type_in The LIP filter's type. - * @param is_anti_filter_in Whether this LIPFilter is an anti-filter. + * @param build_attribute_in The attribute to build the LIP filter with. + */ + LIPFilterBuildInfo(const LIPFilterType &filter_type, + const expressions::AttributeReferencePtr &build_attribute) + : filter_type_(filter_type), + build_attribute_(build_attribute) {} + + private: + const LIPFilterType filter_type_; + const expressions::AttributeReferencePtr build_attribute_; + + DISALLOW_COPY_AND_ASSIGN(LIPFilterBuildInfo); +}; + +/** + * @brief Subclass that contains extra information for SingleIdentityHashFilter + * builder. + */ +class SingleIdentityHashFilterBuildInfo : public LIPFilterBuildInfo { + public: + /** + * @return The cardinality of this SingleIdentityHashFilter. */ - LIPFilterBuildInfo(const expressions::AttributeReferencePtr &build_attribute_in, - const std::size_t filter_cardinality_in, - const LIPFilterType &filter_type_in, - const bool is_anti_filter_in) - : build_attribute(build_attribute_in), - filter_cardinality(filter_cardinality_in), - filter_type(filter_type_in), - is_anti_filter(is_anti_filter_in) {} + std::size_t filter_cardinality() const { + return filter_cardinality_; + } /** - * @brief Copy constructor. + * @brief Creates a shared SingleIdentityHashFilterBuildInfo. * - * @param info The LIPFilter build info to copy. - */ - LIPFilterBuildInfo(const LIPFilterBuildInfo &info) - : build_attribute(info.build_attribute), - filter_cardinality(info.filter_cardinality), - filter_type(info.filter_type), - is_anti_filter(info.is_anti_filter) {} - - const expressions::AttributeReferencePtr build_attribute; - const std::size_t filter_cardinality; - const LIPFilterType filter_type; - const bool is_anti_filter; + * @param build_attribute The attribute to build the filter with. + * @param filter_cardinality The cardinality of this SingleIdentityHashFilter. + */ + static LIPFilterBuildInfoPtr Create( + const expressions::AttributeReferencePtr &build_attribute, + const std::size_t filter_cardinality) { + return LIPFilterBuildInfoPtr( + new SingleIdentityHashFilterBuildInfo(build_attribute, + filter_cardinality)); + } + + private: + SingleIdentityHashFilterBuildInfo(const expressions::AttributeReferencePtr &build_attribute, + const std::size_t filter_cardinality) + : LIPFilterBuildInfo(LIPFilterType::kBitVectorExactFilter, + build_attribute), + filter_cardinality_(filter_cardinality) {} + + const std::size_t filter_cardinality_; + + DISALLOW_COPY_AND_ASSIGN(SingleIdentityHashFilterBuildInfo); }; /** - * @brief Optimizer information for a LIP filter prober. + * @brief Subclass that contains extra information for BitVectorExactFilter + * builder. */ -struct LIPFilterProbeInfo { +class BitVectorExactFilterBuildInfo : public LIPFilterBuildInfo { + public: /** - * @brief Constructor. + * @return The minimum possible value for this BitVectorExactFilter. + */ + std::int64_t min_value() const { + return min_value_; + } + + /** + * @return The maximum possible value for this BitVectorExactFilter. + */ + std::int64_t max_value() const { + return max_value_; + } + + /** + * @return Whether this is an anti-filter. + */ + bool is_anti_filter() const { + return is_anti_filter_; + } + + /** + * @brief Creates a shared BitVectorExactFilterBuildInfo. * - * @param probe_attribute_in The attribute to probe the LIP filter with. - * @param build_attribute_in The attribute that the LIP filter is built with. - * @param builder_in The physical node that the LIP filter's builder is attached to. + * @param build_attribute The attribute to build the filter with. + * @param min_value The minimum possible value for this BitVectorExactFilter + * to set. + * @param max_value The maximum possible value for this BitVectorExactFilter + * to set. + * @param is_anti_filter Whether this is an anti-filter. + */ + static LIPFilterBuildInfoPtr Create( + const expressions::AttributeReferencePtr &build_attribute, + const std::int64_t min_value, + const std::int64_t max_value, + const bool is_anti_filter) { + return LIPFilterBuildInfoPtr( + new BitVectorExactFilterBuildInfo(build_attribute, + min_value, + max_value, + is_anti_filter)); + } + + private: + BitVectorExactFilterBuildInfo(const expressions::AttributeReferencePtr &build_attribute, + const std::int64_t min_value, + const std::int64_t max_value, + const bool is_anti_filter) + : LIPFilterBuildInfo(LIPFilterType::kBitVectorExactFilter, + build_attribute), + min_value_(min_value), + max_value_(max_value), + is_anti_filter_(is_anti_filter) {} + + const std::int64_t min_value_; + const std::int64_t max_value_; + const bool is_anti_filter_; + + DISALLOW_COPY_AND_ASSIGN(BitVectorExactFilterBuildInfo); +}; + + +/** + * @brief Optimizer information for a LIP filter prober. + */ +class LIPFilterProbeInfo { + public: + /** + * @return The attribute to probe the LIP Filter with. + */ + const expressions::AttributeReferencePtr& probe_attribute() const { + return probe_attribute_; + } + + /** + * @return The attribute that the LIP filter is built with. + */ + const expressions::AttributeReferencePtr& build_attribute() const { + return build_attribute_; + } + + /** + * @return The physical node that the LIP filter's builder is attached to. */ - LIPFilterProbeInfo(const expressions::AttributeReferencePtr &probe_attribute_in, - const expressions::AttributeReferencePtr &build_attribute_in, - const PhysicalPtr &builder_in) - : probe_attribute(probe_attribute_in), - build_attribute(build_attribute_in), - builder(builder_in) {} + const PhysicalPtr& builder() const { + return builder_; + } /** - * @brief Copy constructor. + * @brief Creates a shared LIPFilterProbeInfo. * - * @param info The LIPFilter probe info to copy. + * @param probe_attribute The attribute to probe the LIP filter with. + * @param build_attribute The attribute that the LIP filter is built with. + * @param builder The physical node that the LIP filter's builder is attached to. */ - LIPFilterProbeInfo(const LIPFilterProbeInfo &info) - : probe_attribute(info.probe_attribute), - build_attribute(info.build_attribute), - builder(info.builder) {} + static LIPFilterProbeInfoPtr Create( + const expressions::AttributeReferencePtr &probe_attribute, + const expressions::AttributeReferencePtr &build_attribute, + const PhysicalPtr &builder) { + return LIPFilterProbeInfoPtr( + new LIPFilterProbeInfo(probe_attribute, build_attribute, builder)); + } + + private: + LIPFilterProbeInfo(const expressions::AttributeReferencePtr &probe_attribute, + const expressions::AttributeReferencePtr &build_attribute, + const PhysicalPtr &builder) + : probe_attribute_(probe_attribute), + build_attribute_(build_attribute), + builder_(builder) {} + + const expressions::AttributeReferencePtr probe_attribute_; + const expressions::AttributeReferencePtr build_attribute_; + const PhysicalPtr builder_; - const expressions::AttributeReferencePtr probe_attribute; - const expressions::AttributeReferencePtr build_attribute; - const PhysicalPtr builder; + DISALLOW_COPY_AND_ASSIGN(LIPFilterProbeInfo); }; @@ -129,34 +266,23 @@ class LIPFilterConfiguration { /** * @brief Add information for a LIP filter builder. * - * @param build_attribute The attribute to build the LIP filter with. + * @param build_info A shared_ptr to LIPFilterBuildInfo. * @param builder The physical node to attach the LIP filter builder to. - * @param filter_size The LIP filter's cardinality. - * @param filter_type The LIP filter's type. */ - void addBuildInfo(const expressions::AttributeReferencePtr &build_attribute, - const PhysicalPtr &builder, - const std::size_t filter_size, - const LIPFilterType &filter_type, - const bool is_anti_filter) { - build_info_map_[builder].emplace_back( - build_attribute, filter_size, filter_type, is_anti_filter); + void addBuildInfo(const LIPFilterBuildInfoPtr &build_info, + const PhysicalPtr &builder) { + build_info_map_[builder].emplace_back(build_info); } /** * @brief Add information for a LIP filter prober. * - * @param probe_attribute The attribute to probe the LIP filter with. + * @param probe_info A shared_ptr to LIPFilterProbeInfo. * @param prober The physical node to attach the LIP filter prober to. - * @param build_attribute The attribute that the LIP filter is built with. - * @param builder The physical node that the LIP filter's builder is attached to. */ - void addProbeInfo(const expressions::AttributeReferencePtr &probe_attribute, - const PhysicalPtr &prober, - const expressions::AttributeReferencePtr &build_attribute, - const PhysicalPtr &builder) { - probe_info_map_[prober].emplace_back( - probe_attribute, build_attribute, builder); + void addProbeInfo(const LIPFilterProbeInfoPtr &probe_info, + const PhysicalPtr &prober) { + probe_info_map_[prober].emplace_back(probe_info); } /** @@ -166,7 +292,7 @@ class LIPFilterConfiguration { * a vector of all the LIP filter builders that are attached to the * physical node. */ - const std::map>& getBuildInfoMap() const { + const std::map>& getBuildInfoMap() const { return build_info_map_; } @@ -177,7 +303,7 @@ class LIPFilterConfiguration { * a vector of all the LIP filter probers that are attached to the * physical node. */ - const std::map>& getProbeInfoMap() const { + const std::map>& getProbeInfoMap() const { return probe_info_map_; } @@ -189,26 +315,14 @@ class LIPFilterConfiguration { */ LIPFilterConfiguration* clone() const { LIPFilterConfiguration *new_conf = new LIPFilterConfiguration(); - for (const auto &build_pair : build_info_map_) { - auto &new_build_vec = new_conf->build_info_map_[build_pair.first]; - const auto &build_vec = build_pair.second; - for (const auto &info : build_vec) { - new_build_vec.emplace_back(info); - } - } - for (const auto &probe_pair : probe_info_map_) { - auto &new_probe_vec = new_conf->probe_info_map_[probe_pair.first]; - const auto &probe_vec = probe_pair.second; - for (const auto &info : probe_vec) { - new_probe_vec.emplace_back(info); - } - } + new_conf->build_info_map_ = build_info_map_; + new_conf->probe_info_map_ = probe_info_map_; return new_conf; } private: - std::map> build_info_map_; - std::map> probe_info_map_; + std::map> build_info_map_; + std::map> probe_info_map_; DISALLOW_COPY_AND_ASSIGN(LIPFilterConfiguration); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/query_optimizer/rules/AttachLIPFilters.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/AttachLIPFilters.cpp b/query_optimizer/rules/AttachLIPFilters.cpp index c4551bc..48b68bc 100644 --- a/query_optimizer/rules/AttachLIPFilters.cpp +++ b/query_optimizer/rules/AttachLIPFilters.cpp @@ -126,16 +126,16 @@ void AttachLIPFilters::attachLIPFilters( if (already_filtered_attributes->find(source_attr_id) == already_filtered_attributes->end()) { lip_filter_configuration_->addBuildInfo( - pair.second->source_attribute, - pair.second->source, - pair.second->estimated_cardinality * 8, - LIPFilterType::kSingleIdentityHashFilter, - false); - lip_filter_configuration_->addProbeInfo( - pair.first, - node, - pair.second->source_attribute, + P::SingleIdentityHashFilterBuildInfo::Create( + pair.second->source_attribute, + pair.second->estimated_cardinality * 8), pair.second->source); + lip_filter_configuration_->addProbeInfo( + P::LIPFilterProbeInfo::Create( + pair.first, + pair.second->source_attribute, + pair.second->source), + node); already_filtered_attributes->emplace(source_attr_id); } } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/query_optimizer/rules/InjectJoinFilters.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/InjectJoinFilters.cpp b/query_optimizer/rules/InjectJoinFilters.cpp index f77d532..92600f6 100644 --- a/query_optimizer/rules/InjectJoinFilters.cpp +++ b/query_optimizer/rules/InjectJoinFilters.cpp @@ -134,7 +134,9 @@ bool InjectJoinFilters::isTransformable( // TODO(jianqiao): implement SimpleHashSetExactFilter to relax this requirement. // Note that 1G bits = 128MB. - if (min_cpp_value < 0 || max_cpp_value > kMaxFilterSize) { + const std::int64_t value_range = max_cpp_value - min_cpp_value; + DCHECK_GE(value_range, 0); + if (value_range > kMaxFilterSize) { return false; } @@ -370,22 +372,23 @@ void InjectJoinFilters::concretizeAsLIPFilters( &min_cpp_value, &max_cpp_value); DCHECK(has_exact_min_max_stats); - DCHECK_GE(min_cpp_value, 0); - DCHECK_GE(max_cpp_value, 0); - DCHECK_LE(max_cpp_value, kMaxFilterSize); + DCHECK_GE(max_cpp_value, min_cpp_value); + DCHECK_LE(max_cpp_value - min_cpp_value, kMaxFilterSize); CHECK(anchor_node != nullptr); + + lip_filter_configuration_->addBuildInfo( - build_attr, - filter_join, - static_cast(max_cpp_value), - LIPFilterType::kBitVectorExactFilter, - filter_join->is_anti_join()); - lip_filter_configuration_->addProbeInfo( - filter_join->probe_attributes().front(), - anchor_node, - build_attr, + P::BitVectorExactFilterBuildInfo::Create(build_attr, + min_cpp_value, + max_cpp_value, + filter_join->is_anti_join()), filter_join); + lip_filter_configuration_->addProbeInfo( + P::LIPFilterProbeInfo::Create(filter_join->probe_attributes().front(), + build_attr, + filter_join), + anchor_node); concretizeAsLIPFilters(filter_join->left(), anchor_node); concretizeAsLIPFilters(filter_join->right(), filter_join); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/utility/PlanVisualizer.cpp ---------------------------------------------------------------------- diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp index e533e15..7ea8d9e 100644 --- a/utility/PlanVisualizer.cpp +++ b/utility/PlanVisualizer.cpp @@ -195,7 +195,7 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) { if (build_it != build_filters.end()) { for (const auto &build_info : build_it->second) { node_info.labels.emplace_back( - std::string("[LIP build] ") + build_info.build_attribute->attribute_alias()); + std::string("[LIP build] ") + build_info->build_attribute()->attribute_alias()); } } const auto &probe_filters = lip_filter_conf_->getProbeInfoMap(); @@ -203,7 +203,7 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) { if (probe_it != probe_filters.end()) { for (const auto &probe_info : probe_it->second) { node_info.labels.emplace_back( - std::string("[LIP probe] ") + probe_info.probe_attribute->attribute_alias()); + std::string("[LIP probe] ") + probe_info->probe_attribute()->attribute_alias()); } } } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/utility/lip_filter/BitVectorExactFilter.hpp ---------------------------------------------------------------------- diff --git a/utility/lip_filter/BitVectorExactFilter.hpp b/utility/lip_filter/BitVectorExactFilter.hpp index 0964a8e..2751b3f 100644 --- a/utility/lip_filter/BitVectorExactFilter.hpp +++ b/utility/lip_filter/BitVectorExactFilter.hpp @@ -21,7 +21,6 @@ #define QUICKSTEP_UTILITY_LIP_FILTER_BIT_VECTOR_EXACT_FILTER_HPP_ #include -#include #include #include #include @@ -57,16 +56,22 @@ class BitVectorExactFilter : public LIPFilter { /** * @brief Constructor. * - * @param filter_cardinality The cardinality of this bit vector. + * @param min_value The minimum possible value for this filter to set. + * @param max_value The maximum possible value for this filter to set. */ - explicit BitVectorExactFilter(const std::size_t filter_cardinality) + explicit BitVectorExactFilter(const std::int64_t min_value, + const std::int64_t max_value) : LIPFilter(LIPFilterType::kBitVectorExactFilter), - filter_cardinality_(filter_cardinality), - bit_array_(GetByteSize(filter_cardinality)) { - DCHECK_GE(filter_cardinality, 0u); + min_value_(static_cast(min_value)), + max_value_(static_cast(max_value)), + bit_array_(GetByteSize(max_value - min_value + 1)) { + DCHECK_EQ(min_value, static_cast(min_value_)); + DCHECK_EQ(max_value, static_cast(max_value_)); + DCHECK_GE(max_value_, min_value_); + std::memset(bit_array_.data(), 0x0, - sizeof(std::atomic) * GetByteSize(filter_cardinality)); + sizeof(std::atomic) * GetByteSize(max_value - min_value + 1)); } void insertValueAccessor(ValueAccessor *accessor, @@ -155,8 +160,11 @@ class BitVectorExactFilter : public LIPFilter { * @brief Inserts a given value into the exact filter. */ inline void insert(const void *key_begin) { - const CppType loc = *reinterpret_cast(key_begin); - DCHECK_LE(loc, filter_cardinality_); + const CppType value = *reinterpret_cast(key_begin); + DCHECK_GE(value, min_value_); + DCHECK_LE(value, max_value_); + + const CppType loc = value - min_value_; bit_array_[loc >> 3u].fetch_or(1u << (loc & 7u), std::memory_order_relaxed); } @@ -164,18 +172,21 @@ class BitVectorExactFilter : public LIPFilter { * @brief Test membership of a given value in the exact filter. */ inline bool contains(const void *key_begin) const { - const CppType loc = *reinterpret_cast(key_begin); - DCHECK_LE(loc, filter_cardinality_); - const bool is_bit_set = - (bit_array_[loc >> 3u].load(std::memory_order_relaxed) & (1u << (loc & 7u))) != 0; + const CppType value = *reinterpret_cast(key_begin); + if (value < min_value_ || value > max_value_) { + return is_anti_filter; + } + + const CppType loc = value - min_value_; if (is_anti_filter) { - return !is_bit_set; + return (bit_array_[loc >> 3u].load(std::memory_order_relaxed) & (1u << (loc & 7u))) == 0; } else { - return is_bit_set; + return (bit_array_[loc >> 3u].load(std::memory_order_relaxed) & (1u << (loc & 7u))) != 0; } } - std::size_t filter_cardinality_; + const CppType min_value_; + const CppType max_value_; alignas(kCacheLineBytes) std::vector> bit_array_; DISALLOW_COPY_AND_ASSIGN(BitVectorExactFilter); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/utility/lip_filter/LIPFilter.proto ---------------------------------------------------------------------- diff --git a/utility/lip_filter/LIPFilter.proto b/utility/lip_filter/LIPFilter.proto index fcbaa7b..45843f3 100644 --- a/utility/lip_filter/LIPFilter.proto +++ b/utility/lip_filter/LIPFilter.proto @@ -36,9 +36,10 @@ message LIPFilter { message BitVectorExactFilter { extend LIPFilter { // All required - optional uint64 filter_cardinality = 16; - optional uint64 attribute_size = 17; - optional bool is_anti_filter = 18; + optional int64 min_value = 16; + optional int64 max_value = 17; + optional uint64 attribute_size = 18; + optional bool is_anti_filter = 19; } } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/84e95cb2/utility/lip_filter/LIPFilterFactory.cpp ---------------------------------------------------------------------- diff --git a/utility/lip_filter/LIPFilterFactory.cpp b/utility/lip_filter/LIPFilterFactory.cpp index a96151f..1de3d4f 100644 --- a/utility/lip_filter/LIPFilterFactory.cpp +++ b/utility/lip_filter/LIPFilterFactory.cpp @@ -35,35 +35,37 @@ LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter case serialization::LIPFilterType::BIT_VECTOR_EXACT_FILTER: { const std::size_t attr_size = proto.GetExtension(serialization::BitVectorExactFilter::attribute_size); - const std::size_t filter_cardinality = - proto.GetExtension(serialization::BitVectorExactFilter::filter_cardinality); + const std::int64_t min_value = + proto.GetExtension(serialization::BitVectorExactFilter::min_value); + const std::int64_t max_value = + proto.GetExtension(serialization::BitVectorExactFilter::max_value); const bool is_anti_filter = proto.GetExtension(serialization::BitVectorExactFilter::is_anti_filter); switch (attr_size) { case 1: if (is_anti_filter) { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } else { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } case 2: if (is_anti_filter) { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } else { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } case 4: if (is_anti_filter) { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } else { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } case 8: if (is_anti_filter) { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } else { - return new BitVectorExactFilter(filter_cardinality); + return new BitVectorExactFilter(min_value, max_value); } default: LOG(FATAL) << "Invalid attribute size for BitVectorExactFilter: " @@ -99,9 +101,11 @@ bool LIPFilterFactory::ProtoIsValid(const serialization::LIPFilter &proto) { case serialization::LIPFilterType::BIT_VECTOR_EXACT_FILTER: { const std::size_t attr_size = proto.GetExtension(serialization::BitVectorExactFilter::attribute_size); - const std::size_t filter_cardinality = - proto.GetExtension(serialization::BitVectorExactFilter::filter_cardinality); - return (attr_size != 0 && filter_cardinality != 0); + const std::int64_t min_value = + proto.GetExtension(serialization::BitVectorExactFilter::min_value); + const std::int64_t max_value = + proto.GetExtension(serialization::BitVectorExactFilter::max_value); + return (attr_size != 0 && max_value >= min_value); } case serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER: { const std::size_t attr_size =