Use BitVector as LIPFilter implementation when applicable
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2b2d7ba1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2b2d7ba1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2b2d7ba1 Branch: refs/heads/reorder-partitioned-hash-join Commit: 2b2d7ba1970ade47b1170cd7974cb2fc53f7ba71 Parents: 1572762 Author: Jianqiao Zhu <jianq...@cs.wisc.edu> Authored: Wed Feb 22 14:06:55 2017 -0600 Committer: Jianqiao Zhu <jianq...@cs.wisc.edu> Committed: Wed Feb 22 14:06:55 2017 -0600 ---------------------------------------------------------------------- query_optimizer/rules/AttachLIPFilters.cpp | 74 +++++++++++++++++++++++-- query_optimizer/rules/AttachLIPFilters.hpp | 9 +++ query_optimizer/rules/CMakeLists.txt | 1 + 3 files changed, 79 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2b2d7ba1/query_optimizer/rules/AttachLIPFilters.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/AttachLIPFilters.cpp b/query_optimizer/rules/AttachLIPFilters.cpp index 4b6ac59..9a13b48 100644 --- a/query_optimizer/rules/AttachLIPFilters.cpp +++ b/query_optimizer/rules/AttachLIPFilters.cpp @@ -20,6 +20,7 @@ #include "query_optimizer/rules/AttachLIPFilters.hpp" #include <algorithm> +#include <cstdint> #include <map> #include <set> #include <unordered_set> @@ -37,6 +38,7 @@ #include "query_optimizer/physical/PhysicalType.hpp" #include "query_optimizer/physical/Selection.hpp" #include "query_optimizer/physical/TopLevelPlan.hpp" +#include "types/TypeID.hpp" #include "types/TypedValue.hpp" #include "utility/lip_filter/LIPFilter.hpp" @@ -126,11 +128,40 @@ void AttachLIPFilters::attachLIPFilters( const E::ExprId source_attr_id = pair.second->source_attribute->id(); if (already_filtered_attributes->find(source_attr_id) == already_filtered_attributes->end()) { - lip_filter_configuration_->addBuildInfo( - P::SingleIdentityHashFilterBuildInfo::Create( - pair.second->source_attribute, - std::max(64uL, pair.second->estimated_cardinality * 8u)), - pair.second->source); + bool use_exact_filter = false; + std::int64_t min_cpp_value; + std::int64_t max_cpp_value; + const bool has_exact_min_max_stats = + findExactMinMaxValuesForAttributeHelper(pair.second->source, + pair.second->source_attribute, + &min_cpp_value, + &max_cpp_value); + if (has_exact_min_max_stats) { + const std::int64_t value_range = max_cpp_value - min_cpp_value; + DCHECK_GE(value_range, 0); + // TODO(jianqiao): Add this threshold as a gflag (together with + // InjectJoinFilters::kMaxFilterSize). + if (value_range <= 1000000000L) { + use_exact_filter = true; + } + } + + if (use_exact_filter) { + lip_filter_configuration_->addBuildInfo( + P::BitVectorExactFilterBuildInfo::Create( + pair.second->source_attribute, + min_cpp_value, + max_cpp_value, + false), + pair.second->source); + } else { + lip_filter_configuration_->addBuildInfo( + P::SingleIdentityHashFilterBuildInfo::Create( + pair.second->source_attribute, + std::max(64uL, pair.second->estimated_cardinality * 8u)), + pair.second->source); + } + lip_filter_configuration_->addProbeInfo( P::LIPFilterProbeInfo::Create( pair.first, @@ -258,5 +289,38 @@ const std::vector<AttachLIPFilters::LIPFilterInfoPtr>& AttachLIPFilters return probe_side_info_.at(node); } +bool AttachLIPFilters::findExactMinMaxValuesForAttributeHelper( + const physical::PhysicalPtr &physical_plan, + const expressions::AttributeReferencePtr &attribute, + std::int64_t *min_cpp_value, + std::int64_t *max_cpp_value) const { + bool min_value_is_exact; + bool max_value_is_exact; + + const TypedValue min_value = + cost_model_->findMinValueStat(physical_plan, attribute, &min_value_is_exact); + const TypedValue max_value = + cost_model_->findMaxValueStat(physical_plan, attribute, &max_value_is_exact); + if (min_value.isNull() || max_value.isNull() || + (!min_value_is_exact) || (!max_value_is_exact)) { + return false; + } + + switch (attribute->getValueType().getTypeID()) { + case TypeID::kInt: { + *min_cpp_value = min_value.getLiteral<int>(); + *max_cpp_value = max_value.getLiteral<int>(); + return true; + } + case TypeID::kLong: { + *min_cpp_value = min_value.getLiteral<std::int64_t>(); + *max_cpp_value = max_value.getLiteral<std::int64_t>(); + return true; + } + default: + return false; + } +} + } // namespace optimizer } // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2b2d7ba1/query_optimizer/rules/AttachLIPFilters.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/AttachLIPFilters.hpp b/query_optimizer/rules/AttachLIPFilters.hpp index b8cfc39..36cb010 100644 --- a/query_optimizer/rules/AttachLIPFilters.hpp +++ b/query_optimizer/rules/AttachLIPFilters.hpp @@ -21,6 +21,7 @@ #define QUICKSTEP_QUERY_OPTIMIZER_RULES_ATTACH_LIP_FILTERS_HPP_ #include <cstddef> +#include <cstdint> #include <map> #include <memory> #include <set> @@ -135,6 +136,14 @@ class AttachLIPFilters : public Rule<physical::Physical> { const std::vector<LIPFilterInfoPtr>& getProbeSideInfo(const NodeList &path); + // TODO(jianqiao): refactor this method as it is a duplication of + // InjectJoinFilters::findExactMinMaxValuesForAttributeHelper(). + bool findExactMinMaxValuesForAttributeHelper( + const physical::PhysicalPtr &physical_plan, + const expressions::AttributeReferencePtr &attribute, + std::int64_t *min_cpp_value, + std::int64_t *max_cpp_value) const; + std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_; std::map<physical::PhysicalPtr, std::vector<LIPFilterInfoPtr>> build_side_info_; std::map<physical::PhysicalPtr, std::vector<LIPFilterInfoPtr>> probe_side_info_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2b2d7ba1/query_optimizer/rules/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt index 427500d..6847951 100644 --- a/query_optimizer/rules/CMakeLists.txt +++ b/query_optimizer/rules/CMakeLists.txt @@ -59,6 +59,7 @@ target_link_libraries(quickstep_queryoptimizer_rules_AttachLIPFilters quickstep_queryoptimizer_physical_Selection quickstep_queryoptimizer_physical_TopLevelPlan quickstep_queryoptimizer_rules_Rule + quickstep_types_TypeID quickstep_types_TypedValue quickstep_utility_Macros quickstep_utility_lipfilter_LIPFilter)