Repository: incubator-quickstep Updated Branches: refs/heads/LIP-for-tpch-merged 3d8b738ea -> 11b01099e
Partial work, try 02 Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a9cfdd1a Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a9cfdd1a Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a9cfdd1a Branch: refs/heads/LIP-for-tpch-merged Commit: a9cfdd1ab962680108a90a5a4a9198ae99967eb5 Parents: 3d8b738 Author: Jianqiao Zhu <jianq...@cs.wisc.edu> Authored: Tue Aug 23 15:11:52 2016 -0500 Committer: Jianqiao Zhu <jianq...@cs.wisc.edu> Committed: Tue Aug 23 15:11:52 2016 -0500 ---------------------------------------------------------------------- relational_operators/HashJoinOperator.cpp | 47 +++++++++++++------- storage/CountedReference.hpp | 4 +- storage/EvictionPolicy.cpp | 17 +++---- types/containers/ColumnVector.hpp | 43 +++++++++++++++--- types/containers/ColumnVectorsValueAccessor.hpp | 25 ++++++----- utility/BitVector.hpp | 31 ++++++++++++- 6 files changed, 124 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a9cfdd1a/relational_operators/HashJoinOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp index 70fbb1a..a45eb24 100644 --- a/relational_operators/HashJoinOperator.cpp +++ b/relational_operators/HashJoinOperator.cpp @@ -471,6 +471,7 @@ void HashInnerJoinWorkOrder::execute() { // materialize_line->emplace_back(); // iterate_line->emplace_back(); +// std::cout << "here!\n"; ColumnVectorsValueAccessor temp_result; for (std::pair<const block_id, std::vector<std::pair<tuple_id, tuple_id>>> &build_block_entry : *collector.getJoinedTuples()) { @@ -529,18 +530,34 @@ void HashInnerJoinWorkOrder::execute() { // benefit (probably only a real performance win when there are very few // matching tuples in each individual inner block but very many inner // blocks with at least one match). - std::size_t i = 0; + std::vector<std::unique_ptr<ColumnVector>> columns; for (vector<unique_ptr<const Scalar>>::const_iterator selection_cit = selection_.begin(); selection_cit != selection_.end(); - ++selection_cit, ++i) { - temp_result.appendColumn((*selection_cit)->getAllValuesForJoin(build_relation_id, - build_accessor.get(), - probe_relation_id, - probe_accessor.get(), - build_block_entry.second), - i); + ++selection_cit) { + columns.emplace_back( + std::unique_ptr<ColumnVector>( + (*selection_cit)->getAllValuesForJoin(build_relation_id, + build_accessor.get(), + probe_relation_id, + probe_accessor.get(), + build_block_entry.second))); } + temp_result.appendColumns(&columns, build_block_entry.second.size()); + +// ColumnVectorsValueAccessor temp_result; +// for (vector<unique_ptr<const Scalar>>::const_iterator selection_cit = selection_.begin(); +// selection_cit != selection_.end(); +// ++selection_cit) { +// temp_result.addColumn( +// (*selection_cit)->getAllValuesForJoin(build_relation_id, +// build_accessor.get(), +// probe_relation_id, +// probe_accessor.get(), +// build_block_entry.second)); +// } +// output_destination_->bulkInsertTuples(&temp_result); + // iterate_line->emplace_back(); } // iterate_line->back().endEvent(); @@ -551,13 +568,13 @@ void HashInnerJoinWorkOrder::execute() { // materialize_line->back().endEvent(); // materialize_line->back().setPayload(getOperatorIndex(), collector.getJoinedTuples()->size()); - if (build_relation_id == 0 && - probe_relation_id == 0 && - residual_predicate_ == nullptr && - output_destination_ == nullptr && - selection_.empty()) { - return; - } +// if (build_relation_id == 0 && +// probe_relation_id == 0 && +// residual_predicate_ == nullptr && +// output_destination_ == nullptr && +// selection_.empty()) { +// return; +// } } void HashSemiJoinWorkOrder::execute() { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a9cfdd1a/storage/CountedReference.hpp ---------------------------------------------------------------------- diff --git a/storage/CountedReference.hpp b/storage/CountedReference.hpp index 49d0f73..2d9cec3 100644 --- a/storage/CountedReference.hpp +++ b/storage/CountedReference.hpp @@ -66,7 +66,7 @@ class CountedReference { **/ CountedReference(T *block, EvictionPolicy *eviction_policy) : block_(block), eviction_policy_(eviction_policy) { -// eviction_policy_->blockReferenced(block_->getID()); + eviction_policy_->blockReferenced(block_->getID()); #ifdef QUICKSTEP_DEBUG block_->ref(); #endif @@ -111,7 +111,7 @@ class CountedReference { #ifdef QUICKSTEP_DEBUG block_->unref(); #endif -// eviction_policy_->blockUnreferenced(block_->getID()); + eviction_policy_->blockUnreferenced(block_->getID()); } } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a9cfdd1a/storage/EvictionPolicy.cpp ---------------------------------------------------------------------- diff --git a/storage/EvictionPolicy.cpp b/storage/EvictionPolicy.cpp index 2f10f09..7af7f93 100644 --- a/storage/EvictionPolicy.cpp +++ b/storage/EvictionPolicy.cpp @@ -44,14 +44,15 @@ using std::chrono::microseconds; namespace quickstep { EvictionPolicy::Status EvictAnyBlockEvictionPolicy::chooseBlockToEvict(block_id* block) { - MutexLock lock(mutex_); - if (nonreferenced_blocks_.empty()) { - return Status::kBlockNotFound; - } - std::unordered_set<block_id>::const_iterator it = nonreferenced_blocks_.begin(); - *block = *it; - ref_counts_.erase(*it); - nonreferenced_blocks_.erase(it); +// MutexLock lock(mutex_); +// if (nonreferenced_blocks_.empty()) { +// return Status::kBlockNotFound; +// } +// std::unordered_set<block_id>::const_iterator it = nonreferenced_blocks_.begin(); +// *block = *it; +// ref_counts_.erase(*it); +// nonreferenced_blocks_.erase(it); + LOG(FATAL) << "No eviction allowed"; return Status::kOk; } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a9cfdd1a/types/containers/ColumnVector.hpp ---------------------------------------------------------------------- diff --git a/types/containers/ColumnVector.hpp b/types/containers/ColumnVector.hpp index b95ae98..c1cbff6 100644 --- a/types/containers/ColumnVector.hpp +++ b/types/containers/ColumnVector.hpp @@ -107,7 +107,7 @@ class ColumnVector { **/ virtual bool isNative() const = 0; - virtual bool append(ColumnVector *column_vector) = 0; + virtual bool append(const ColumnVector *column_vector) = 0; protected: const Type &type_; @@ -401,12 +401,13 @@ class NativeColumnVector : public ColumnVector { } } - bool append(ColumnVector *column_vector) override { + bool append(const ColumnVector *column_vector) override { // Other ColumnVector also has to be native. if (!column_vector->isNative()) { return false; } - NativeColumnVector *casted_column_vector = static_cast<NativeColumnVector*>(column_vector); + const NativeColumnVector *casted_column_vector = + static_cast<const NativeColumnVector*>(column_vector); // Both ColumnVectors has to have same type to be appended. if (!type_.equals(casted_column_vector->type_) || type_length_ != casted_column_vector->type_length_) { @@ -421,7 +422,9 @@ class NativeColumnVector : public ColumnVector { new_reserved_length = reserved_length_; } - void *new_buffer = std::realloc(values_, new_reserved_length); + void *new_buffer = std::realloc(values_, + type_length_ * new_reserved_length); + if (new_buffer == nullptr) { return false; } @@ -433,6 +436,11 @@ class NativeColumnVector : public ColumnVector { reserved_length_ = new_reserved_length; actual_length_ = new_actual_length; + + if (null_bitmap_) { + return null_bitmap_->append((casted_column_vector->null_bitmap_).get()); + } + return true; } @@ -628,13 +636,36 @@ class IndirectColumnVector : public ColumnVector { values_[position] = std::move(value); } - bool append(ColumnVector *column_vector) override { + bool append(const ColumnVector *column_vector) override { + if (column_vector->isNative()) { + return false; + } + const IndirectColumnVector *casted_column_vector = + static_cast<const IndirectColumnVector*>(column_vector); + // Both ColumnVectors has to have same type to be appended. + if (!type_.equals(casted_column_vector->type_) + || type_is_nullable_ != casted_column_vector->type_is_nullable_) { + return false; + } + + std::size_t new_actual_length = values_.size() + casted_column_vector->values_.size(); + std::size_t new_reserved_length + = (new_actual_length > reserved_length_) + ? (new_actual_length * 2) + : (reserved_length_); + + values_.reserve(new_reserved_length); + values_.insert(values_.end(), + casted_column_vector->values_.begin(), + casted_column_vector->values_.end()); + reserved_length_ = new_reserved_length; + return true; } private: const bool type_is_nullable_; - const std::size_t reserved_length_; + std::size_t reserved_length_; std::vector<TypedValue> values_; DISALLOW_COPY_AND_ASSIGN(IndirectColumnVector); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a9cfdd1a/types/containers/ColumnVectorsValueAccessor.hpp ---------------------------------------------------------------------- diff --git a/types/containers/ColumnVectorsValueAccessor.hpp b/types/containers/ColumnVectorsValueAccessor.hpp index 05ec7ee..ed2ee76 100644 --- a/types/containers/ColumnVectorsValueAccessor.hpp +++ b/types/containers/ColumnVectorsValueAccessor.hpp @@ -22,6 +22,7 @@ #include <cstddef> #include <limits> +#include <memory> #include <utility> #include <vector> @@ -92,20 +93,24 @@ class ColumnVectorsValueAccessor : public ValueAccessor { : static_cast<const IndirectColumnVector*>(column)->size(); } - void appendColumn(ColumnVector *column, const std::size_t index, const bool owns = true) { - if (index >= columns_.size()) { - addColumn(column, owns); + void appendColumns(std::vector<std::unique_ptr<ColumnVector>> *columns, + const std::size_t length) { + if (columns_.empty()) { + for (auto &column : *columns) { + addColumn(column.release(), true); + } } else { - ColumnVector *old_column = columns_[index]; - old_column->append(column); - const int appended_column_length - = column->isNative() - ? static_cast<const NativeColumnVector*>(column)->size() - : static_cast<const IndirectColumnVector*>(column)->size(); - column_length_ += appended_column_length; + for (std::size_t i = 0; i < columns_.size(); ++i) { + columns_[i]->append(columns->at(i).get()); + } + column_length_ += length; } } + void increaseColumnLength(const std::size_t delta_length) { + column_length_ += delta_length; + } + inline void beginIteration() { current_position_ = std::numeric_limits<std::size_t>::max(); } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a9cfdd1a/utility/BitVector.hpp ---------------------------------------------------------------------- diff --git a/utility/BitVector.hpp b/utility/BitVector.hpp index bb76315..c47d150 100644 --- a/utility/BitVector.hpp +++ b/utility/BitVector.hpp @@ -829,6 +829,33 @@ class BitVector { return num_bits_; } + bool append(BitVector *other) { + std::size_t total_data_array_size = data_array_size_ + other->data_array_size_; + + std::size_t *new_data_array_ = reinterpret_cast<std::size_t*>(std::realloc(data_array_, total_data_array_size)); + if (new_data_array_ == nullptr) { + return false; + } + // Swap pointers. + std::swap(data_array_, new_data_array_); + + // Copy other BitVector's data. + std::memcpy(reinterpret_cast<std::uint8_t*>(data_array_) + data_array_size_, + other->data_array_, + other->data_array_size_); + + // Not complete (it is working only under vector<std::size_t> impl.) + const std::size_t excess_bits_at_original = num_bits_ % kSizeTBits; + const std::size_t old_num_bits = num_bits_; + // Update private fields to make shiftTailForward work correctly. + data_array_size_ = total_data_array_size; + num_bits_ = num_bits_ + other->num_bits_; + + shiftTailForward(old_num_bits, excess_bits_at_original); + + return true; + } + private: // This works as long as the bit-width of size_t is power of 2: static const std::size_t kLowerOrderMask = (sizeof(std::size_t) << 3) - 1; @@ -1166,8 +1193,8 @@ class BitVector { const bool owned_; const bool short_version_; // Vector is 32 bits or less, so we use not even a full size_t. std::size_t *data_array_; - const std::size_t num_bits_; - const std::size_t data_array_size_; + std::size_t num_bits_; + std::size_t data_array_size_; DISALLOW_COPY_AND_ASSIGN(BitVector); };
