[2/3] incubator-quickstep git commit: Updates for grail

2018-05-16 Thread jianqiao 
Updates for grail


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a0024a2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a0024a2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a0024a2e

Branch: refs/heads/trace
Commit: a0024a2e9ec76d14013fcdee0e8d714d8776c1a3
Parents: 56e0a4e
Author: Jianqiao Zhu 
Authored: Tue May 8 17:53:57 2018 -0500
Committer: Jianqiao Zhu 
Committed: Tue May 8 17:53:57 2018 -0500

--
 .../aggregation/AggregationHandleMax.hpp|  2 +
 .../aggregation/AggregationHandleMin.hpp|  2 +
 query_execution/WorkOrdersContainer.hpp |  8 ++-
 query_optimizer/ExecutionGenerator.cpp  |  6 +--
 .../cost_model/StarSchemaSimpleCostModel.cpp|  6 +--
 relational_operators/UnionAllOperator.cpp   |  6 ++-
 storage/AggregationOperationState.cpp   |  2 +-
 storage/PackedPayloadHashTable.cpp  |  8 +--
 storage/StorageManager.cpp  | 42 ++--
 utility/ShardedLockManager.hpp  | 52 +---
 10 files changed, 29 insertions(+), 105 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/expressions/aggregation/AggregationHandleMax.hpp
--
diff --git a/expressions/aggregation/AggregationHandleMax.hpp 
b/expressions/aggregation/AggregationHandleMax.hpp
index 8f8c0d8..d6fdd12 100644
--- a/expressions/aggregation/AggregationHandleMax.hpp
+++ b/expressions/aggregation/AggregationHandleMax.hpp
@@ -191,6 +191,7 @@ class AggregationHandleMax : public 
AggregationConcreteHandle {
 if (state->max_.isNull() ||
 fast_comparator_->compareTypedValues(value, state->max_)) {
   state->max_ = value;
+  state->max_.ensureNotReference();
 }
   }
 
@@ -200,6 +201,7 @@ class AggregationHandleMax : public 
AggregationConcreteHandle {
 if (max_ptr->isNull() ||
 fast_comparator_->compareTypedValues(value, *max_ptr)) {
   *max_ptr = value;
+  max_ptr->ensureNotReference();
 }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/expressions/aggregation/AggregationHandleMin.hpp
--
diff --git a/expressions/aggregation/AggregationHandleMin.hpp 
b/expressions/aggregation/AggregationHandleMin.hpp
index 0e62be5..441af52 100644
--- a/expressions/aggregation/AggregationHandleMin.hpp
+++ b/expressions/aggregation/AggregationHandleMin.hpp
@@ -191,6 +191,7 @@ class AggregationHandleMin : public 
AggregationConcreteHandle {
 if (state->min_.isNull() ||
 fast_comparator_->compareTypedValues(value, state->min_)) {
   state->min_ = value;
+  state->min_.ensureNotReference();
 }
   }
 
@@ -200,6 +201,7 @@ class AggregationHandleMin : public 
AggregationConcreteHandle {
 if (min_ptr->isNull() ||
 fast_comparator_->compareTypedValues(value, *min_ptr)) {
   *min_ptr = value;
+  min_ptr->ensureNotReference();
 }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/query_execution/WorkOrdersContainer.hpp
--
diff --git a/query_execution/WorkOrdersContainer.hpp 
b/query_execution/WorkOrdersContainer.hpp
index 431a270..55ec185 100644
--- a/query_execution/WorkOrdersContainer.hpp
+++ b/query_execution/WorkOrdersContainer.hpp
@@ -362,7 +362,13 @@ class WorkOrdersContainer {
 return nullptr;
   }
 
-  std::cerr << "# work orders: " << workorders_.size() << "\r";
+  /*
+  const std::size_t num_remaining_workorders = workorders_.size();
+  if (num_remaining_workorders % 1000 == 0) {
+std::cerr << "# work orders: " << num_remaining_workorders
+  << "\r";
+  }
+  */
 
   WorkOrder *work_order = workorders_.front().release();
   workorders_.pop();

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 8ca367b..75f15ac 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -190,7 +190,7 @@ static const volatile bool num_aggregation_partitions_dummy
 = gflags::RegisterFlagValidator(_num_aggregation_partitions, 
);
 
 DEFINE_uint64(partition_aggregation_num_groups_threshold,
-  10,
+  1,
   "The threshold used for deciding whether the aggregation is done

[2/3] incubator-quickstep git commit: Updates

2017-02-03 Thread jianqiao 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e7a92a9/storage/CollisionFreeVectorTable.hpp
--
diff --git a/storage/CollisionFreeVectorTable.hpp 
b/storage/CollisionFreeVectorTable.hpp
new file mode 100644
index 000..25f7786
--- /dev/null
+++ b/storage/CollisionFreeVectorTable.hpp
@@ -0,0 +1,621 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_COLLISION_FREE_AGGREGATION_STATE_HASH_TABLE_HPP_
+#define QUICKSTEP_STORAGE_COLLISION_FREE_AGGREGATION_STATE_HASH_TABLE_HPP_
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/aggregation/AggregationHandle.hpp"
+#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "storage/ValueAccessorMultiplexer.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
+#include "utility/BarrieredReadWriteConcurrentBitVector.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class ColumnVectorsValueAccessor;
+class StorageMnager;
+
+/** \addtogroup Storage
+ *  @{
+ */
+
+class CollisionFreeVectorTable : public AggregationStateHashTableBase {
+ public:
+  CollisionFreeVectorTable(
+  const std::vector _types,
+  const std::size_t num_entries,
+  const std::vector ,
+  StorageManager *storage_manager);
+
+  ~CollisionFreeVectorTable() override;
+
+  void destroyPayload() override;
+
+  inline std::size_t getNumInitializationPartitions() const {
+return num_init_partitions_;
+  }
+
+  inline std::size_t getNumFinalizationPartitions() const {
+return num_finalize_partitions_;
+  }
+
+  inline std::size_t getNumTuplesInPartition(
+  const std::size_t partition_id) const {
+const std::size_t start_position =
+calculatePartitionStartPosition(partition_id);
+const std::size_t end_position =
+calculatePartitionEndPosition(partition_id);
+return existence_map_->onesCountInRange(start_position, end_position);
+  }
+
+  inline void initialize(const std::size_t partition_id) {
+const std::size_t memory_segment_size =
+(memory_size_ + num_init_partitions_ - 1) / num_init_partitions_;
+const std::size_t memory_start = memory_segment_size * partition_id;
+std::memset(reinterpret_cast(blob_->getMemoryMutable()) + 
memory_start,
+0,
+std::min(memory_segment_size, memory_size_ - memory_start));
+  }
+
+  bool upsertValueAccessor(
+  const std::vector _ids,
+  const std::vector _ids,
+  ValueAccessorMultiplexer *accessor_mux) override;
+
+  void finalizeKey(const std::size_t partition_id,
+   NativeColumnVector *output_cv) const;
+
+  void finalizeState(const std::size_t partition_id,
+ std::size_t handle_id,
+ NativeColumnVector *output_cv) const;
+
+ private:
+  inline static std::size_t CacheLineAlignedBytes(const std::size_t 
actual_bytes) {
+return (actual_bytes + kCacheLineBytes - 1) / kCacheLineBytes * 
kCacheLineBytes;
+  }
+
+  inline std::size_t calculatePartitionLength() const {
+const std::size_t partition_length =
+(num_entries_ + num_finalize_partitions_ - 1) / 
num_finalize_partitions_;
+DCHECK_GE(partition_length, 0u);
+return partition_length;
+  }
+
+  inline std::size_t calculatePartitionStartPosition(
+  const std::size_t partition_id) const {
+return calculatePartitionLength() * partition_id;
+  }
+
+  inline std::size_t calculatePartitionEndPosition(
+  const std::size_t partition_id) const {
+return std::min(calculatePartitionLength() * (partition_id + 1),
+num_entries_);
+  }
+
+  template 
+  inline void upsertValueAccessorDispatchHelper(
+  const bool is_key_nullable,
+  const bool is_argument_nullable,
+  ArgTypes &&...args);
+
+  template 
+  inline

[2/3] incubator-quickstep git commit: Updates

2016-07-29 Thread jianqiao 
Updates


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/91e49820
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/91e49820
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/91e49820

Branch: refs/heads/adaptive-bloom-filters
Commit: 91e498202d7b19cb5c7d4d8c61218d112c446b71
Parents: 9cc47e5
Author: Jianqiao Zhu 
Authored: Fri Jul 29 17:47:42 2016 -0500
Committer: Jianqiao Zhu 
Committed: Fri Jul 29 17:47:42 2016 -0500

--
 query_execution/QueryContext.cpp| 11 ++-
 query_optimizer/ExecutionGenerator.cpp  | 17 
 query_optimizer/ExecutionHeuristics.cpp | 36 +--
 query_optimizer/ExecutionHeuristics.hpp | 51 +++---
 query_optimizer/physical/Aggregate.hpp  | 20 +++-
 query_optimizer/physical/HashJoin.hpp   | 50 --
 query_optimizer/physical/Physical.hpp   | 50 ++
 query_optimizer/rules/AttachBloomFilters.cpp| 76 ++-
 .../StarSchemaHashJoinOrderOptimization.cpp |  4 +-
 .../StarSchemaHashJoinOrderOptimization.hpp | 25 +++--
 storage/AggregationOperationState.cpp   | 98 +++-
 storage/AggregationOperationState.hpp   | 10 +-
 storage/AggregationOperationState.proto |  6 ++
 storage/HashTable.hpp   | 27 ++
 storage/HashTable.proto |  6 +-
 storage/HashTableFactory.hpp|  9 +-
 storage/StorageBlock.cpp| 28 +-
 storage/StorageBlock.hpp|  7 +-
 utility/PlanVisualizer.cpp  | 24 +
 19 files changed, 377 insertions(+), 178 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/91e49820/query_execution/QueryContext.cpp
--
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 7019b6a..fd0ed08 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -61,15 +61,16 @@ QueryContext::QueryContext(const 
serialization::QueryContext ,
   << "Attempted to create QueryContext from an invalid proto 
description:\n"
   << proto.DebugString();
 
+  for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+bloom_filters_.emplace_back(new BloomFilter(proto.bloom_filters(i)));
+  }
+
   for (int i = 0; i < proto.aggregation_states_size(); ++i) {
 aggregation_states_.emplace_back(
 
AggregationOperationState::ReconstructFromProto(proto.aggregation_states(i),
 database,
-storage_manager));
-  }
-
-  for (int i = 0; i < proto.bloom_filters_size(); ++i) {
-bloom_filters_.emplace_back(new BloomFilter(proto.bloom_filters(i)));
+storage_manager,
+bloom_filters_));
   }
 
   for (int i = 0; i < proto.generator_functions_size(); ++i) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/91e49820/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index fe6b6e7..e10f991 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1344,6 +1344,16 @@ void ExecutionGenerator::convertAggregate(
   findRelationInfoOutputByPhysical(physical_plan->input());
   aggr_state_proto->set_relation_id(input_relation_info->relation->getID());
 
+  const P::BloomFilterConfig _filter_config =
+  physical_plan->bloom_filter_config();
+  std::vector bloom_filter_attribute_ids;
+
+  for (const auto  : bloom_filter_config.probe_side_bloom_filters) {
+const CatalogAttribute *bf_catalog_attribute
+= attribute_substitution_map_[bf.attribute->id()];
+bloom_filter_attribute_ids.emplace_back(bf_catalog_attribute->getID());
+  }
+
   std::vector group_by_types;
   for (const E::NamedExpressionPtr _expression : 
physical_plan->grouping_expressions()) {
 unique_ptr execution_group_by_expression;
@@ -1458,6 +1468,13 @@ void ExecutionGenerator::convertAggregate(
   std::forward_as_tuple(finalize_aggregation_operator_index, 
output_relation));
   
temporary_relation_info_vec_.emplace_back(finalize_aggregation_operator_index,
 output_relation);
+
+  if (FLAGS_optimize_joins) {
+execution_heuristics_->addAggregateInfo(aggregation_operator_index,
+