[2/2] incubator-quickstep git commit: Fix the bug with min/max aggregation

2017-08-02 Thread jianqiao 
Fix the bug with min/max aggregation


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/88a1598c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/88a1598c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/88a1598c

Branch: refs/heads/fix-compare-aggregate
Commit: 88a1598c787cfce215e10ba532bc8faf72d70441
Parents: e80e5c8
Author: Jianqiao Zhu 
Authored: Wed Aug 2 15:47:30 2017 -0500
Committer: jianqiao 
Committed: Wed Aug 2 19:54:54 2017 -0500

--
 .../comparisons/LiteralComparators-inl.hpp  | 72 
 1 file changed, 57 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/88a1598c/types/operations/comparisons/LiteralComparators-inl.hpp
--
diff --git a/types/operations/comparisons/LiteralComparators-inl.hpp 
b/types/operations/comparisons/LiteralComparators-inl.hpp
index fd59e2e..b7271ad 100644
--- a/types/operations/comparisons/LiteralComparators-inl.hpp
+++ b/types/operations/comparisons/LiteralComparators-inl.hpp
@@ -543,7 +543,11 @@ TypedValue LiteralUncheckedComparator
   column_accessor(accessor->template 
getColumnAccessor(value_accessor_id));
   DCHECK(column_accessor != nullptr);
-  while (accessor->next()) {
-const void *va_value = column_accessor->getUntypedValue();
-if (left_nullable && !va_value) {
-  continue;
+
+  // Locate the first non-null value.
+  if (is_null) {
+const void *va_value = nullptr;
+while (accessor->next()) {
+  va_value = column_accessor->getUntypedValue();
+  if (!left_nullable || va_value) {
+break;
+  }
+}
+if (va_value != nullptr) {
+  is_null = false;
+  current_literal = *static_cast(va_value);
 }
-if (!current_literal || this->compareDataPtrsHelper(va_value, 
current_literal)) {
-  current_literal = va_value;
+  }
+
+  // Aggregate on the remaining values.
+  if (!accessor->iterationFinished()) {
+while (accessor->next()) {
+  const void *va_value = column_accessor->getUntypedValue();
+  if (left_nullable && !va_value) {
+continue;
+  }
+  if (this->compareDataPtrsHelper(va_value, _literal)) {
+current_literal = *static_cast(va_value);
+  }
 }
   }
 } else {
-  while (accessor->next()) {
-const void *va_value = accessor->template 
getUntypedValue(value_accessor_id);
-if (left_nullable && !va_value) {
-  continue;
+  // Locate the first non-null value.
+  if (is_null) {
+const void *va_value = nullptr;
+while (accessor->next()) {
+  va_value = accessor->template 
getUntypedValue(value_accessor_id);
+  if (!left_nullable || va_value) {
+break;
+  }
 }
-if (!current_literal || this->compareDataPtrsHelper(va_value, 
current_literal)) {
-  current_literal = va_value;
+if (va_value != nullptr) {
+  is_null = false;
+  current_literal = *static_cast(va_value);
+}
+  }
+
+  // Aggregate on the remaining values.
+  if (!accessor->iterationFinished()) {
+while (accessor->next()) {
+  const void *va_value =
+  accessor->template 
getUntypedValue(value_accessor_id);
+  if (left_nullable && !va_value) {
+continue;
+  }
+  if (this->compareDataPtrsHelper(va_value, _literal)) {
+current_literal = *static_cast(va_value);
+  }
 }
   }
 }
   });
 
-  if (current_literal) {
-return TypedValue(*static_cast(current_literal));
+  if (!is_null) {
+return TypedValue(current_literal);
   } else {
 return TypedValue(current.getTypeID());
   }

incubator-quickstep git commit: Disabled NetworkUtil_unittest.

2017-08-02 Thread zuyuz 
Repository: incubator-quickstep
Updated Branches:
  refs/heads/master e80e5c828 -> 9dc9dfbfc


Disabled NetworkUtil_unittest.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9dc9dfbf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9dc9dfbf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9dc9dfbf

Branch: refs/heads/master
Commit: 9dc9dfbfcbb072b2280394d78abc8f2554881d9a
Parents: e80e5c8
Author: Zuyu Zhang 
Authored: Wed Aug 2 19:46:16 2017 -0500
Committer: Zuyu Zhang 
Committed: Wed Aug 2 20:46:21 2017 -0500

--
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9dc9dfbf/.travis.yml
--
diff --git a/.travis.yml b/.travis.yml
index 868ffaf..9820599 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -83,7 +83,7 @@ script:
   - ./validate_cmakelists.py
   - ./cyclic_dependency.py
   - (cd build && make)
-  - (cd build && ctest --output-on-failure -j$TEST_JOBS)
+  - (cd build && ctest --output-on-failure -E NetworkUtil_unittest 
-j$TEST_JOBS)
 
 after_failure:
   - df -h

incubator-quickstep git commit: Determine #InitPartitions for CollisionFreeVectorTable in the optimizer.

2017-08-02 Thread jianqiao 
Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 302f2cb88 -> e80e5c828


Determine #InitPartitions for CollisionFreeVectorTable in the optimizer.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e80e5c82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e80e5c82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e80e5c82

Branch: refs/heads/master
Commit: e80e5c828b317ed64e8b140643c3c601cc4d23c3
Parents: 302f2cb
Author: Zuyu Zhang 
Authored: Fri Jul 21 12:51:35 2017 -0500
Committer: Zuyu Zhang 
Committed: Wed Aug 2 18:27:22 2017 -0500

--
 query_optimizer/CMakeLists.txt  |  3 +
 query_optimizer/ExecutionGenerator.cpp  | 98 ++--
 .../InitializeAggregationOperator.cpp   |  2 +-
 .../InitializeAggregationOperator.hpp   |  8 +-
 storage/AggregationOperationState.cpp   | 62 ++---
 storage/AggregationOperationState.hpp   | 11 ++-
 storage/AggregationOperationState.proto |  3 +
 storage/CMakeLists.txt  |  1 +
 storage/CollisionFreeVectorTable.cpp| 54 ++-
 storage/CollisionFreeVectorTable.hpp| 27 ++
 storage/HashTable.proto |  6 ++
 storage/HashTableFactory.hpp| 15 ++-
 storage/StorageConstants.hpp|  3 +
 13 files changed, 202 insertions(+), 91 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e80e5c82/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 69105e6..fdf8796 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -65,6 +65,7 @@ 
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
   quickstep_expressions_Expressions_proto
   quickstep_expressions_aggregation_AggregateFunction
   quickstep_expressions_aggregation_AggregateFunction_proto
+  quickstep_expressions_aggregation_AggregationID
   quickstep_expressions_predicate_Predicate
   quickstep_expressions_scalar_Scalar
   quickstep_expressions_scalar_ScalarAttribute
@@ -150,12 +151,14 @@ 
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
   quickstep_storage_InsertDestination_proto
   quickstep_storage_StorageBlockLayout
   quickstep_storage_StorageBlockLayout_proto
+  quickstep_storage_StorageConstants
   quickstep_storage_SubBlockTypeRegistry
   quickstep_types_Type
   quickstep_types_Type_proto
   quickstep_types_TypedValue
   quickstep_types_TypedValue_proto
   quickstep_types_containers_Tuple_proto
+  quickstep_utility_BarrieredReadWriteConcurrentBitVector
   quickstep_utility_Macros
   quickstep_utility_SqlError)
 if (ENABLE_DISTRIBUTED)

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e80e5c82/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 88dc505..9bfd136 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -20,6 +20,7 @@
 #include "query_optimizer/ExecutionGenerator.hpp"
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -52,6 +53,7 @@
 #include "expressions/Expressions.pb.h"
 #include "expressions/aggregation/AggregateFunction.hpp"
 #include "expressions/aggregation/AggregateFunction.pb.h"
+#include "expressions/aggregation/AggregationID.hpp"
 #include "expressions/predicate/Predicate.hpp"
 #include "expressions/scalar/Scalar.hpp"
 #include "expressions/scalar/ScalarAttribute.hpp"
@@ -133,19 +135,23 @@
 #include "storage/InsertDestination.pb.h"
 #include "storage/StorageBlockLayout.hpp"
 #include "storage/StorageBlockLayout.pb.h"
+#include "storage/StorageConstants.hpp"
 #include "storage/SubBlockTypeRegistry.hpp"
 #include "types/Type.hpp"
 #include "types/Type.pb.h"
 #include "types/TypedValue.hpp"
 #include "types/TypedValue.pb.h"
 #include "types/containers/Tuple.pb.h"
+#include "utility/BarrieredReadWriteConcurrentBitVector.hpp"
 #include "utility/SqlError.hpp"
 
 #include "gflags/gflags.h"
 #include "glog/logging.h"
 
+using std::atomic;
 using

incubator-quickstep git commit: Fix the bug with min/max aggregation

2017-08-02 Thread jianqiao 
Repository: incubator-quickstep
Updated Branches:
  refs/heads/fix-compare-aggregate [created] 5616a646b


Fix the bug with min/max aggregation


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5616a646
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5616a646
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5616a646

Branch: refs/heads/fix-compare-aggregate
Commit: 5616a646bba0aa189bb7179f260d21431a8d1aa6
Parents: 302f2cb
Author: Jianqiao Zhu 
Authored: Wed Aug 2 15:47:30 2017 -0500
Committer: Jianqiao Zhu 
Committed: Wed Aug 2 15:47:30 2017 -0500

--
 .../comparisons/LiteralComparators-inl.hpp  | 72 
 1 file changed, 57 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5616a646/types/operations/comparisons/LiteralComparators-inl.hpp
--
diff --git a/types/operations/comparisons/LiteralComparators-inl.hpp 
b/types/operations/comparisons/LiteralComparators-inl.hpp
index fd59e2e..b7271ad 100644
--- a/types/operations/comparisons/LiteralComparators-inl.hpp
+++ b/types/operations/comparisons/LiteralComparators-inl.hpp
@@ -543,7 +543,11 @@ TypedValue LiteralUncheckedComparator
   column_accessor(accessor->template 
getColumnAccessor(value_accessor_id));
   DCHECK(column_accessor != nullptr);
-  while (accessor->next()) {
-const void *va_value = column_accessor->getUntypedValue();
-if (left_nullable && !va_value) {
-  continue;
+
+  // Locate the first non-null value.
+  if (is_null) {
+const void *va_value = nullptr;
+while (accessor->next()) {
+  va_value = column_accessor->getUntypedValue();
+  if (!left_nullable || va_value) {
+break;
+  }
+}
+if (va_value != nullptr) {
+  is_null = false;
+  current_literal = *static_cast(va_value);
 }
-if (!current_literal || this->compareDataPtrsHelper(va_value, 
current_literal)) {
-  current_literal = va_value;
+  }
+
+  // Aggregate on the remaining values.
+  if (!accessor->iterationFinished()) {
+while (accessor->next()) {
+  const void *va_value = column_accessor->getUntypedValue();
+  if (left_nullable && !va_value) {
+continue;
+  }
+  if (this->compareDataPtrsHelper(va_value, _literal)) {
+current_literal = *static_cast(va_value);
+  }
 }
   }
 } else {
-  while (accessor->next()) {
-const void *va_value = accessor->template 
getUntypedValue(value_accessor_id);
-if (left_nullable && !va_value) {
-  continue;
+  // Locate the first non-null value.
+  if (is_null) {
+const void *va_value = nullptr;
+while (accessor->next()) {
+  va_value = accessor->template 
getUntypedValue(value_accessor_id);
+  if (!left_nullable || va_value) {
+break;
+  }
 }
-if (!current_literal || this->compareDataPtrsHelper(va_value, 
current_literal)) {
-  current_literal = va_value;
+if (va_value != nullptr) {
+  is_null = false;
+  current_literal = *static_cast(va_value);
+}
+  }
+
+  // Aggregate on the remaining values.
+  if (!accessor->iterationFinished()) {
+while (accessor->next()) {
+  const void *va_value =
+  accessor->template 
getUntypedValue(value_accessor_id);
+  if (left_nullable && !va_value) {
+continue;
+  }
+  if (this->compareDataPtrsHelper(va_value, _literal)) {
+current_literal = *static_cast(va_value);
+  }
 }
   }
 }
   });
 
-  if (current_literal) {
-return TypedValue(*static_cast(current_literal));
+  if (!is_null) {
+return TypedValue(current_literal);
   } else {
 return TypedValue(current.getTypeID());
   }

incubator-quickstep git commit: Determine #Partitions for Aggr State Hash Table in the optimizer.

2017-08-02 Thread jianqiao 
Repository: incubator-quickstep
Updated Branches:
  refs/heads/master beda9cb84 -> 302f2cb88


Determine #Partitions for Aggr State Hash Table in the optimizer.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/302f2cb8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/302f2cb8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/302f2cb8

Branch: refs/heads/master
Commit: 302f2cb88c11e06f87350117ad5b3a1398b828b6
Parents: beda9cb
Author: Zuyu Zhang 
Authored: Thu Jul 20 15:33:54 2017 -0500
Committer: Zuyu Zhang 
Committed: Fri Jul 21 12:53:22 2017 -0500

--
 query_optimizer/CMakeLists.txt  |   1 +
 query_optimizer/ExecutionGenerator.cpp  | 183 ++-
 .../FinalizeAggregationOperator.cpp |   2 +-
 .../FinalizeAggregationOperator.hpp |   4 +-
 .../tests/AggregationOperator_unittest.cpp  |   2 +
 storage/AggregationOperationState.cpp   |  96 ++
 storage/AggregationOperationState.hpp   |  24 +--
 storage/AggregationOperationState.proto |   3 +
 storage/CMakeLists.txt  |   1 -
 storage/CollisionFreeVectorTable.cpp|   4 +-
 storage/CollisionFreeVectorTable.hpp|  14 +-
 storage/HashTableFactory.hpp|   7 +-
 12 files changed, 181 insertions(+), 160 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/302f2cb8/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 564c5c8..69105e6 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -61,6 +61,7 @@ 
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
   quickstep_catalog_CatalogTypedefs
   quickstep_catalog_PartitionScheme
   quickstep_catalog_PartitionSchemeHeader
+  quickstep_cli_Flags
   quickstep_expressions_Expressions_proto
   quickstep_expressions_aggregation_AggregateFunction
   quickstep_expressions_aggregation_AggregateFunction_proto

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/302f2cb8/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 4f6f807..88dc505 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -21,7 +21,9 @@
 
 #include 
 #include 
+#include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -46,6 +48,7 @@
 #include "catalog/CatalogTypedefs.hpp"
 #include "catalog/PartitionScheme.hpp"
 #include "catalog/PartitionSchemeHeader.hpp"
+#include "cli/Flags.hpp"
 #include "expressions/Expressions.pb.h"
 #include "expressions/aggregation/AggregateFunction.hpp"
 #include "expressions/aggregation/AggregateFunction.pb.h"
@@ -167,10 +170,83 @@ static const volatile bool aggregate_hashtable_type_dummy
 
 DEFINE_bool(parallelize_load, true, "Parallelize loading data files.");
 
+static bool ValidateNumAggregationPartitions(const char *flagname, int value) {
+  return value > 0;
+}
+DEFINE_int32(num_aggregation_partitions,
+ 41,
+ "The number of partitions in PartitionedHashTablePool used for "
+ "performing the aggregation");
+static const volatile bool num_aggregation_partitions_dummy
+= gflags::RegisterFlagValidator(_num_aggregation_partitions, 
);
+
+DEFINE_uint64(partition_aggregation_num_groups_threshold,
+  10,
+  "The threshold used for deciding whether the aggregation is done 
"
+  "in a partitioned way or not");
+
 namespace E = ::quickstep::optimizer::expressions;
 namespace P = ::quickstep::optimizer::physical;
 namespace S = ::quickstep::serialization;
 
+namespace {
+
+size_t CalculateNumFinalizationPartitionsForCollisionFreeVectorTable(const 
size_t num_entries) {
+  // Set finalization segment size as 4096 entries.
+  constexpr size_t kFinalizeSegmentSize = 4uL * 1024L;
+
+  // At least 1 partition, at most (#workers * 2) partitions.
+  return std::max(1uL, std::min(num_entries / kFinalizeSegmentSize,
+static_cast(2 * FLAGS_num_workers)));
+}
+
+bool CheckAggregatePartitioned(const std::size_t num_aggregate_functions,
+   const std::vector _distincts,
+   const std::vector _by_attrs,
+   const std::size_t