Repository: incubator-quickstep Updated Branches: refs/heads/frontend-backend 96908588b -> 0221d1358
Deserialization switch logic is implemented. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0221d135 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0221d135 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0221d135 Branch: refs/heads/frontend-backend Commit: 0221d13584ec75e9f51c8527804cd1e761249ef8 Parents: 9690858 Author: Hakan Memisoglu <hakanmemiso...@apache.org> Authored: Sun Apr 23 17:38:42 2017 -0500 Committer: Hakan Memisoglu <hakanmemiso...@apache.org> Committed: Sun Apr 23 17:38:42 2017 -0500 ---------------------------------------------------------------------- query_execution/CMakeLists.txt | 2 + query_execution/ExecutionDeserializer.cpp | 112 +++++++++++++++++++++++++ query_execution/ExecutionDeserializer.hpp | 98 ++++++++++++++++++++++ query_optimizer/ExecutionSerializer.cpp | 80 ++++++++++++------ query_optimizer/Optimizer.cpp | 9 +- utility/DAG.hpp | 4 +- 6 files changed, 271 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0221d135/query_execution/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt index eeed791..e4440fc 100644 --- a/query_execution/CMakeLists.txt +++ b/query_execution/CMakeLists.txt @@ -29,6 +29,7 @@ if (ENABLE_DISTRIBUTED) add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp) add_library(quickstep_queryexecution_BlockLocatorUtil BlockLocatorUtil.cpp BlockLocatorUtil.hpp) endif(ENABLE_DISTRIBUTED) +add_library(quickstep_queryexecution_ExecutionDeserializer ExecutionDeserializer.cpp ExecutionDeserializer.hpp) add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp) if (ENABLE_DISTRIBUTED) add_library(quickstep_queryexecution_ForemanDistributed ForemanDistributed.cpp ForemanDistributed.hpp) @@ -357,6 +358,7 @@ target_link_libraries(quickstep_queryexecution_WorkerSelectionPolicy add_library(quickstep_queryexecution ../empty_src.cpp QueryExecutionModule.hpp) target_link_libraries(quickstep_queryexecution quickstep_queryexecution_AdmitRequestMessage + quickstep_queryexecution_ExecutionDeserializer quickstep_queryexecution_ForemanBase quickstep_queryexecution_ForemanSingleNode quickstep_queryexecution_PolicyEnforcerBase http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0221d135/query_execution/ExecutionDeserializer.cpp ---------------------------------------------------------------------- diff --git a/query_execution/ExecutionDeserializer.cpp b/query_execution/ExecutionDeserializer.cpp new file mode 100644 index 0000000..bb2b4b0 --- /dev/null +++ b/query_execution/ExecutionDeserializer.cpp @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#include "query_execution/ExecutionDeserializer.hpp" + +namespace quickstep { +namespace optimizer { + +void ExecutionDeserializer::deserializePlan() const { + +} + +void ExecutionDeserializer::deserializeInternal(const RelationalOperator &relational_operator) { + switch (relational_operator.getOperatorType()) { + case RelationalOperator::kAggregation: + deserializeAggregation( + static_cast<AggregationOperator&>(relational_operator)); + case RelationalOperator::kBuildAggregationExistenceMap: + deserializeBuildAggregationExistenceMap( + static_cast<BuildAggregationExistenceMapOperator&>(relational_operator)); + case RelationalOperator::kBuildHash: + deserializeBuildHash( + static_cast<BuildHashOperator&>(relational_operator)); + case RelationalOperator::kBuildLIPFilter: + deserializeBuildLIPFilter( + static_cast<BuildLIPFilterOperator&>(relational_operator)); + case RelationalOperator::kCreateIndex: + deserializeCreateIndex( + static_cast<CreateIndexOperator&>(relational_operator)); + case RelationalOperator::kCreateTable: + deserializeCreateTable( + static_cast<CreateTableOperator&>(relational_operator)); + case RelationalOperator::kDelete: + deserializeDelete( + static_cast<DeleteOperator&>(relational_operator)); + case RelationalOperator::kDestroyAggregationState: + deserializeDestroyAggregationState( + static_cast<DestroyAggregationStateOperator&>(relational_operator)); + case RelationalOperator::kDestroyHash: + deserializeDestroyHash( + static_cast<DestroyHashOperator&>(relational_operator)); + case RelationalOperator::kDropTable: + deserializeDropTable( + static_cast<DropTableOperator&>(relational_operator)); + case RelationalOperator::kFinalizeAggregation: + deserializeFinalizeAggregation( + static_cast<FinalizeAggregationOperator&>(relational_operator)); + case RelationalOperator::kInitializeAggregation: + deserializeInitializeAggregation( + static_cast<InitializeAggregationOperator&>(relational_operator); + case RelationalOperator::kInnerJoin: + case RelationalOperator::kLeftAntiJoin: + case RelationalOperator::kLeftOuterJoin: + case RelationalOperator::kLeftSemiJoin: + deserializeHashJoin( + static_cast<HashJoinOperator&>(relational_operator)); + case RelationalOperator::kInsert: + deserializeInsert( + static_cast<InsertOperator&>(relational_operator)); + case RelationalOperator::kNestedLoopsJoin: + deserializeNestedLoopsJoin( + static_cast<NestedLoopsJoinOperator&>(relational_operator)); + case RelationalOperator::kSample: + deserializeSample( + static_cast<SampleOperator&>(relational_operator)); + case RelationalOperator::kSaveBlocks: + deserializeSaveBlocks( + static_cast<SaveBlocksOperator&>(relational_operator)); + case RelationalOperator::kSelect: + deserializeSelect( + static_cast<SelectOperator&>(relational_operator)); + case RelationalOperator::kSortMergeRun: + deserializeSortMergeRun( + static_cast<SortMergeRunOperator&>(relational_operator)); + case RelationalOperator::kSortRunGeneration: + deserializeSortRunGeneration( + static_cast<SortRunGenerationOperator&>(relational_operator)); + case RelationalOperator::kTableGenerator: + deserializeTableGenerator( + static_cast<TableGeneratorOperator&>(relational_operator)); + case RelationalOperator::kTextScan: + deserializeTextScan( + static_cast<TextScanOperator&>(relational_operator)); + case RelationalOperator::kUpdate: + deserializeUpdate( + static_cast<UpdateOperator&>(relational_operator)); + case RelationalOperator::kWindowAggregation: + deserializeWindowAggregation( + static_cast<WindowAggregationOperator&>(relational_operator)); + case RelationalOperator::kMockOperator: + break; + } +} + +} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0221d135/query_execution/ExecutionDeserializer.hpp ---------------------------------------------------------------------- diff --git a/query_execution/ExecutionDeserializer.hpp b/query_execution/ExecutionDeserializer.hpp new file mode 100644 index 0000000..3521f04 --- /dev/null +++ b/query_execution/ExecutionDeserializer.hpp @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#ifndef QUICKSTEP_QUERY_EXECUTION_EXECUTION_DESERIALIZER_HPP +#define QUICKSTEP_QUERY_EXECUTION_EXECUTION_DESERIALIZER_HPP + +#include "relational_operators/AggregationOperator.hpp" +#include "relational_operators/BuildAggregationExistenceMapOperator.hpp" +#include "relational_operators/BuildHashOperator.hpp" +#include "relational_operators/BuildLIPFilterOperator.hpp" +#include "relational_operators/CreateIndexOperator.hpp" +#include "relational_operators/CreateTableOperator.hpp" +#include "relational_operators/RelationalOperator.hpp" +#include "relational_operators/DeleteOperator.hpp" +#include "relational_operators/DestroyAggregationStateOperator.hpp" +#include "relational_operators/DestroyHashOperator.hpp" +#include "relational_operators/DropTableOperator.hpp" +#include "relational_operators/FinalizeAggregationOperator.hpp" +#include "relational_operators/HashJoinOperator.hpp" +#include "relational_operators/InitializeAggregationOperator.hpp" +#include "relational_operators/InsertOperator.hpp" +#include "relational_operators/NestedLoopsJoinOperator.hpp" +#include "relational_operators/SampleOperator.hpp" +#include "relational_operators/SaveBlocksOperator.hpp" +#include "relational_operators/SelectOperator.hpp" +#include "relational_operators/SortMergeRunOperator.hpp" +#include "relational_operators/SortRunGenerationOperator.hpp" +#include "relational_operators/TableGeneratorOperator.hpp" +#include "relational_operators/TextScanOperator.hpp" +#include "relational_operators/UpdateOperator.hpp" +#include "relational_operators/WindowAggregationOperator.hpp" + +#include "query_optimizer/QueryPlan.pb.h" + +namespace quickstep { +namespace optimizer { + +namespace S = ::quickstep::serialization; + +class ExecutionDeserializer { + public: + ExecutionDeserializer(const S::QueryPlan &query_plan) + : query_plan_proto_(query_plan) { + } + + void deserializePlan() const; + + private: + void deserializeInternal(const RelationalOperator &relational_operator); + + void deserializeAggregation(const AggregationOperator &aggregation_operator); + void deserializeBuildAggregationExistenceMap(const BuildAggregationExistenceMapOperator &op); + void deserializeBuildHash(const BuildHashOperator &build_hash_operator); + void deserializeBuildLIPFilter(const BuildLIPFilterOperator &build_lip_filter_operator); + void deserializeCreateIndex(const CreateIndexOperator &create_index_operator); + void deserializeCreateTable(const CreateTableOperator &create_table_operator); + void deserializeDelete(const DeleteOperator &delete_operator); + void deserializeDestroyAggregationState(const DestroyAggregationStateOperator &destroy_aggregation_state_operator); + void deserializeDestroyHash(const DestroyHashOperator &destroy_hash_operator); + void deserializeDropTable(const DropTableOperator &drop_table_operator); + void deserializeFinalizeAggregation(const FinalizeAggregationOperator &finalize_aggregation_operator); + void deserializeHashJoin(const HashJoinOperator &finalize_hash_join); + void deserializeInitializeAggregation(const InitializeAggregationOperator &initialize_aggregation_operator); + void deserializeInsert(const InsertOperator &insert_operator); + void deserializeNestedLoopsJoin(const NestedLoopsJoinOperator &nested_loops_join_operator); + void deserializeSample(const SampleOperator &sample_operator); + void deserializeSaveBlocks(const SaveBlocksOperator &save_blocks_operator); + void deserializeSelect(const SelectOperator &select_operator); + void deserializeSortMergeRun(const SortMergeRunOperator &sort_merge_run_operator); + void deserializeSortRunGeneration(const SortRunGenerationOperator &sort_run_generation_operator); + void deserializeTableGenerator(const TableGeneratorOperator &table_generator_operator); + void deserializeTextScan(const TextScanOperator &text_scan_operator); + void deserializeUpdate(const UpdateOperator &update_operator); + void deserializeWindowAggregation(const WindowAggregationOperator &window_aggregation_operator); + + const S::QueryPlan &query_plan_proto_; +}; + +} +} + +#endif //QUICKSTEP_EXECUTIONDESERIALIZER_HPP http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0221d135/query_optimizer/ExecutionSerializer.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionSerializer.cpp b/query_optimizer/ExecutionSerializer.cpp index 892567b..885ec93 100644 --- a/query_optimizer/ExecutionSerializer.cpp +++ b/query_optimizer/ExecutionSerializer.cpp @@ -22,11 +22,31 @@ #include <cstddef> #include "query_optimizer/QueryPlan.hpp" -#include "relational_operators/RelationalOperator.hpp" #include "relational_operators/AggregationOperator.hpp" #include "relational_operators/BuildAggregationExistenceMapOperator.hpp" #include "relational_operators/BuildHashOperator.hpp" - +#include "relational_operators/BuildLIPFilterOperator.hpp" +#include "relational_operators/CreateIndexOperator.hpp" +#include "relational_operators/CreateTableOperator.hpp" +#include "relational_operators/DeleteOperator.hpp" +#include "relational_operators/DestroyAggregationStateOperator.hpp" +#include "relational_operators/DestroyHashOperator.hpp" +#include "relational_operators/DropTableOperator.hpp" +#include "relational_operators/FinalizeAggregationOperator.hpp" +#include "relational_operators/HashJoinOperator.hpp" +#include "relational_operators/InitializeAggregationOperator.hpp" +#include "relational_operators/InsertOperator.hpp" +#include "relational_operators/NestedLoopsJoinOperator.hpp" +#include "relational_operators/RelationalOperator.hpp" +#include "relational_operators/SampleOperator.hpp" +#include "relational_operators/SaveBlocksOperator.hpp" +#include "relational_operators/SelectOperator.hpp" +#include "relational_operators/SortMergeRunOperator.hpp" +#include "relational_operators/SortRunGenerationOperator.hpp" +#include "relational_operators/TableGeneratorOperator.hpp" +#include "relational_operators/TextScanOperator.hpp" +#include "relational_operators/UpdateOperator.hpp" +#include "relational_operators/WindowAggregationOperator.hpp" #include "query_optimizer/QueryPlan.pb.h" #include "relational_operators/Operator.pb.h" @@ -38,6 +58,13 @@ namespace S = ::quickstep::serialization; void ExecutionSerializer::serializePlan(const QueryPlan &query_plan) { const auto &dag = query_plan.getQueryPlanDAG(); for (std::size_t i = 0; i < dag.size(); ++i) { + const auto &edges = dag.getDependencies(i); + S::OutgoingEdges *outgoing_edges = query_plan_proto_.add_outgoing(); + for (const auto &edge : edges) { + S::Edge *edge_proto = outgoing_edges->add_edges(); + edge_proto->set_outgoing_id(edge); + edge_proto->set_payload(dag.getLinkMetadata(i, edge)); + } const auto &relational_operator = dag.getNodePayload(i); serializeInternal(relational_operator); } @@ -47,79 +74,79 @@ void ExecutionSerializer::serializeInternal(const RelationalOperator &relational switch (relational_operator.getOperatorType()) { case RelationalOperator::OperatorType::kAggregation: return serializeAggregation( - dynamic_cast<const AggregationOperator&>(relational_operator)); + static_cast<const AggregationOperator&>(relational_operator)); case RelationalOperator::kBuildAggregationExistenceMap: return serializeBuildAggregationExistenceMap( - dynamic_cast<const BuildAggregationExistenceMapOperator&>(relational_operator)); + static_cast<const BuildAggregationExistenceMapOperator&>(relational_operator)); case RelationalOperator::kBuildHash: return serializeBuildHash( - dynamic_cast<const BuildHashOperator&>(relational_operator)); + static_cast<const BuildHashOperator&>(relational_operator)); case RelationalOperator::kBuildLIPFilter: return serializeBuildLIPFilter( - dynamic_cast<const BuildLIPFilterOperator&>(relational_operator)); + static_cast<const BuildLIPFilterOperator&>(relational_operator)); case RelationalOperator::kCreateIndex: return serializeCreateIndex( - dynamic_cast<const CreateIndexOperator&>(relational_operator)); + static_cast<const CreateIndexOperator&>(relational_operator)); case RelationalOperator::kCreateTable: return serializeCreateTable( - dynamic_cast<const CreateTableOperator&>(relational_operator)); + static_cast<const CreateTableOperator&>(relational_operator)); case RelationalOperator::kDelete: return serializeDelete( - dynamic_cast<const DeleteOperator&>(relational_operator)); + static_cast<const DeleteOperator&>(relational_operator)); case RelationalOperator::kDestroyAggregationState: return serializeDestroyAggregationState( - dynamic_cast<const DestroyAggregationStateOperator&>(relational_operator)); + static_cast<const DestroyAggregationStateOperator&>(relational_operator)); case RelationalOperator::kDestroyHash: return serializeDestroyHash( - dynamic_cast<const DestroyHashOperator&>(relational_operator)); + static_cast<const DestroyHashOperator&>(relational_operator)); case RelationalOperator::kDropTable: return serializeDropTable( - dynamic_cast<const DropTableOperator&>(relational_operator)); + static_cast<const DropTableOperator&>(relational_operator)); case RelationalOperator::kFinalizeAggregation: return serializeFinalizeAggregation( - dynamic_cast<const FinalizeAggregationOperator&>(relational_operator)); + static_cast<const FinalizeAggregationOperator&>(relational_operator)); case RelationalOperator::kInitializeAggregation: return serializeInitializeAggregation( - dynamic_cast<const InitializeAggregationOperator&>(relational_operator)); + static_cast<const InitializeAggregationOperator&>(relational_operator)); case RelationalOperator::kInsert: return serializeInsert( - dynamic_cast<const InsertOperator&>(relational_operator)); + static_cast<const InsertOperator&>(relational_operator)); case RelationalOperator::kInnerJoin: case RelationalOperator::kLeftAntiJoin: case RelationalOperator::kLeftOuterJoin: case RelationalOperator::kLeftSemiJoin: return serializeHashJoin( - dynamic_cast<const HashJoinOperator&>(relational_operator)); + static_cast<const HashJoinOperator&>(relational_operator)); case RelationalOperator::kNestedLoopsJoin: return serializeNestedLoopsJoin( - dynamic_cast<const NestedLoopsJoinOperator&>(relational_operator)); + static_cast<const NestedLoopsJoinOperator&>(relational_operator)); case RelationalOperator::kSample: return serializeSample( - dynamic_cast<const SampleOperator&>(relational_operator)); + static_cast<const SampleOperator&>(relational_operator)); case RelationalOperator::kSaveBlocks: return serializeSaveBlocks( - dynamic_cast<const SaveBlocksOperator&>(relational_operator)); + static_cast<const SaveBlocksOperator&>(relational_operator)); case RelationalOperator::kSelect: return serializeSelect( - dynamic_cast<const SelectOperator&>(relational_operator)); + static_cast<const SelectOperator&>(relational_operator)); case RelationalOperator::kSortMergeRun: return serializeSortMergeRun( - dynamic_cast<const SortMergeRunOperator&>(relational_operator)); + static_cast<const SortMergeRunOperator&>(relational_operator)); case RelationalOperator::kSortRunGeneration: return serializeSortRunGeneration( - dynamic_cast<const SortRunGenerationOperator&>(relational_operator)); + static_cast<const SortRunGenerationOperator&>(relational_operator)); case RelationalOperator::kTableGenerator: return serializeTableGenerator( - dynamic_cast<const TableGeneratorOperator&>(relational_operator)); + static_cast<const TableGeneratorOperator&>(relational_operator)); case RelationalOperator::kTextScan: return serializeTextScan( - dynamic_cast<const TextScanOperator&>(relational_operator)); + static_cast<const TextScanOperator&>(relational_operator)); case RelationalOperator::kUpdate: return serializeUpdate( - dynamic_cast<const UpdateOperator&>(relational_operator)); + static_cast<const UpdateOperator&>(relational_operator)); case RelationalOperator::kWindowAggregation: return serializeWindowAggregation( - dynamic_cast<const WindowAggregationOperator&>(relational_operator)); + static_cast<const WindowAggregationOperator&>(relational_operator)); case RelationalOperator::kMockOperator: break; } @@ -437,6 +464,5 @@ void ExecutionSerializer::serializeWindowAggregation(const WindowAggregationOper window_aggregation_operator.getWindowAggregationStateIndex()); } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0221d135/query_optimizer/Optimizer.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/Optimizer.cpp b/query_optimizer/Optimizer.cpp index 35ed4ba..cb9cb24 100644 --- a/query_optimizer/Optimizer.cpp +++ b/query_optimizer/Optimizer.cpp @@ -20,8 +20,10 @@ #include "query_optimizer/Optimizer.hpp" #include "query_optimizer/ExecutionGenerator.hpp" +#include "query_optimizer/ExecutionSerializer.hpp" #include "query_optimizer/LogicalGenerator.hpp" + namespace quickstep { namespace optimizer { @@ -32,16 +34,13 @@ void Optimizer::generateQueryHandle(const ParseStatement &parse_statement, LogicalGenerator logical_generator(optimizer_context); PhysicalGenerator physical_generator(optimizer_context); ExecutionGenerator execution_generator(catalog_database, query_handle); + ExecutionSerializer execution_serializer; execution_generator.generatePlan( physical_generator.generatePlan( logical_generator.generatePlan(*catalog_database, parse_statement))); - // TODO(hakan): Break at this point. - // execution_serializer.serializePlan( - // execution_generator.generatePlan( - // physical_generator.generatePlan( - // logical_generator.generatePlan(*catalog_database, parse_statement)))); + execution_serializer.serializePlan(execution_generator); } } // namespace optimizer http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0221d135/utility/DAG.hpp ---------------------------------------------------------------------- diff --git a/utility/DAG.hpp b/utility/DAG.hpp index a1f2619..fcfe1ce 100644 --- a/utility/DAG.hpp +++ b/utility/DAG.hpp @@ -165,7 +165,7 @@ class DAG { * @param dependent_index Index of the dependent node in the DAG. **/ const LinkMetadataT& getLinkMetadata(const size_type_nodes dependency_index, - const size_type_nodes dependent_index) { + const size_type_nodes dependent_index) const { return nodes_[dependency_index].getLinkMetadata(dependent_index); } @@ -317,7 +317,7 @@ class DAG { * * @return The metadata of the link. **/ - inline const LinkMetadataT& getLinkMetadata(const size_type_nodes node_index) { + inline const LinkMetadataT& getLinkMetadata(const size_type_nodes node_index) const { DCHECK(dependents_with_metadata_.find(node_index) != dependents_with_metadata_.end()); return dependents_with_metadata_[node_index]; }