Repository: incubator-quickstep Updated Branches: refs/heads/frontend-backend 596cb529f -> 96908588b
Implemented serialization logic of all relational operators. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/96908588 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/96908588 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/96908588 Branch: refs/heads/frontend-backend Commit: 96908588b2464674b6f5dfcaa8a2a4dc4b3908c2 Parents: 596cb52 Author: Hakan Memisoglu <hakanmemiso...@apache.org> Authored: Mon Apr 17 13:25:16 2017 -0500 Committer: Hakan Memisoglu <hakanmemiso...@apache.org> Committed: Mon Apr 17 13:25:16 2017 -0500 ---------------------------------------------------------------------- query_optimizer/CMakeLists.txt | 6 + query_optimizer/ExecutionGenerator.cpp | 4 +- query_optimizer/ExecutionGenerator.hpp | 2 +- query_optimizer/ExecutionSerializer.cpp | 442 +++++++++++++++++++ query_optimizer/ExecutionSerializer.hpp | 105 +++++ query_optimizer/Optimizer.cpp | 5 +- query_optimizer/QueryPlan.proto | 82 ++-- relational_operators/AggregationOperator.hpp | 5 +- .../BuildAggregationExistenceMapOperator.hpp | 8 + relational_operators/BuildHashOperator.hpp | 16 + relational_operators/BuildLIPFilterOperator.hpp | 4 + relational_operators/CreateIndexOperator.hpp | 12 + relational_operators/CreateTableOperator.hpp | 4 + relational_operators/DeleteOperator.hpp | 4 + .../DestroyAggregationStateOperator.hpp | 4 + relational_operators/DestroyHashOperator.hpp | 8 + relational_operators/DropTableOperator.hpp | 10 +- .../FinalizeAggregationOperator.hpp | 4 + relational_operators/HashJoinOperator.hpp | 28 ++ .../InitializeAggregationOperator.hpp | 4 + relational_operators/InsertOperator.hpp | 4 + .../NestedLoopsJoinOperator.hpp | 16 + relational_operators/Operator.proto | 301 +++++++------ relational_operators/SampleOperator.hpp | 12 + relational_operators/SaveBlocksOperator.hpp | 8 + relational_operators/SelectOperator.hpp | 12 + relational_operators/SortMergeRunOperator.hpp | 24 + .../SortRunGenerationOperator.hpp | 8 + relational_operators/TableGeneratorOperator.hpp | 4 + relational_operators/TextScanOperator.hpp | 12 + relational_operators/UpdateOperator.hpp | 8 + .../WindowAggregationOperator.hpp | 8 + 32 files changed, 1006 insertions(+), 168 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt index abe59e2..79710fb 100644 --- a/query_optimizer/CMakeLists.txt +++ b/query_optimizer/CMakeLists.txt @@ -41,6 +41,7 @@ add_subdirectory(tests) # Declare micro-libs: add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp) +add_library(quickstep_queryoptimizer_ExecutionSerializer ExecutionSerializer.cpp ExecutionSerializer.hpp) add_library(quickstep_queryoptimizer_LIPFilterGenerator LIPFilterGenerator.cpp LIPFilterGenerator.hpp) add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp) add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper @@ -165,6 +166,10 @@ if (ENABLE_DISTRIBUTED) target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator quickstep_catalog_Catalog_proto) endif() +target_link_libraries(quickstep_queryoptimizer_ExecutionSerializer + quickstep_queryoptimizer_QueryPlan_proto + quickstep_relationaloperators_AggregationOperator + quickstep_relationaloperators_Operator_proto) target_link_libraries(quickstep_queryoptimizer_LIPFilterGenerator glog quickstep_catalog_CatalogAttribute @@ -203,6 +208,7 @@ target_link_libraries(quickstep_queryoptimizer_LogicalToPhysicalMapper quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_Optimizer quickstep_queryoptimizer_ExecutionGenerator + quickstep_queryoptimizer_ExecutionSerializer quickstep_queryoptimizer_LogicalGenerator quickstep_queryoptimizer_PhysicalGenerator quickstep_utility_Macros) http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/ExecutionGenerator.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp index 6fec85b..2878918 100644 --- a/query_optimizer/ExecutionGenerator.cpp +++ b/query_optimizer/ExecutionGenerator.cpp @@ -169,7 +169,7 @@ namespace S = ::quickstep::serialization; constexpr QueryPlan::DAGNodeIndex ExecutionGenerator::CatalogRelationInfo::kInvalidOperatorIndex; -void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) { +const QueryPlan& ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) { CHECK(P::SomeTopLevelPlan::MatchesWithConditionalCast(physical_plan, &top_level_physical_plan_)) << "The physical plan must be rooted by a TopLevelPlan"; @@ -244,6 +244,8 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) { catalog_database_cache_proto_->add_relations()->MergeFrom(relation.getProto()); } #endif + + return query_handle_->getQueryPlan(); } void ExecutionGenerator::generatePlanInternal( http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/ExecutionGenerator.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp index f4e614a..86f7264 100644 --- a/query_optimizer/ExecutionGenerator.hpp +++ b/query_optimizer/ExecutionGenerator.hpp @@ -125,7 +125,7 @@ class ExecutionGenerator { * @param physical_plan The physical plan from which * the execution plan is created. */ - void generatePlan(const physical::PhysicalPtr &physical_plan); + const QueryPlan& generatePlan(const physical::PhysicalPtr &physical_plan); private: /** http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/ExecutionSerializer.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionSerializer.cpp b/query_optimizer/ExecutionSerializer.cpp new file mode 100644 index 0000000..892567b --- /dev/null +++ b/query_optimizer/ExecutionSerializer.cpp @@ -0,0 +1,442 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#include "query_optimizer/ExecutionSerializer.hpp" + +#include <cstddef> + +#include "query_optimizer/QueryPlan.hpp" +#include "relational_operators/RelationalOperator.hpp" +#include "relational_operators/AggregationOperator.hpp" +#include "relational_operators/BuildAggregationExistenceMapOperator.hpp" +#include "relational_operators/BuildHashOperator.hpp" + +#include "query_optimizer/QueryPlan.pb.h" +#include "relational_operators/Operator.pb.h" + +namespace quickstep { +namespace optimizer { + +namespace S = ::quickstep::serialization; + +void ExecutionSerializer::serializePlan(const QueryPlan &query_plan) { + const auto &dag = query_plan.getQueryPlanDAG(); + for (std::size_t i = 0; i < dag.size(); ++i) { + const auto &relational_operator = dag.getNodePayload(i); + serializeInternal(relational_operator); + } +} + +void ExecutionSerializer::serializeInternal(const RelationalOperator &relational_operator) { + switch (relational_operator.getOperatorType()) { + case RelationalOperator::OperatorType::kAggregation: + return serializeAggregation( + dynamic_cast<const AggregationOperator&>(relational_operator)); + case RelationalOperator::kBuildAggregationExistenceMap: + return serializeBuildAggregationExistenceMap( + dynamic_cast<const BuildAggregationExistenceMapOperator&>(relational_operator)); + case RelationalOperator::kBuildHash: + return serializeBuildHash( + dynamic_cast<const BuildHashOperator&>(relational_operator)); + case RelationalOperator::kBuildLIPFilter: + return serializeBuildLIPFilter( + dynamic_cast<const BuildLIPFilterOperator&>(relational_operator)); + case RelationalOperator::kCreateIndex: + return serializeCreateIndex( + dynamic_cast<const CreateIndexOperator&>(relational_operator)); + case RelationalOperator::kCreateTable: + return serializeCreateTable( + dynamic_cast<const CreateTableOperator&>(relational_operator)); + case RelationalOperator::kDelete: + return serializeDelete( + dynamic_cast<const DeleteOperator&>(relational_operator)); + case RelationalOperator::kDestroyAggregationState: + return serializeDestroyAggregationState( + dynamic_cast<const DestroyAggregationStateOperator&>(relational_operator)); + case RelationalOperator::kDestroyHash: + return serializeDestroyHash( + dynamic_cast<const DestroyHashOperator&>(relational_operator)); + case RelationalOperator::kDropTable: + return serializeDropTable( + dynamic_cast<const DropTableOperator&>(relational_operator)); + case RelationalOperator::kFinalizeAggregation: + return serializeFinalizeAggregation( + dynamic_cast<const FinalizeAggregationOperator&>(relational_operator)); + case RelationalOperator::kInitializeAggregation: + return serializeInitializeAggregation( + dynamic_cast<const InitializeAggregationOperator&>(relational_operator)); + case RelationalOperator::kInsert: + return serializeInsert( + dynamic_cast<const InsertOperator&>(relational_operator)); + case RelationalOperator::kInnerJoin: + case RelationalOperator::kLeftAntiJoin: + case RelationalOperator::kLeftOuterJoin: + case RelationalOperator::kLeftSemiJoin: + return serializeHashJoin( + dynamic_cast<const HashJoinOperator&>(relational_operator)); + case RelationalOperator::kNestedLoopsJoin: + return serializeNestedLoopsJoin( + dynamic_cast<const NestedLoopsJoinOperator&>(relational_operator)); + case RelationalOperator::kSample: + return serializeSample( + dynamic_cast<const SampleOperator&>(relational_operator)); + case RelationalOperator::kSaveBlocks: + return serializeSaveBlocks( + dynamic_cast<const SaveBlocksOperator&>(relational_operator)); + case RelationalOperator::kSelect: + return serializeSelect( + dynamic_cast<const SelectOperator&>(relational_operator)); + case RelationalOperator::kSortMergeRun: + return serializeSortMergeRun( + dynamic_cast<const SortMergeRunOperator&>(relational_operator)); + case RelationalOperator::kSortRunGeneration: + return serializeSortRunGeneration( + dynamic_cast<const SortRunGenerationOperator&>(relational_operator)); + case RelationalOperator::kTableGenerator: + return serializeTableGenerator( + dynamic_cast<const TableGeneratorOperator&>(relational_operator)); + case RelationalOperator::kTextScan: + return serializeTextScan( + dynamic_cast<const TextScanOperator&>(relational_operator)); + case RelationalOperator::kUpdate: + return serializeUpdate( + dynamic_cast<const UpdateOperator&>(relational_operator)); + case RelationalOperator::kWindowAggregation: + return serializeWindowAggregation( + dynamic_cast<const WindowAggregationOperator&>(relational_operator)); + case RelationalOperator::kMockOperator: + break; + } +} + +void ExecutionSerializer::serializeAggregation(const AggregationOperator &aggregation_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_AGGREGATION); + + serialization::AggregationOperator *aggregation = vertex->mutable_aggregation_operator(); + aggregation->set_relation_id(aggregation_operator.input_relation().getID()); + aggregation->set_relation_is_stored(!aggregation_operator.input_relation().isTemporary()); + aggregation->set_aggr_state_index(aggregation_operator.getAggregationStateIndex()); +} + +void ExecutionSerializer::serializeBuildAggregationExistenceMap(const BuildAggregationExistenceMapOperator &op) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_BUILD_AGGREGATION_EXISTENCE_MAP); + + serialization::BuildAggregationExistenceMapOperator *build_aggregation_existence_map_work_order = vertex->mutable_build_aggregation_existence_map_operator(); + build_aggregation_existence_map_work_order->set_relation_id(op.input_relation().getID()); + build_aggregation_existence_map_work_order->set_attribute_id(op.getBuildAttributeID()); + build_aggregation_existence_map_work_order->set_relation_is_stored(!op.input_relation().isTemporary()); + build_aggregation_existence_map_work_order->set_aggr_state_index(op.getAggregationStateIndex()); +} + +void ExecutionSerializer::serializeBuildHash(const BuildHashOperator &build_hash_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_BUILD_HASH); + + serialization::BuildHashOperator *build_hash = vertex->mutable_build_hash_operator(); + build_hash->set_relation_id(build_hash_operator.input_relation().getID()); + build_hash->set_relation_is_stored(!build_hash_operator.input_relation().isTemporary()); + for (const auto &join_key : build_hash_operator.getJoinKeyAttributes()) { + build_hash->add_join_key_attributes(join_key); + } + build_hash->set_any_join_key_nullable(build_hash_operator.anyJoinKeyNullable()); + build_hash->set_num_partitions(build_hash_operator.getNumberOfPartitions()); + build_hash->set_hash_table_index(build_hash_operator.getHashTableIndex()); +} + +void ExecutionSerializer::serializeBuildLIPFilter( + const BuildLIPFilterOperator &build_lip_filter_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_BUILD_LIP_FILTER); + + serialization::BuildLIPFilterOperator *build_lip_filter = vertex->mutable_build_lip_filter_operator(); + build_lip_filter->set_relation_id(build_lip_filter_operator.input_relation().getID()); + build_lip_filter->set_relation_is_stored(!build_lip_filter_operator.input_relation().isTemporary()); + build_lip_filter->set_build_side_predicate_index(build_lip_filter_operator.getPredicateIndex()); +} + +void ExecutionSerializer::serializeCreateIndex( + const CreateIndexOperator &create_index_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_CREATE_INDEX); + + serialization::CreateIndexOperator *create_index = vertex->mutable_create_index_operator(); + create_index->set_relation_id(create_index_operator.getRelation().getID()); + IndexSubBlockDescription *index_description = create_index->mutable_index_description(); + index_description->CopyFrom(create_index_operator.getIndexDescription()); + create_index->set_index_name(create_index_operator.getIndexName()); +} + +void ExecutionSerializer::serializeCreateTable(const CreateTableOperator &create_table_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_CREATE_TABLE); + + serialization::CreateTableOperator *create_table = vertex->mutable_create_table_operator(); + // TODO(hakan): Add database id serialization. + serialization::CatalogRelation *catalog_relation = create_table->mutable_relation_catalog(); + catalog_relation->CopyFrom(create_table_operator.getRelation().getProto()); +} + +void ExecutionSerializer::serializeDelete(const DeleteOperator &delete_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_DELETE); + + serialization::DeleteOperator *delete_op = vertex->mutable_delete_operator(); + delete_op->set_relation_id(delete_operator.getOutputRelationID()); + delete_op->set_predicate_index(delete_operator.getPredicateIndex()); + +} +void ExecutionSerializer::serializeDestroyAggregationState( + const DestroyAggregationStateOperator &destroy_aggregation_state_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_DESTROY_AGGREGATION_STATE); + + serialization::DestroyAggregationStateOperator *destroy_aggregation + = vertex->mutable_destroy_aggregation_state_operator(); + destroy_aggregation->set_aggr_state_index( + destroy_aggregation_state_operator.getAggregationStateIndex()); +} +void ExecutionSerializer::serializeDestroyHash(const DestroyHashOperator &destroy_hash_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_DESTROY_HASH); + + serialization::DestroyHashOperator *destroy_hash = vertex->mutable_destroy_hash_operator(); + destroy_hash->set_hash_table_index(destroy_hash_operator.getHashTableIndex()); + destroy_hash->set_build_num_partitions(destroy_hash_operator.getNumPartitions()); +} +void ExecutionSerializer::serializeDropTable(const DropTableOperator &drop_table_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_DROP_TABLE); + + serialization::DropTableOperator *drop_table = vertex->mutable_drop_table_operator(); + // TODO(hakan): Add database id serialization. + drop_table->set_relation_id(drop_table_operator.getRelationID()); + drop_table->set_only_drop_blocks(drop_table_operator.onlyDropBlocks()); +} + +void ExecutionSerializer::serializeFinalizeAggregation( + const FinalizeAggregationOperator &finalize_aggregation_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_FINALIZE_AGGREGATION); + + serialization::FinalizeAggregationOperator *finalize_aggregation + = vertex->mutable_finalize_aggregation_operator(); + finalize_aggregation->set_aggr_state_index(finalize_aggregation_operator.getAggregationStateIndex()); + finalize_aggregation->set_output_relation_id(finalize_aggregation_operator.getOutputRelationID()); + finalize_aggregation->set_output_destination_index(finalize_aggregation_operator.getInsertDestinationID()); +} + +void ExecutionSerializer::serializeHashJoin(const HashJoinOperator &finalize_hash_join) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_HASH_JOIN); + + serialization::HashJoinOperator *hash_join = vertex->mutable_hash_join_operator(); + hash_join->set_build_relation_id(finalize_hash_join.build_relation().getID()); + hash_join->set_probe_relation_id(finalize_hash_join.probe_relation().getID()); + for (const auto &join_key_attr : finalize_hash_join.getJoinKeyAttributes()) { + hash_join->add_join_key_attributes(join_key_attr); + } + hash_join->set_any_join_key_attributes_nullable(finalize_hash_join.anyJoinKeyNullable()); + hash_join->set_build_num_partitions(finalize_hash_join.getBuildNumPartitions()); + hash_join->set_output_relation_id(finalize_hash_join.getOutputRelationID()); + hash_join->set_output_destionation_index(finalize_hash_join.getInsertDestinationID()); + hash_join->set_hash_table_index(finalize_hash_join.getJoinHashTableIndex()); + hash_join->set_residiual_predicate_index(finalize_hash_join.getResidualPredicateIndex()); + hash_join->set_selection_index(finalize_hash_join.getSelectionIndex()); + for (const auto &is_selection_on_build : finalize_hash_join.getSelectionsOnBuild()) { + hash_join->add_is_selection_on_build(is_selection_on_build); + } + + switch (finalize_hash_join.getOperatorType()) { + case RelationalOperator::kInnerJoin: { + hash_join->set_join_type(S::HashJoinOperator_JoinType_LEFT_INNER); + break; + } + case RelationalOperator::kLeftSemiJoin: { + hash_join->set_join_type(S::HashJoinOperator_JoinType_LEFT_SEMI); + break; + } + case RelationalOperator::kLeftAntiJoin: { + hash_join->set_join_type(S::HashJoinOperator_JoinType_LEFT_ANTI); + } + case RelationalOperator::kLeftOuterJoin: { + hash_join->set_join_type(S::HashJoinOperator_JoinType_LEFT_OUTER); + break; + } + default: { + // TODO(hakan): Write exception log message. + } + } +} +void ExecutionSerializer::serializeInitializeAggregation( + const InitializeAggregationOperator &initialize_aggregation_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_INITIALIZE_AGGREGATION); + + serialization::InitializeAggregationOperator *initialize_aggregation + = vertex->mutable_initialize_aggregation_operator(); + initialize_aggregation->set_aggr_state_index( + initialize_aggregation_operator.getAggregationStateIndex()); +} + +void ExecutionSerializer::serializeInsert(const InsertOperator &insert_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_INSERT); + + serialization::InsertOperator *insert = vertex->mutable_insert_operator(); + insert->set_output_relation_id(insert_operator.getOutputRelationID()); + insert->set_output_destination_index(insert_operator.getInsertDestinationID()); + insert->set_tuple_index(insert_operator.getTupleIndex()); +} + +void ExecutionSerializer::serializeNestedLoopsJoin(const NestedLoopsJoinOperator &nested_loops_join_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_NESTED_LOOP_JOIN); + + serialization::NestedLoopsJoinOperator *nested_loops_join = vertex->mutable_nested_loops_join_operator(); + nested_loops_join->set_left_input_relation_id(nested_loops_join_operator.getLeftInputRelation().getID()); + nested_loops_join->set_right_input_relation_id(nested_loops_join_operator.getRightInputRelation().getID()); + nested_loops_join->set_output_relation_id(nested_loops_join_operator.getOutputRelationID()); + nested_loops_join->set_output_destination_index(nested_loops_join_operator.getInsertDestinationID()); + nested_loops_join->set_join_predicate_index(nested_loops_join_operator.getJoinPredicateIndex()); + nested_loops_join->set_selection_index(nested_loops_join_operator.getSelectionIndex()); + nested_loops_join->set_left_relation_is_stored(!nested_loops_join_operator.getLeftInputRelation().isTemporary()); + nested_loops_join->set_right_relation_is_stored(!nested_loops_join_operator.getRightInputRelation().isTemporary()); +} + +void ExecutionSerializer::serializeSample(const SampleOperator &sample_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_SAMPLE); + + serialization::SampleOperator *sample = vertex->mutable_sample_operator(); + sample->set_input_relation_id(sample_operator.getIntputRelation().getID()); + sample->set_output_relation_id(sample_operator.getOutputRelationID()); + sample->set_output_destination_index(sample_operator.getInsertDestinationID()); + sample->set_input_relation_is_stored(!sample_operator.getIntputRelation().isTemporary()); + sample->set_is_block_sample(sample_operator.isBlockSample()); + sample->set_percentage(sample_operator.getPercentage()); +} + +void ExecutionSerializer::serializeSaveBlocks(const SaveBlocksOperator &save_blocks_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_SAVE_BLOCKS); + + serialization::SaveBlocksOperator *save_blocks = vertex->mutable_save_blocks_operator(); + serialization::CatalogRelation *relation_catalog = save_blocks->mutable_relation(); + relation_catalog->CopyFrom(save_blocks_operator.getRelation().getProto()); + save_blocks->set_force(save_blocks_operator.isForced()); +} + +void ExecutionSerializer::serializeSelect(const SelectOperator &select_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_SELECT); + + serialization::SelectOperator *select = vertex->mutable_select_operator(); + select->set_input_relation_id(select_operator.input_relation().getID()); + select->set_output_relation_id(select_operator.getOutputRelationID()); + select->set_output_destination_index(select_operator.getInsertDestinationID()); + select->set_predicate_index(select_operator.getPredicateIndex()); + select->set_input_relation_is_stored(!select_operator.input_relation().isTemporary()); + select->set_num_partitions(select_operator.getNumPartitions()); + for (const auto &attribute : select_operator.getSelectionAttributes()) { + select->add_selection_attributes(attribute); + } +} + +void ExecutionSerializer::serializeSortMergeRun(const SortMergeRunOperator &sort_merge_run_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_SORT_MERGE_RUN); + + serialization::SortMergeRunOperator *sort_merge_run = vertex->mutable_sort_merge_run_operator(); + sort_merge_run->set_input_relation_id(sort_merge_run_operator.getInputRelation().getID()); + sort_merge_run->set_output_relation_id(sort_merge_run_operator.getOutputRelationID()); + sort_merge_run->set_output_destination_index(sort_merge_run_operator.getInsertDestinationID()); + sort_merge_run->set_run_relation_id(sort_merge_run_operator.getRunRelation().getID()); + sort_merge_run->set_run_block_destination_index(sort_merge_run_operator.getRunBlockDestinationIndex()); + sort_merge_run->set_sort_config_index(sort_merge_run_operator.getSortConfigIndex()); + sort_merge_run->set_merge_factor(sort_merge_run_operator.getMergeFactor()); + sort_merge_run->set_top_k(sort_merge_run_operator.getTopK()); + sort_merge_run->set_is_input_stored(!sort_merge_run_operator.getInputRelation().isTemporary()); +} + +void ExecutionSerializer::serializeSortRunGeneration(const SortRunGenerationOperator &sort_run_generation_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_SORT_RUN_GENERATION); + + serialization::SortRunGenerationOperator *sort_run_generation = vertex->mutable_sort_run_generation_operator(); + sort_run_generation->set_input_relation_id(sort_run_generation_operator.getInputRelation().getID()); + sort_run_generation->set_output_relation_id(sort_run_generation_operator.getOutputRelationID()); + sort_run_generation->set_output_destination_index(sort_run_generation_operator.getInsertDestinationID()); + sort_run_generation->set_sort_config_index(sort_run_generation_operator.getSortConfigIndex()); + sort_run_generation->set_is_input_stored(!sort_run_generation_operator.getInputRelation().isTemporary()); +} + +void ExecutionSerializer::serializeTableGenerator(const TableGeneratorOperator &table_generator_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_TABLE_GENERATOR); + + serialization::TableGeneratorOperator *table_generator = vertex->mutable_table_generator_operator(); + table_generator->set_output_relation_id(table_generator_operator.getOutputRelationID()); + table_generator->set_output_destination_index(table_generator_operator.getInsertDestinationID()); + table_generator->set_generator_function_index(table_generator_operator.getGeneratorFunctionIndex()); +} + +void ExecutionSerializer::serializeTextScan(const TextScanOperator &text_scan_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_TEXT_SCAN); + + serialization::TextScanOperator *text_scan = vertex->mutable_text_scan_operator(); + text_scan->set_output_relation_id(text_scan_operator.getOutputRelationID()); + text_scan->set_output_destination_index(text_scan_operator.getInsertDestinationID()); + text_scan->set_file_pattern(text_scan_operator.getFilePattern()); + text_scan->set_field_terminator(text_scan_operator.getFieldTerminator()); + text_scan->set_process_escape_sequences(text_scan_operator.getProcessEscapeSequences()); +} + +void ExecutionSerializer::serializeUpdate(const UpdateOperator &update_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_UPDATE); + + serialization::UpdateOperator *update = vertex->mutable_update_operator(); + update->set_predicate_index(update_operator.getPredicateIndex()); + update->set_relation_id(update_operator.getOutputRelationID()); + update->set_relocation_destionation_index(update_operator.getInsertDestinationID()); + update->set_update_group_index(update_operator.getUpdateGroupIndex()); +} + +void ExecutionSerializer::serializeWindowAggregation(const WindowAggregationOperator &window_aggregation_operator) { + serialization::Vertex *vertex = query_plan_proto_.add_vertex(); + vertex->set_operator_type(S::OP_WINDOW_AGGREGATION); + + serialization::WindowAggregationOperator *window_aggregation + = vertex->mutable_window_aggregation_operator(); + window_aggregation->set_output_relation_id( + window_aggregation_operator.getOutputRelationID()); + window_aggregation->set_output_destination_index( + window_aggregation_operator.getInsertDestinationID()); + window_aggregation->set_input_relation_id( + window_aggregation_operator.getInputRelation().getID()); + window_aggregation->set_window_aggregation_state_index( + window_aggregation_operator.getWindowAggregationStateIndex()); +} + + +} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/ExecutionSerializer.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionSerializer.hpp b/query_optimizer/ExecutionSerializer.hpp new file mode 100644 index 0000000..937105a --- /dev/null +++ b/query_optimizer/ExecutionSerializer.hpp @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#ifndef QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_SERIALIZER_HPP_ +#define QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_SERIALIZER_HPP_ + +#include "query_optimizer/QueryPlan.hpp" +#include "relational_operators/AggregationOperator.hpp" +#include "relational_operators/BuildAggregationExistenceMapOperator.hpp" +#include "relational_operators/BuildHashOperator.hpp" +#include "relational_operators/BuildLIPFilterOperator.hpp" +#include "relational_operators/CreateIndexOperator.hpp" +#include "relational_operators/CreateTableOperator.hpp" +#include "relational_operators/DeleteOperator.hpp" +#include "relational_operators/DestroyAggregationStateOperator.hpp" +#include "relational_operators/DestroyHashOperator.hpp" +#include "relational_operators/DropTableOperator.hpp" +#include "relational_operators/FinalizeAggregationOperator.hpp" +#include "relational_operators/HashJoinOperator.hpp" +#include "relational_operators/InitializeAggregationOperator.hpp" +#include "relational_operators/InsertOperator.hpp" +#include "relational_operators/NestedLoopsJoinOperator.hpp" +#include "relational_operators/RelationalOperator.hpp" +#include "relational_operators/SampleOperator.hpp" +#include "relational_operators/SaveBlocksOperator.hpp" +#include "relational_operators/SelectOperator.hpp" +#include "relational_operators/SortMergeRunOperator.hpp" +#include "relational_operators/SortRunGenerationOperator.hpp" +#include "relational_operators/TableGeneratorOperator.hpp" +#include "relational_operators/TextScanOperator.hpp" +#include "relational_operators/UpdateOperator.hpp" +#include "relational_operators/WindowAggregationOperator.hpp" + +#include "query_optimizer/QueryPlan.pb.h" + +namespace quickstep { +namespace optimizer { + +namespace S = quickstep::serialization; + +/** + * @brief + */ +class ExecutionSerializer { + public: + ExecutionSerializer() { + } + + /** + * @brief + * @param query_plan + */ + void serializePlan(const QueryPlan &query_plan); + +private: + void serializeInternal(const RelationalOperator &relational_operator); + + void serializeAggregation(const AggregationOperator &aggregation_operator); + void serializeBuildAggregationExistenceMap(const BuildAggregationExistenceMapOperator &op); + void serializeBuildHash(const BuildHashOperator &build_hash_operator); + void serializeBuildLIPFilter(const BuildLIPFilterOperator &build_lip_filter_operator); + void serializeCreateIndex(const CreateIndexOperator &create_index_operator); + void serializeCreateTable(const CreateTableOperator &create_table_operator); + void serializeDelete(const DeleteOperator &delete_operator); + void serializeDestroyAggregationState(const DestroyAggregationStateOperator &destroy_aggregation_state_operator); + void serializeDestroyHash(const DestroyHashOperator &destroy_hash_operator); + void serializeDropTable(const DropTableOperator &drop_table_operator); + void serializeFinalizeAggregation(const FinalizeAggregationOperator &finalize_aggregation_operator); + void serializeHashJoin(const HashJoinOperator &finalize_hash_join); + void serializeInitializeAggregation(const InitializeAggregationOperator &initialize_aggregation_operator); + void serializeInsert(const InsertOperator &insert_operator); + void serializeNestedLoopsJoin(const NestedLoopsJoinOperator &nested_loops_join_operator); + void serializeSample(const SampleOperator &sample_operator); + void serializeSaveBlocks(const SaveBlocksOperator &save_blocks_operator); + void serializeSelect(const SelectOperator &select_operator); + void serializeSortMergeRun(const SortMergeRunOperator &sort_merge_run_operator); + void serializeSortRunGeneration(const SortRunGenerationOperator &sort_run_generation_operator); + void serializeTableGenerator(const TableGeneratorOperator &table_generator_operator); + void serializeTextScan(const TextScanOperator &text_scan_operator); + void serializeUpdate(const UpdateOperator &update_operator); + void serializeWindowAggregation(const WindowAggregationOperator &window_aggregation_operator); + + S::QueryPlan query_plan_proto_; +}; + +} // namespace optimizer +} // namespace quickstep + +#endif // QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_SERIALIZER_HPP_ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/Optimizer.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/Optimizer.cpp b/query_optimizer/Optimizer.cpp index 9ce517f..35ed4ba 100644 --- a/query_optimizer/Optimizer.cpp +++ b/query_optimizer/Optimizer.cpp @@ -37,10 +37,11 @@ void Optimizer::generateQueryHandle(const ParseStatement &parse_statement, physical_generator.generatePlan( logical_generator.generatePlan(*catalog_database, parse_statement))); - // TODO(Hakan): Break at this point. + // TODO(hakan): Break at this point. // execution_serializer.serializePlan( + // execution_generator.generatePlan( // physical_generator.generatePlan( - // logical_generator.generatePlan(*catalog_database, parse_statement))); + // logical_generator.generatePlan(*catalog_database, parse_statement)))); } } // namespace optimizer http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/query_optimizer/QueryPlan.proto ---------------------------------------------------------------------- diff --git a/query_optimizer/QueryPlan.proto b/query_optimizer/QueryPlan.proto index a255b34..2747111 100644 --- a/query_optimizer/QueryPlan.proto +++ b/query_optimizer/QueryPlan.proto @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + syntax = "proto2"; package quickstep.serialization; @@ -5,40 +22,45 @@ package quickstep.serialization; import "relational_operators/Operator.proto"; message Vertex { - required OperatorType = 1; + required OperatorType operator_type = 1; oneof operator { - AggregationOperator aggregate_operator = 2; - BuildHashOperator build_hash_operator = 3; - BuildLIPFilterOperator build_lip_filter_operator = 4; - CreateIndexOperator create_index_operator = 5; - CreateTableOperator create_table_operator = 6; - DeleteOperator delete_operator = 7; - DestroyAggregationStateOperator destroy_aggregation_state_operator = 8; - DestroyHashOperator destroy_hash_operator = 9; - DropTableOperator drop_table_operator = 10; - FinalizeAggregationOperator finalize_aggregation_operator = 11; - HashJoinOperator hash_join_operator = 12; - InitializeAggregationOperator initialize_aggregation_operator = 13; - InsertOperator insert_operator = 14; - NestedLoopsJoinOperator nested_loops_join_operator = 15; - SampleOperator sample_operator = 16; - SaveBlocksOperator save_blocks_operator = 17; - SelectOperator select_operator = 18; - SortMergeRunOperator sort_merge_run_operator = 19; - SortRunGenerationOperator sort_run_generation_operator = 20; - TableGeneratorOperator table_generator_operator = 21; - TextScanOperator text_scan_operator = 22; - UpdateOperator update_operator = 23; - WindowAggregationOperator window_aggregation_operator = 24; + AggregationOperator aggregation_operator = 2; + BuildAggregationExistenceMapOperator build_aggregation_existence_map_operator = 3; + BuildHashOperator build_hash_operator = 4; + BuildLIPFilterOperator build_lip_filter_operator = 5; + CreateIndexOperator create_index_operator = 6; + CreateTableOperator create_table_operator = 7; + DeleteOperator delete_operator = 8; + DestroyAggregationStateOperator destroy_aggregation_state_operator = 9; + DestroyHashOperator destroy_hash_operator = 10; + DropTableOperator drop_table_operator = 11; + FinalizeAggregationOperator finalize_aggregation_operator = 12; + HashJoinOperator hash_join_operator = 13; + InitializeAggregationOperator initialize_aggregation_operator = 14; + InsertOperator insert_operator = 15; + NestedLoopsJoinOperator nested_loops_join_operator = 16; + SampleOperator sample_operator = 17; + SaveBlocksOperator save_blocks_operator = 18; + SelectOperator select_operator = 19; + SortMergeRunOperator sort_merge_run_operator = 20; + SortRunGenerationOperator sort_run_generation_operator = 21; + TableGeneratorOperator table_generator_operator = 22; + TextScanOperator text_scan_operator = 23; + UpdateOperator update_operator = 24; + WindowAggregationOperator window_aggregation_operator = 25; } } -message Edges { - repeated uint64 outgoing_id = 1; - repeated bool payload = 2; +message Edge { + required uint64 outgoing_id = 1; + required bool payload = 2; +} + +message OutgoingEdges { + repeated Edge edges = 1; } -message DAG { - repeated Node = 1; - repeated Edges = 2; +message QueryPlan { + repeated Vertex vertex = 1; + repeated OutgoingEdges outgoing = 2; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/AggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp index 93f4550..4bb2823 100644 --- a/relational_operators/AggregationOperator.hpp +++ b/relational_operators/AggregationOperator.hpp @@ -107,6 +107,10 @@ class AggregationOperator : public RelationalOperator { input_relation_block_ids_.push_back(input_block_id); } + QueryContext::aggregation_state_id getAggregationStateIndex() const { + return aggr_state_index_; + } + private: /** * @brief Create Work Order proto. @@ -119,7 +123,6 @@ class AggregationOperator : public RelationalOperator { const bool input_relation_is_stored_; std::vector<block_id> input_relation_block_ids_; const QueryContext::aggregation_state_id aggr_state_index_; - std::vector<block_id>::size_type num_workorders_generated_; bool started_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/BuildAggregationExistenceMapOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildAggregationExistenceMapOperator.hpp b/relational_operators/BuildAggregationExistenceMapOperator.hpp index b28b0b4..90580fe 100644 --- a/relational_operators/BuildAggregationExistenceMapOperator.hpp +++ b/relational_operators/BuildAggregationExistenceMapOperator.hpp @@ -116,6 +116,14 @@ class BuildAggregationExistenceMapOperator : public RelationalOperator { input_relation_block_ids_.push_back(input_block_id); } + QueryContext::aggregation_state_id getAggregationStateIndex() const { + return aggr_state_index_; + } + + attribute_id getBuildAttributeID() const { + return build_attribute_; + } + private: serialization::WorkOrder* createWorkOrderProto(const block_id block); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/BuildHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp index 634e1dd..a250dba 100644 --- a/relational_operators/BuildHashOperator.hpp +++ b/relational_operators/BuildHashOperator.hpp @@ -139,6 +139,22 @@ class BuildHashOperator : public RelationalOperator { input_relation_block_ids_[part_id].push_back(input_block_id); } + std::vector<attribute_id> getJoinKeyAttributes() const { + return join_key_attributes_; + } + + QueryContext::join_hash_table_id getHashTableIndex() const { + return hash_table_index_; + } + + std::size_t getNumberOfPartitions() const { + return num_partitions_; + } + + bool anyJoinKeyNullable() const { + return any_join_key_attributes_nullable_; + } + private: /** * @brief Create Work Order proto. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/BuildLIPFilterOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildLIPFilterOperator.hpp b/relational_operators/BuildLIPFilterOperator.hpp index 68bd307..d37f639 100644 --- a/relational_operators/BuildLIPFilterOperator.hpp +++ b/relational_operators/BuildLIPFilterOperator.hpp @@ -120,6 +120,10 @@ class BuildLIPFilterOperator : public RelationalOperator { input_relation_block_ids_.push_back(input_block_id); } + QueryContext::predicate_id getPredicateIndex() const { + return build_side_predicate_index_; + } + private: /** * @brief Create Work Order proto. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/CreateIndexOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/CreateIndexOperator.hpp b/relational_operators/CreateIndexOperator.hpp index c08947f..7c1e3e4 100644 --- a/relational_operators/CreateIndexOperator.hpp +++ b/relational_operators/CreateIndexOperator.hpp @@ -96,6 +96,18 @@ class CreateIndexOperator : public RelationalOperator { void updateCatalogOnCompletion() override; + IndexSubBlockDescription getIndexDescription() const { + return index_description_; + } + + const CatalogRelation& getRelation() const { + return *relation_; + } + + const std::string& getIndexName() const { + return index_name_; + } + private: CatalogRelation *relation_; const std::string index_name_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/CreateTableOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/CreateTableOperator.hpp b/relational_operators/CreateTableOperator.hpp index 4b06abc..d196c79 100644 --- a/relational_operators/CreateTableOperator.hpp +++ b/relational_operators/CreateTableOperator.hpp @@ -95,6 +95,10 @@ class CreateTableOperator : public RelationalOperator { void updateCatalogOnCompletion() override; + const CatalogRelation& getRelation() const { + return *relation_; + } + private: std::unique_ptr<CatalogRelation> relation_; CatalogDatabase *database_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/DeleteOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp index aed37f6..e97cdc8 100644 --- a/relational_operators/DeleteOperator.hpp +++ b/relational_operators/DeleteOperator.hpp @@ -110,6 +110,10 @@ class DeleteOperator : public RelationalOperator { relation_block_ids_.push_back(input_block_id); } + QueryContext::predicate_id getPredicateIndex() const { + return predicate_index_; + } + private: /** * @brief Create Work Order proto. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/DestroyAggregationStateOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DestroyAggregationStateOperator.hpp b/relational_operators/DestroyAggregationStateOperator.hpp index 70ab45c..5dd5fb5 100644 --- a/relational_operators/DestroyAggregationStateOperator.hpp +++ b/relational_operators/DestroyAggregationStateOperator.hpp @@ -79,6 +79,10 @@ class DestroyAggregationStateOperator : public RelationalOperator { bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override; + QueryContext::aggregation_state_id getAggregationStateIndex() const { + return aggr_state_index_; + }; + private: const QueryContext::aggregation_state_id aggr_state_index_; bool work_generated_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/DestroyHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DestroyHashOperator.hpp b/relational_operators/DestroyHashOperator.hpp index a65e739..300b22b 100644 --- a/relational_operators/DestroyHashOperator.hpp +++ b/relational_operators/DestroyHashOperator.hpp @@ -82,6 +82,14 @@ class DestroyHashOperator : public RelationalOperator { bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override; + std::size_t getNumPartitions() const { + return build_num_partitions_; + } + + QueryContext::join_hash_table_id getHashTableIndex() const { + return hash_table_index_; + } + private: const std::size_t build_num_partitions_; const QueryContext::join_hash_table_id hash_table_index_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/DropTableOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DropTableOperator.hpp b/relational_operators/DropTableOperator.hpp index 99d6e7f..8fabbfe 100644 --- a/relational_operators/DropTableOperator.hpp +++ b/relational_operators/DropTableOperator.hpp @@ -25,6 +25,7 @@ #include <utility> #include <vector> +#include "catalog/CatalogRelation.hpp" #include "catalog/CatalogTypedefs.hpp" #include "relational_operators/RelationalOperator.hpp" #include "relational_operators/WorkOrder.hpp" @@ -41,7 +42,6 @@ namespace quickstep { class CatalogDatabase; class CatalogDatabaseLite; -class CatalogRelation; class QueryContext; class StorageManager; class WorkOrderProtosContainer; @@ -95,6 +95,14 @@ class DropTableOperator : public RelationalOperator { void updateCatalogOnCompletion() override; + relation_id getRelationID() const { + return relation_.getID(); + } + + bool onlyDropBlocks() const { + return only_drop_blocks_; + } + private: const CatalogRelation &relation_; CatalogDatabase *database_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/FinalizeAggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp index 87533af..9bb896e 100644 --- a/relational_operators/FinalizeAggregationOperator.hpp +++ b/relational_operators/FinalizeAggregationOperator.hpp @@ -100,6 +100,10 @@ class FinalizeAggregationOperator : public RelationalOperator { return output_relation_.getID(); } + QueryContext::aggregation_state_id getAggregationStateIndex() const { + return aggr_state_index_; + } + private: const QueryContext::aggregation_state_id aggr_state_index_; const CatalogRelation &output_relation_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/HashJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp index 8e9f2d7..a8e60d6 100644 --- a/relational_operators/HashJoinOperator.hpp +++ b/relational_operators/HashJoinOperator.hpp @@ -246,6 +246,34 @@ class HashJoinOperator : public RelationalOperator { return output_relation_.getID(); } + const std::vector<attribute_id>& getJoinKeyAttributes() const { + return join_key_attributes_; + } + + bool anyJoinKeyNullable() const { + return any_join_key_attributes_nullable_; + } + + std::size_t getBuildNumPartitions() const { + return build_num_partitions_; + } + + QueryContext::join_hash_table_id getJoinHashTableIndex() const { + return hash_table_index_; + } + + QueryContext::predicate_id getResidualPredicateIndex() const { + return residual_predicate_index_; + } + + QueryContext::scalar_group_id getSelectionIndex() const { + return selection_index_; + } + + const std::vector<bool>& getSelectionsOnBuild() const { + return is_selection_on_build_; + } + void doneFeedingInputBlocks(const relation_id rel_id) override { // The HashJoinOperator depends on BuildHashOperator too, but it // should ignore a doneFeedingInputBlocks() message that comes http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/InitializeAggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/InitializeAggregationOperator.hpp b/relational_operators/InitializeAggregationOperator.hpp index e81264a..77b45ea 100644 --- a/relational_operators/InitializeAggregationOperator.hpp +++ b/relational_operators/InitializeAggregationOperator.hpp @@ -81,6 +81,10 @@ class InitializeAggregationOperator : public RelationalOperator { bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override; + QueryContext::aggregation_state_id getAggregationStateIndex() const { + return aggr_state_index_; + } + private: const QueryContext::aggregation_state_id aggr_state_index_; bool started_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/InsertOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp index b0bdf87..c76a83e 100644 --- a/relational_operators/InsertOperator.hpp +++ b/relational_operators/InsertOperator.hpp @@ -100,6 +100,10 @@ class InsertOperator : public RelationalOperator { return output_relation_.getID(); } + QueryContext::tuple_id getTupleIndex() const { + return tuple_index_; + } + private: const CatalogRelation &output_relation_; const QueryContext::insert_destination_id output_destination_index_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/NestedLoopsJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp index 3012114..bf53bc2 100644 --- a/relational_operators/NestedLoopsJoinOperator.hpp +++ b/relational_operators/NestedLoopsJoinOperator.hpp @@ -165,6 +165,22 @@ class NestedLoopsJoinOperator : public RelationalOperator { return output_relation_.getID(); } + const CatalogRelation& getLeftInputRelation() const { + return left_input_relation_; + }; + + const CatalogRelation& getRightInputRelation() const { + return right_input_relation_; + } + + QueryContext::predicate_id getJoinPredicateIndex() const { + return join_predicate_index_; + } + + QueryContext::scalar_group_id getSelectionIndex() const { + return selection_index_; + } + private: /** * @brief Pairs block IDs from left and right relation block IDs and generates http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/Operator.proto ---------------------------------------------------------------------- diff --git a/relational_operators/Operator.proto b/relational_operators/Operator.proto index d96f183..164ab35 100644 --- a/relational_operators/Operator.proto +++ b/relational_operators/Operator.proto @@ -1,95 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + syntax = "proto2"; package quickstep.serialization; +import "storage/StorageBlockLayout.proto"; +import "catalog/Catalog.proto"; + enum OperatorType { - AGGREGATION = 1; - BUILD_AGGREGATION_EXISTENCE_MAP = 2; - BUILD_HASH = 3; - BUILD_LIP_FILTER = 4; - CREATE_INDEX = 5; // Placeholder. - CREATE_TABLE = 6; // Placeholder. - DELETE = 7; - DESTROY_AGGREGATION_STATE = 8; - DESTROY_HASH = 9; - DROP_TABLE = 10; - FINALIZE_AGGREGATION = 11; - HASH_JOIN = 12; - INITIALIZE_AGGREGATION = 13; - INSERT = 14; - NESTED_LOOP_JOIN = 15; - SAMPLE = 16; - SAVE_BLOCKS = 17; - SELECT = 18; - SORT_MERGE_RUN = 19; - SORT_RUN_GENERATION = 20; - TABLE_GENERATOR = 21; - TEXT_SCAN = 22; - UPDATE = 23; - WINDOW_AGGREGATION = 24; -} - -message Operator { - required uint64 query_id = 1; - required OperatorType operator_type = 2; - extensions 16 to max; + OP_AGGREGATION = 1; + OP_BUILD_AGGREGATION_EXISTENCE_MAP = 2; + OP_BUILD_HASH = 3; + OP_BUILD_LIP_FILTER = 4; + OP_CREATE_INDEX = 5; // Placeholder. + OP_CREATE_TABLE = 6; // Placeholder. + OP_DELETE = 7; + OP_DESTROY_AGGREGATION_STATE = 8; + OP_DESTROY_HASH= 9; + OP_DROP_TABLE = 10; + OP_FINALIZE_AGGREGATION = 11; + OP_HASH_JOIN = 12; + OP_INITIALIZE_AGGREGATION = 13; + OP_INSERT = 14; + OP_NESTED_LOOP_JOIN = 15; + OP_SAMPLE = 16; + OP_SAVE_BLOCKS = 17; + OP_SELECT = 18; + OP_SORT_MERGE_RUN = 19; + OP_SORT_RUN_GENERATION = 20; + OP_TABLE_GENERATOR = 21; + OP_TEXT_SCAN = 22; + OP_UPDATE = 23; + OP_WINDOW_AGGREGATION = 24; } message AggregationOperator { - extend Operator { - // We need relation_id to retrieve CatalogRelation object. (?) - required int32 relation_id = 16; - // Learn whether we can check this one from relation id. (?) - required bool relation_is_stored = 17; - // Aggregate state index in QueryContext - required uint32 aggr_state_index = 18; - } + // We need relation_id to retrieve CatalogRelation object. (?) + required int32 relation_id = 1; + // Learn whether we can check this one from relation id. (?) + required bool relation_is_stored = 2; + // Aggregate state index in QueryContext + required uint32 aggr_state_index = 3; +} + +message BuildAggregationExistenceMapOperator { + required int32 relation_id = 1; + required int32 attribute_id = 2; + required bool relation_is_stored = 3; + required uint32 aggr_state_index = 4; } message BuildHashOperator { - extend Operator { - required int32 relation_id = 16; - required bool relation_is_stored = 17; - repeated int32 join_key_attributes = 18; - required bool any_join_key_nullable = 19; - required uint64 num_partitions = 20; - required uint32 hash_table_index = 21; - } + required int32 relation_id = 1; + required bool relation_is_stored = 2; + repeated int32 join_key_attributes = 3; + required bool any_join_key_nullable = 4; + required uint64 num_partitions = 5; + required uint32 hash_table_index = 6; } -message CreateTableOperator { - extend Operator { +message BuildLIPFilterOperator { + required int32 relation_id = 1; + required int32 build_side_predicate_index = 2; + required bool relation_is_stored = 3; +} - } +message CreateIndexOperator { + required int32 relation_id = 1; + required string index_name = 2; + required IndexSubBlockDescription index_description = 3; +} + +message CreateTableOperator { + required int32 database_id = 1; + required CatalogRelation relation_catalog = 2; } message DeleteOperator { - extend Operator { + required int32 relation_id = 1; + required int32 predicate_index = 2; +} - } +message DestroyAggregationStateOperator { + required uint32 aggr_state_index = 1; } message DestroyHashOperator { - extend Operator { - required uint64 build_num_partitions = 16; - required uint32 hash_table_index = 17; - } + required uint64 build_num_partitions = 1; + required uint32 hash_table_index = 2; } message DropTableOperator { - extend Operator { - required int32 relation_id = 16; - required int32 database_id = 17; - required bool only_drop_blocks = 18; - } + required int32 relation_id = 1; + required int32 database_id = 2; + required bool only_drop_blocks = 3; } message FinalizeAggregationOperator { - extend Operator { - required uint32 aggr_state_index = 16; - required int32 output_relation_id = 17; - required uint32 output_destination_index = 18; - } + required uint32 aggr_state_index = 1; + required int32 output_relation_id = 2; + required int32 output_destination_index = 3; } @@ -100,99 +125,111 @@ message HashJoinOperator { LEFT_ANTI = 3; LEFT_OUTER = 4; } - - extend Operator { - required int32 build_relation_id = 16; - required int32 probe_relation_id = 17; - repeated int32 join_key_attributes = 18; - required bool any_join_key_attributes_nullable = 19; - required uint64 build_num_partitions = 20; - required int32 output_relation_id = 21; - required int32 output_destionation_index = 22; - required int32 hash_table_index = 23; - required int32 residiual_predicate_index = 24; - repeated bool is_selection_on_build = 25; - required JoinType join_type = 26; - } + required int32 build_relation_id = 1; + required int32 probe_relation_id = 2; + required bool is_probe_relation_stored = 3; + repeated int32 join_key_attributes = 4; + required bool any_join_key_attributes_nullable = 5; + required uint64 build_num_partitions = 6; + required int32 output_relation_id = 7; + required int32 output_destionation_index = 8; + required int32 hash_table_index = 9; + required int32 residiual_predicate_index = 10; + required int32 selection_index = 11; + repeated bool is_selection_on_build = 12; + required JoinType join_type = 13; } message InitializeAggregationOperator { - extend Operator { - required uint32 aggr_state_index = 16; - } + required uint32 aggr_state_index = 1; } message InsertOperator { - extend Operator { - required int32 output_relation_id = 16; - required uint32 output_destination_index = 17; - required uint32 tuple_index = 18; - } + required int32 output_relation_id = 1; + required int32 output_destination_index = 2; + required uint32 tuple_index = 3; +} + +message NestedLoopsJoinOperator { + required int32 left_input_relation_id = 1; + required int32 right_input_relation_id = 2; + required int32 output_relation_id = 3; + required int32 output_destination_index = 4; + required int32 join_predicate_index = 5; + required int32 selection_index = 6; + required bool left_relation_is_stored = 7; + required bool right_relation_is_stored = 8; } message SampleOperator { - extend Operator { - required int32 input_relation_id = 16; - required int32 output_relation_id = 17; - required int32 output_destination_index = 18; - required bool input_relation_is_stored = 19; - required bool is_block_sample = 20; - required int32 percentage = 21; - } + required int32 input_relation_id = 1; + required int32 output_relation_id = 2; + required int32 output_destination_index = 3; + required bool input_relation_is_stored = 4; + required bool is_block_sample = 5; + required int32 percentage = 6; } message SaveBlocksOperator { - extend Operator { - required int32 relation_id = 16; - required bool force = 17; - } + required CatalogRelation relation = 1; + required bool force = 2; } message SelectOperator { - extend Operator { - required int32 input_relation_id = 16; - required int32 output_relation_id = 17; - required int32 output_destination_index = 18; - required int32 predicate_index = 19; - required int32 selection_index = 20; - required bool input_relation_is_stored = 21; - required uint64 num_partitions = 22; - } + required int32 input_relation_id = 1; + required int32 output_relation_id = 2; + required int32 output_destination_index = 3; + required int32 predicate_index = 4; + repeated int32 selection_attributes = 5; + required bool input_relation_is_stored = 6; + required uint64 num_partitions = 7; +} + +message SortMergeRunOperator { + required int32 input_relation_id = 1; + required int32 output_relation_id = 2; + required int32 output_destination_index = 3; + required int32 run_relation_id = 4; + required int32 run_block_destination_index = 5; + required uint32 sort_config_index = 6; + required uint64 merge_factor = 7; + required uint64 top_k = 8; + required bool is_input_stored = 9; +} + +message SortRunGenerationOperator { + required int32 input_relation_id = 1; + required int32 output_relation_id = 2; + required int32 output_destination_index = 3; + required int32 sort_config_index = 4; + required bool is_input_stored = 5; } message TableGeneratorOperator { - extend Operator { - required int32 output_relation_id = 16; - required int32 output_destination_index = 17; - required uint32 generator_function_index = 18; - } + required int32 output_relation_id = 1; + required int32 output_destination_index = 2; + required uint32 generator_function_index = 3; } message TextScanOperator { - extend Operator { - required string file_pattern = 16; - required int32 field_terminator = 17; - required bool process_escap_sequences = 18; - required int32 output_relation_id = 19; - required int32 output_destination_index = 20; - } + required string file_pattern = 1; + required int32 field_terminator = 2; + required bool process_escape_sequences = 3; + required int32 output_relation_id = 4; + required int32 output_destination_index = 5; } message UpdateOperator { - extend Operator { - required int32 relation_id = 16; - required int32 relocation_destionation_index = 17; - required int32 predicate_index = 18; - required uint32 update_group_index = 19; - } + required int32 relation_id = 1; + required int32 relocation_destionation_index = 2; + required int32 predicate_index = 3; + required uint32 update_group_index = 4; } message WindowAggregationOperator { - extend Operator { - required int32 input_relation_id = 16; - required int32 output_relation_id = 17; - required uint32 window_aggregation_state_index = 18; - required int32 output_destination_index = 19; - } + required int32 input_relation_id = 1; + required int32 output_relation_id = 2; + required uint32 window_aggregation_state_index = 3; + required int32 output_destination_index = 4; } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/SampleOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp index 331c85d..ddcb2a1 100644 --- a/relational_operators/SampleOperator.hpp +++ b/relational_operators/SampleOperator.hpp @@ -125,6 +125,18 @@ class SampleOperator : public RelationalOperator { return output_relation_.getID(); } + const CatalogRelation& getIntputRelation() const { + return input_relation_; + } + + bool isBlockSample() const { + return is_block_sample_; + } + + int getPercentage() const { + return percentage_; + } + private: /** * @brief Create Work Order proto. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/SaveBlocksOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp index 4489299..8e79087 100644 --- a/relational_operators/SaveBlocksOperator.hpp +++ b/relational_operators/SaveBlocksOperator.hpp @@ -94,6 +94,14 @@ class SaveBlocksOperator : public RelationalOperator { void updateCatalogOnCompletion() override; + const CatalogRelation& getRelation() const { + return *relation_; + } + + bool isForced() const { + return force_; + } + private: const bool force_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/SelectOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp index b8161b7..0a722c4 100644 --- a/relational_operators/SelectOperator.hpp +++ b/relational_operators/SelectOperator.hpp @@ -222,6 +222,18 @@ class SelectOperator : public RelationalOperator { return output_relation_.getID(); } + QueryContext::predicate_id getPredicateIndex() const { + return predicate_index_; + } + + std::size_t getNumPartitions() const { + return num_partitions_; + } + + const std::vector<attribute_id>& getSelectionAttributes() const { + return simple_selection_; + } + private: /** * @brief Create Work Order proto. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/SortMergeRunOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp index a8ecb88..a159fad 100644 --- a/relational_operators/SortMergeRunOperator.hpp +++ b/relational_operators/SortMergeRunOperator.hpp @@ -168,6 +168,30 @@ class SortMergeRunOperator : public RelationalOperator { return output_relation_.getID(); } + const CatalogRelation& getInputRelation() const { + return input_relation_; + } + + const CatalogRelation& getRunRelation() const { + return run_relation_; + } + + QueryContext::insert_destination_id getRunBlockDestinationIndex() const { + return run_block_destination_index_; + } + + QueryContext::sort_config_id getSortConfigIndex() const { + return sort_config_index_; + } + + std::size_t getTopK() const { + return top_k_; + } + + std::size_t getMergeFactor() const { + return merge_factor_; + } + private: // Initialize runs for the first pass. This needs to be called every time new // blocks are input to the operator in the case of pipelined input. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/SortRunGenerationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp index 716d4f7..834120f 100644 --- a/relational_operators/SortRunGenerationOperator.hpp +++ b/relational_operators/SortRunGenerationOperator.hpp @@ -142,6 +142,14 @@ class SortRunGenerationOperator : public RelationalOperator { return output_relation_.getID(); } + const CatalogRelation& getInputRelation() const { + return input_relation_; + } + + QueryContext::sort_config_id getSortConfigIndex() const { + return sort_config_index_; + } + private: /** * @brief Create Work Order proto. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/TableGeneratorOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp index adfe2bb..569b935 100644 --- a/relational_operators/TableGeneratorOperator.hpp +++ b/relational_operators/TableGeneratorOperator.hpp @@ -102,6 +102,10 @@ class TableGeneratorOperator : public RelationalOperator { return output_relation_.getID(); } + QueryContext::generator_function_id getGeneratorFunctionIndex() const { + return generator_function_index_; + } + private: const CatalogRelation &output_relation_; const QueryContext::insert_destination_id output_destination_index_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/TextScanOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp index f6be8c8..09da6ff 100644 --- a/relational_operators/TextScanOperator.hpp +++ b/relational_operators/TextScanOperator.hpp @@ -159,6 +159,18 @@ class TextScanOperator : public RelationalOperator { return output_relation_.getID(); } + const std::string& getFilePattern() const { + return file_pattern_; + } + + char getFieldTerminator() const { + return field_terminator_; + } + + bool getProcessEscapeSequences() const { + return process_escape_sequences_; + } + private: serialization::WorkOrder* createWorkOrderProto(const std::string &filename, const std::size_t text_offset, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/UpdateOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp index 8e020d8..26812f4 100644 --- a/relational_operators/UpdateOperator.hpp +++ b/relational_operators/UpdateOperator.hpp @@ -121,6 +121,14 @@ class UpdateOperator : public RelationalOperator { return relation_.getID(); } + QueryContext::update_group_id getUpdateGroupIndex() const { + return update_group_index_; + } + + QueryContext::predicate_id getPredicateIndex() const { + return predicate_index_; + } + private: const CatalogRelation &relation_; const QueryContext::insert_destination_id relocation_destination_index_; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/96908588/relational_operators/WindowAggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/WindowAggregationOperator.hpp b/relational_operators/WindowAggregationOperator.hpp index 10546b4..afffcd0 100644 --- a/relational_operators/WindowAggregationOperator.hpp +++ b/relational_operators/WindowAggregationOperator.hpp @@ -103,6 +103,14 @@ class WindowAggregationOperator : public RelationalOperator { return output_destination_index_; } + QueryContext::window_aggregation_state_id getWindowAggregationStateIndex() const { + return window_aggregation_state_index_; + } + + const CatalogRelation& getInputRelation() const { + return input_relation_; + } + private: /** * @brief Create Work Order proto.