This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
new 3b8e78d196 [Enhancement](table_function) table function node
enhancement (#12206)
3b8e78d196 is described below
commit 3b8e78d19659d769efd7affbecb44efa18578ad7
Author: camby <[email protected]>
AuthorDate: Wed Aug 31 11:03:48 2022 +0800
[Enhancement](table_function) table function node enhancement (#12206)
* table function node enhancement
* also avoid copy for non-vec table function node
* fix table function node output slots calculation while lateral view
involves subquery
Co-authored-by: cambyzju <[email protected]>
---
be/src/exec/table_function_node.cpp | 29 ++++++++++++++--------
be/src/vec/exec/vtable_function_node.cpp | 6 ++++-
be/src/vec/exec/vtable_function_node.h | 22 +++++++++++++++-
.../apache/doris/planner/TableFunctionNode.java | 13 +++++++++-
4 files changed, 57 insertions(+), 13 deletions(-)
diff --git a/be/src/exec/table_function_node.cpp
b/be/src/exec/table_function_node.cpp
index ff86df9601..e9d69ba3e5 100644
--- a/be/src/exec/table_function_node.cpp
+++ b/be/src/exec/table_function_node.cpp
@@ -279,16 +279,25 @@ Status TableFunctionNode::get_next(RuntimeState* state,
RowBatch* row_batch, boo
memcpy(tuple_ptr, child_tuple,
parent_tuple_desc->byte_size());
// only deep copy the child slot if it is selected and is
var len (Eg: string, bitmap, hll)
for (int j = 0; j < _child_slot_sizes[i]; ++j) {
- SlotDescriptor *child_slot_desc =
child_tuple_desc->slots()[j];
- SlotDescriptor *parent_slot_desc =
parent_tuple_desc->slots()[j];
-
- if (_output_slot_ids[parent_slot_desc->id()] &&
-
!child_tuple->is_null(child_slot_desc->null_indicator_offset())
- && child_slot_desc->type().is_string_type()) {
- void *dest_slot =
tuple_ptr->get_slot(parent_slot_desc->tuple_offset());
-
RawValue::write(child_tuple->get_slot(child_slot_desc->tuple_offset()),
- dest_slot,
parent_slot_desc->type(),
- row_batch->tuple_data_pool());
+ SlotDescriptor* child_slot_desc =
child_tuple_desc->slots()[j];
+ SlotDescriptor* parent_slot_desc =
parent_tuple_desc->slots()[j];
+
+ if
(child_tuple->is_null(child_slot_desc->null_indicator_offset())) {
+ continue;
+ }
+ if (child_slot_desc->type().is_string_type()) {
+ void* dest_slot =
tuple_ptr->get_slot(parent_slot_desc->tuple_offset());
+ if (_output_slot_ids[parent_slot_desc->id()]) {
+ // deep coopy
+ RawValue::write(
+
child_tuple->get_slot(child_slot_desc->tuple_offset()),
+ dest_slot, parent_slot_desc->type(),
+ row_batch->tuple_data_pool());
+ } else {
+ // clear for unused slot
+ StringValue* dest =
reinterpret_cast<StringValue*>(dest_slot);
+ dest->replace(nullptr, 0);
+ }
}
}
parent_tuple_row->set_tuple(tuple_idx, tuple_ptr);
diff --git a/be/src/vec/exec/vtable_function_node.cpp
b/be/src/vec/exec/vtable_function_node.cpp
index 30dd0097f6..e38d5c3726 100644
--- a/be/src/vec/exec/vtable_function_node.cpp
+++ b/be/src/vec/exec/vtable_function_node.cpp
@@ -156,6 +156,10 @@ Status
VTableFunctionNode::get_expanded_block(RuntimeState* state, Block* output
// 1. copy data from child_block.
for (int i = 0; i < _child_slots.size(); i++) {
+ if (!slot_need_copy(i)) {
+ columns[i]->insert_default();
+ continue;
+ }
auto src_column = _child_block->get_by_position(i).column;
columns[i]->insert_from(*src_column, _cur_child_offset);
}
@@ -221,4 +225,4 @@ Status VTableFunctionNode::_process_next_child_row() {
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/vtable_function_node.h
b/be/src/vec/exec/vtable_function_node.h
index 1913cd1d35..72108fa1d9 100644
--- a/be/src/vec/exec/vtable_function_node.h
+++ b/be/src/vec/exec/vtable_function_node.h
@@ -33,6 +33,26 @@ public:
private:
Status _process_next_child_row() override;
+ /* Now the output tuples for table function node is base_table_tuple +
tf1 + tf2 + ...
+ But not all slots are used, the real used slots are inside
table_function_node.outputSlotIds.
+ For case like explode_bitmap:
+ SELECT a2,count(*) as a3 FROM A WHERE a1 IN
+ (SELECT c1 FROM B LATERAL VIEW explode_bitmap(b1) C as c1)
+ GROUP BY a2 ORDER BY a3;
+ Actually we only need to output column c1, no need to output columns
in bitmap table B.
+ Copy large bitmap columns are very expensive and slow.
+
+ Here we check if the slot is realy used, otherwise we avoid copy it
and just insert a default value.
+
+ A better solution is:
+ 1. FE: create a new output tuple based on the real output slots;
+ 2. BE: refractor (V)TableFunctionNode output rows based no the new
tuple;
+ */
+ inline bool slot_need_copy(SlotId slot_id) const {
+ auto id = _output_slots[slot_id]->id();
+ return (id < _output_slot_ids.size()) && (_output_slot_ids[id]);
+ }
+
using TableFunctionNode::get_next;
Status get_expanded_block(RuntimeState* state, Block* output_block, bool*
eos);
@@ -42,4 +62,4 @@ private:
std::vector<SlotDescriptor*> _output_slots;
};
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
index c09a0fdd37..d430539b03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
@@ -40,7 +40,6 @@ import java.util.Set;
import java.util.stream.Collectors;
public class TableFunctionNode extends PlanNode {
-
private List<LateralViewRef> lateralViewRefs;
private ArrayList<Expr> fnCallExprList;
private List<TupleId> lateralViewTupleIds;
@@ -99,11 +98,23 @@ public class TableFunctionNode extends PlanNode {
for (Expr resultExpr : baseTblResultExprs) {
// find all slotRef bound by tupleIds in resultExpr
resultExpr.getSlotRefsBoundByTupleIds(tupleIds, outputSlotRef);
+
+ // For vec engine while lateral view involves subquery
+ Expr dst = outputSmap.get(resultExpr);
+ if (dst != null) {
+ dst.getSlotRefsBoundByTupleIds(tupleIds, outputSlotRef);
+ }
}
// case2
List<Expr> remainConjuncts = analyzer.getRemainConjuncts(tupleIds);
for (Expr expr : remainConjuncts) {
expr.getSlotRefsBoundByTupleIds(tupleIds, outputSlotRef);
+
+ // For vec engine while lateral view involves subquery
+ Expr dst = outputSmap.get(expr);
+ if (dst != null) {
+ dst.getSlotRefsBoundByTupleIds(tupleIds, outputSlotRef);
+ }
}
// set output slot ids
for (SlotRef slotRef : outputSlotRef) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
