This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7f73749b88 [Bug](pipeline) fix distributionColumnIds not updated
correct when outputColumnUnique… (#19704)
7f73749b88 is described below
commit 7f73749b887961e5ce3f83b10f525ee955284363
Author: Pxl <[email protected]>
AuthorDate: Wed May 17 00:13:10 2023 +0800
[Bug](pipeline) fix distributionColumnIds not updated correct when
outputColumnUnique… (#19704)
fix distributionColumnIds not updated correct when outputColumnUnique
---
be/src/vec/core/block.h | 6 +++++-
be/src/vec/exec/scan/pip_scanner_context.h | 3 ++-
.../org/apache/doris/planner/OlapScanNode.java | 23 ++++++++++++++++++----
.../query_p0/join/test_bitmap_filter_nereids.out | 12 +++++++++++
.../join/test_bitmap_filter_nereids.groovy | 4 ++++
5 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h
index 66979a7b6a..381323c049 100644
--- a/be/src/vec/core/block.h
+++ b/be/src/vec/core/block.h
@@ -121,7 +121,11 @@ public:
void initialize_index_by_name();
/// References are invalidated after calling functions above.
- ColumnWithTypeAndName& get_by_position(size_t position) { return
data[position]; }
+ ColumnWithTypeAndName& get_by_position(size_t position) {
+ DCHECK(data.size() > position)
+ << ", data.size()=" << data.size() << ", position=" <<
position;
+ return data[position];
+ }
const ColumnWithTypeAndName& get_by_position(size_t position) const {
return data[position]; }
// need exception safety
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h
b/be/src/vec/exec/scan/pip_scanner_context.h
index f6a9504fc3..0d4509012e 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -17,6 +17,7 @@
#pragma once
+#include "runtime/descriptors.h"
#include "scanner_context.h"
namespace doris {
@@ -156,7 +157,7 @@ public:
return _current_used_bytes < _max_bytes_in_queue / 2 * _max_queue_size;
}
- virtual void _dispose_coloate_blocks_not_in_queue() override {
+ void _dispose_coloate_blocks_not_in_queue() override {
if (_need_colocate_distribute) {
for (int i = 0; i < _max_queue_size; ++i) {
std::scoped_lock s(*_colocate_block_mutexs[i],
*_queue_mutexs[i]);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 8ca8386bae..ea4be00ba0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -200,18 +200,18 @@ public class OlapScanNode extends ScanNode {
olapTable = (OlapTable) desc.getTable();
distributionColumnIds = Sets.newTreeSet();
- Set<String> distColumnName = olapTable != null
- ? olapTable.getDistributionColumnNames() : Sets.newTreeSet();
- int columnId = 0;
+ Set<String> distColumnName = getDistributionColumnNames();
// use for Nereids to generate uniqueId set for inverted index to
avoid scan unnecessary big size column
+
+ int columnId = 0;
for (SlotDescriptor slotDescriptor : desc.getSlots()) {
if (slotDescriptor.getColumn() != null) {
outputColumnUniqueIds.add(slotDescriptor.getColumn().getUniqueId());
if
(distColumnName.contains(slotDescriptor.getColumn().getName().toLowerCase())) {
distributionColumnIds.add(columnId);
}
+ columnId++;
}
- columnId++;
}
}
@@ -1506,13 +1506,28 @@ public class OlapScanNode extends ScanNode {
cardinality = cardinality == -1 ? 0 : cardinality;
}
+ Set<String> getDistributionColumnNames() {
+ return olapTable != null
+ ? olapTable.getDistributionColumnNames()
+ : Sets.newTreeSet();
+ }
+
@Override
public void updateRequiredSlots(PlanTranslatorContext context,
Set<SlotId> requiredByProjectSlotIdSet) {
outputColumnUniqueIds.clear();
+ distributionColumnIds.clear();
+
+ Set<String> distColumnName = getDistributionColumnNames();
+
+ int columnId = 0;
for (SlotDescriptor slot :
context.getTupleDesc(this.getTupleId()).getSlots()) {
if (requiredByProjectSlotIdSet.contains(slot.getId()) &&
slot.getColumn() != null) {
outputColumnUniqueIds.add(slot.getColumn().getUniqueId());
+ if
(distColumnName.contains(slot.getColumn().getName().toLowerCase())) {
+ distributionColumnIds.add(columnId);
+ }
+ columnId++;
}
}
}
diff --git a/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out
b/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out
index d71b056b27..740959669c 100644
--- a/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out
+++ b/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out
@@ -110,3 +110,15 @@
13 1
14 1
+-- !sql15 --
+1 1
+3 1
+5 1
+7 1
+9 1
+10 1
+11 1
+12 1
+13 1
+14 1
+
diff --git
a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
index 82528a44b0..992c2df831 100644
--- a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
+++ b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
@@ -78,6 +78,10 @@ suite("test_bitmap_filter_nereids") {
qt_sql15 "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in
(select k2 from ${tbl2} b2) order by k1;"
+ sql "set enable_pipeline_engine=true;"
+ qt_sql15 "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in
(select k2 from ${tbl2} b2) order by k1;"
+ sql "set enable_pipeline_engine=false;"
+
explain{
sql "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2})
order by k1;"
contains "RF000[bitmap]"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
