This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8140fc737e [Fix](inverted index) fix bug when match condition in hash
join (#23105)
8140fc737e is described below
commit 8140fc737eef7bfb76045ffd20904d8701d3b8ef
Author: airborne12 <[email protected]>
AuthorDate: Wed Aug 23 17:48:31 2023 +0800
[Fix](inverted index) fix bug when match condition in hash join (#23105)
* [Fix](inverted index) fix bug when match condition in hash join
---
.../java/org/apache/doris/analysis/SlotRef.java | 4 ++
.../glue/translator/ExpressionTranslator.java | 52 +++++++++++++++-------
.../glue/translator/PhysicalPlanTranslator.java | 21 ++++++---
.../glue/translator/PlanTranslatorContext.java | 1 +
.../inverted_index_p1/tpcds_sf1_index/sql/q100.out | 8 ++++
.../tpcds_sf1_index/ddl/web_site.sql | 2 +-
.../inverted_index_p1/tpcds_sf1_index/sql/q100.sql | 14 ++++++
7 files changed, 77 insertions(+), 25 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
index de0cb93b60..0bb2a37cdb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
@@ -480,6 +480,10 @@ public class SlotRef extends Expr {
this.table = table;
}
+ public TableIf getTableDirect() {
+ return this.table;
+ }
+
public TableIf getTable() {
if (desc == null && table != null) {
return table;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
index a67513d0f0..7b5a13e73e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
@@ -31,6 +31,7 @@ import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.FunctionParams;
import org.apache.doris.analysis.IndexDef;
+import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.MatchPredicate;
import org.apache.doris.analysis.OrderByElement;
@@ -169,23 +170,39 @@ public class ExpressionTranslator extends
DefaultExpressionVisitor<Expr, PlanTra
NullableMode.DEPEND_ON_ARGUMENT);
}
+ private OlapTable getOlapTableFromSlotDesc(SlotDescriptor slotDesc) {
+ if (slotDesc != null && slotDesc.isScanSlot()) {
+ TupleDescriptor slotParent = slotDesc.getParent();
+ return (OlapTable) slotParent.getTable();
+ }
+ return null;
+ }
+
+ private OlapTable getOlapTableDirectly(SlotRef left) {
+ if (left.getTableDirect() instanceof OlapTable) {
+ return (OlapTable) left.getTableDirect();
+ }
+ return null;
+ }
+
@Override
public Expr visitMatch(Match match, PlanTranslatorContext context) {
- String invertedIndexParser = null;
- String invertedIndexParserMode = null;
+ String invertedIndexParser =
InvertedIndexUtil.INVERTED_INDEX_PARSER_UNKNOWN;
+ String invertedIndexParserMode =
InvertedIndexUtil.INVERTED_INDEX_PARSER_FINE_GRANULARITY;
SlotRef left = (SlotRef) match.left().accept(this, context);
- SlotDescriptor slotDesc = left.getDesc();
- if (slotDesc != null && slotDesc.isScanSlot()) {
- TupleDescriptor slotParent = slotDesc.getParent();
- OlapTable olapTbl = (OlapTable) slotParent.getTable();
- if (olapTbl == null) {
- throw new AnalysisException("slotRef in matchExpression failed
to get OlapTable");
- }
- List<Index> indexes = olapTbl.getIndexes();
+ OlapTable olapTbl =
Optional.ofNullable(getOlapTableFromSlotDesc(left.getDesc()))
+ .orElse(getOlapTableDirectly(left));
+
+ if (olapTbl == null) {
+ throw new AnalysisException("slotRef in matchExpression failed to
get OlapTable");
+ }
+
+ List<Index> indexes = olapTbl.getIndexes();
+ if (indexes != null) {
for (Index index : indexes) {
if (index.getIndexType() == IndexDef.IndexType.INVERTED) {
List<String> columns = index.getColumns();
- if (left.getColumnName().equals(columns.get(0))) {
+ if (columns != null && !columns.isEmpty() &&
left.getColumnName().equals(columns.get(0))) {
invertedIndexParser = index.getInvertedIndexParser();
invertedIndexParserMode =
index.getInvertedIndexParserMode();
break;
@@ -193,14 +210,15 @@ public class ExpressionTranslator extends
DefaultExpressionVisitor<Expr, PlanTra
}
}
}
+
MatchPredicate.Operator op = match.op();
return new MatchPredicate(op,
- match.left().accept(this, context),
- match.right().accept(this, context),
- match.getDataType().toCatalogDataType(),
- NullableMode.DEPEND_ON_ARGUMENT,
- invertedIndexParser,
- invertedIndexParserMode);
+ match.left().accept(this, context),
+ match.right().accept(this, context),
+ match.getDataType().toCatalogDataType(),
+ NullableMode.DEPEND_ON_ARGUMENT,
+ invertedIndexParser,
+ invertedIndexParserMode);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 2a5ba6474e..5a83a7acf4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -1130,7 +1130,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove
it after refactor
sd =
context.getDescTable().copySlotDescriptor(intermediateDescriptor,
leftSlotDescriptor);
} else {
- sd = context.createSlotDesc(intermediateDescriptor, sf);
+ sd = context.createSlotDesc(intermediateDescriptor, sf,
leftSlotDescriptor.getParent().getTable());
+ //sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) !=
null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(leftSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(),
leftSlotDescriptor.getId());
@@ -1150,7 +1151,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove
it after refactor
sd =
context.getDescTable().copySlotDescriptor(intermediateDescriptor,
rightSlotDescriptor);
} else {
- sd = context.createSlotDesc(intermediateDescriptor, sf);
+ sd = context.createSlotDesc(intermediateDescriptor, sf,
rightSlotDescriptor.getParent().getTable());
+ //sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) !=
null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(rightSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(),
rightSlotDescriptor.getId());
@@ -1169,7 +1171,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove
it after refactor
sd =
context.getDescTable().copySlotDescriptor(intermediateDescriptor,
leftSlotDescriptor);
} else {
- sd = context.createSlotDesc(intermediateDescriptor, sf);
+ sd = context.createSlotDesc(intermediateDescriptor, sf,
leftSlotDescriptor.getParent().getTable());
+ //sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) !=
null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(leftSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(),
leftSlotDescriptor.getId());
@@ -1187,7 +1190,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove
it after refactor
sd =
context.getDescTable().copySlotDescriptor(intermediateDescriptor,
rightSlotDescriptor);
} else {
- sd = context.createSlotDesc(intermediateDescriptor, sf);
+ sd = context.createSlotDesc(intermediateDescriptor, sf,
rightSlotDescriptor.getParent().getTable());
+ //sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) !=
null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(rightSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(),
rightSlotDescriptor.getId());
@@ -1355,7 +1359,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove
it after refactor
sd =
context.getDescTable().copySlotDescriptor(intermediateDescriptor,
leftSlotDescriptor);
} else {
- sd = context.createSlotDesc(intermediateDescriptor, sf);
+ sd = context.createSlotDesc(intermediateDescriptor, sf,
leftSlotDescriptor.getParent().getTable());
+ //sd = context.createSlotDesc(intermediateDescriptor, sf);
}
leftIntermediateSlotDescriptor.add(sd);
}
@@ -1369,7 +1374,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove
it after refactor
sd =
context.getDescTable().copySlotDescriptor(intermediateDescriptor,
rightSlotDescriptor);
} else {
- sd = context.createSlotDesc(intermediateDescriptor, sf);
+ sd = context.createSlotDesc(intermediateDescriptor, sf,
rightSlotDescriptor.getParent().getTable());
+ //sd = context.createSlotDesc(intermediateDescriptor, sf);
}
rightIntermediateSlotDescriptor.add(sd);
}
@@ -1533,7 +1539,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
if (requiredByProjectSlotIdSet.size() != requiredSlotIdSet.size()
|| new HashSet<>(projectionExprs).size() !=
projectionExprs.size()
|| projectionExprs.stream().anyMatch(expr -> !(expr
instanceof SlotRef))) {
- projectionTuple = generateTupleDesc(slots, null, context);
+ projectionTuple = generateTupleDesc(slots,
+ ((ScanNode)
inputPlanNode).getTupleDesc().getTable(), context);
inputPlanNode.setProjectList(projectionExprs);
inputPlanNode.setOutputTupleDesc(projectionTuple);
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index a9c9dae012..1c176da409 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -210,6 +210,7 @@ public class PlanTranslatorContext {
} else {
slotRef = new SlotRef(slotDescriptor);
}
+ slotRef.setTable(table);
slotRef.setLabel(slotReference.getName());
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
slotDescriptor.setIsNullable(slotReference.nullable());
diff --git
a/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out
b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out
new file mode 100644
index 0000000000..fbb2285d06
--- /dev/null
+++ b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !q100 --
+site_0 144200
+site_1 519
+site_2 539
+site_3 504
+site_4 563
+
diff --git
a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql
b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql
index b6f930a728..640926c35a 100644
--- a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql
+++ b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS web_site (
INDEX web_site_sk_idx(web_site_sk) USING INVERTED COMMENT "web_site_sk
index",
INDEX web_site_id_idx(web_site_id) USING INVERTED COMMENT "web_site_id
index",
INDEX web_rec_start_date_idx(web_rec_start_date) USING INVERTED COMMENT
"web_rec_start_date index",
- INDEX web_name_idx(web_name) USING INVERTED COMMENT "web_name index",
+ INDEX web_name_idx(web_name) USING INVERTED PROPERTIES("parser"="unicode")
COMMENT "web_name index",
INDEX web_class_idx(web_class) USING INVERTED COMMENT "web_class index",
INDEX web_manager_idx(web_manager) USING INVERTED
PROPERTIES("parser"="standard") COMMENT "web_manager index",
INDEX web_market_manager_idx(web_market_manager) USING INVERTED
PROPERTIES("parser"="none") COMMENT "web_market_manager index",
diff --git
a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql
b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql
new file mode 100644
index 0000000000..8653806f9e
--- /dev/null
+++ b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql
@@ -0,0 +1,14 @@
+SELECT
+ t1.web_name,
+ COUNT(t1.web_name) AS count
+FROM
+ web_site t1
+LEFT JOIN
+ web_sales t2 ON t1.web_site_sk = t2.ws_web_site_sk
+WHERE
+ t1.web_name MATCH 'site_0'
+ OR t2.ws_item_sk > 17934
+GROUP BY
+ t1.web_name
+ORDER BY
+ t1.web_name;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
