This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0bb50c3009 [IOTDB-3644] Fix the incorrect result bug when querying
with value filter & order by time desc (#6422)
0bb50c3009 is described below
commit 0bb50c3009d58b5bf299db6845f97c99f76b3f0a
Author: liuminghui233 <[email protected]>
AuthorDate: Fri Jun 24 21:36:52 2022 +0800
[IOTDB-3644] Fix the incorrect result bug when querying with value filter &
order by time desc (#6422)
---
.../aggregation/IoTDBAggregationLargeDataIT.java | 6 ++---
.../execution/operator/process/FilterOperator.java | 6 +++--
.../operator/process/TransformOperator.java | 5 ++--
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 6 +++--
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 14 +++++++---
.../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 18 ++++++++-----
.../plan/planner/plan/node/process/FilterNode.java | 19 +++++++++-----
.../planner/plan/node/process/TransformNode.java | 30 +++++++++++++++++-----
.../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 18 ++++++++-----
.../plan/node/process/FilterNodeSerdeTest.java | 3 ++-
10 files changed, 87 insertions(+), 38 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java
index 01ef6b3cd9..3c15fd1b87 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationLargeDataIT.java
@@ -21,15 +21,16 @@ package org.apache.iotdb.db.it.aggregation;
import org.apache.iotdb.it.env.ConfigFactory;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -47,8 +48,7 @@ import static
org.apache.iotdb.db.constant.TestConstant.minValue;
import static org.apache.iotdb.db.constant.TestConstant.sum;
import static org.junit.Assert.fail;
-// TODO: remove ignore after supporting value filter
-@Ignore
+@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBAggregationLargeDataIT {
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/FilterOperator.java
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/FilterOperator.java
index d5ab29350c..2611ae791a 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/FilterOperator.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/FilterOperator.java
@@ -58,7 +58,8 @@ public class FilterOperator extends TransformOperator {
Expression[] outputExpressions,
boolean keepNull,
ZoneId zoneId,
- TypeProvider typeProvider)
+ TypeProvider typeProvider,
+ boolean isAscending)
throws QueryProcessException, IOException {
super(
operatorContext,
@@ -68,7 +69,8 @@ public class FilterOperator extends TransformOperator {
bindExpressions(filterExpression, outputExpressions),
keepNull,
zoneId,
- typeProvider);
+ typeProvider,
+ isAscending);
}
private static Expression[] bindExpressions(
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/TransformOperator.java
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/TransformOperator.java
index 5e77c82c63..26d81f71b5 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/TransformOperator.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/TransformOperator.java
@@ -84,7 +84,8 @@ public class TransformOperator implements ProcessOperator {
Expression[] outputExpressions,
boolean keepNull,
ZoneId zoneId,
- TypeProvider typeProvider)
+ TypeProvider typeProvider,
+ boolean isAscending)
throws QueryProcessException, IOException {
this.operatorContext = operatorContext;
this.inputOperator = inputOperator;
@@ -94,7 +95,7 @@ public class TransformOperator implements ProcessOperator {
initUdtfContext(outputExpressions, zoneId);
initTransformers(inputLocations, outputExpressions, typeProvider);
- timeHeap = new TimeSelector(transformers.length << 1, true);
+ timeHeap = new TimeSelector(transformers.length << 1, isAscending);
shouldIterateReadersToNextValid = new boolean[outputExpressions.length];
Arrays.fill(shouldIterateReadersToNextValid, true);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
index 82fd67af9a..b2056696df 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
@@ -778,7 +778,8 @@ public class LocalExecutionPlanner {
node.getOutputExpressions(),
node.isKeepNull(),
node.getZoneId(),
- context.getTypeProvider());
+ context.getTypeProvider(),
+ node.getScanOrder() == OrderBy.TIMESTAMP_ASC);
} catch (QueryProcessException | IOException e) {
throw new RuntimeException(e);
}
@@ -805,7 +806,8 @@ public class LocalExecutionPlanner {
node.getOutputExpressions(),
node.isKeepNull(),
node.getZoneId(),
- context.getTypeProvider());
+ context.getTypeProvider(),
+ node.getScanOrder() == OrderBy.TIMESTAMP_ASC);
} catch (QueryProcessException | IOException e) {
throw new RuntimeException(e);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index 5a9dbd4dac..b90a5d738a 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -578,7 +578,8 @@ public class LogicalPlanBuilder {
Expression queryFilter,
Set<Expression> selectExpressions,
boolean isGroupByTime,
- ZoneId zoneId) {
+ ZoneId zoneId,
+ OrderBy scanOrder) {
if (queryFilter == null) {
return this;
}
@@ -590,12 +591,16 @@ public class LogicalPlanBuilder {
selectExpressions.toArray(new Expression[0]),
queryFilter,
isGroupByTime,
- zoneId);
+ zoneId,
+ scanOrder);
return this;
}
public LogicalPlanBuilder planTransform(
- Set<Expression> transformExpressions, boolean isGroupByTime, ZoneId
zoneId) {
+ Set<Expression> transformExpressions,
+ boolean isGroupByTime,
+ ZoneId zoneId,
+ OrderBy scanOrder) {
boolean needTransform = false;
for (Expression expression : transformExpressions) {
if (ExpressionAnalyzer.checkIsNeedTransform(expression)) {
@@ -613,7 +618,8 @@ public class LogicalPlanBuilder {
this.getRoot(),
transformExpressions.toArray(new Expression[0]),
isGroupByTime,
- zoneId);
+ zoneId,
+ scanOrder);
return this;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
index b94e106bdb..e1abfbf686 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
@@ -202,13 +202,15 @@ public class LogicalPlanner {
queryFilter,
aggregationTransformExpressions,
queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId());
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultOrder());
} else {
planBuilder =
planBuilder.planTransform(
aggregationTransformExpressions,
queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId());
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultOrder());
}
boolean outputPartial =
@@ -254,7 +256,8 @@ public class LogicalPlanner {
planBuilder.planTransform(
transformExpressions,
queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId());
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultOrder());
} else {
if (analysis.hasValueFilter()) {
planBuilder =
@@ -262,13 +265,15 @@ public class LogicalPlanner {
queryFilter,
transformExpressions,
queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId());
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultOrder());
} else {
planBuilder =
planBuilder.planTransform(
transformExpressions,
queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId());
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultOrder());
}
}
} else {
@@ -314,7 +319,8 @@ public class LogicalPlanner {
.planTransform(
transformExpressions,
queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId());
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultOrder());
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/FilterNode.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/FilterNode.java
index 9e1fa43eb3..10df916541 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/FilterNode.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/FilterNode.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.mpp.plan.statement.component.OrderBy;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
@@ -41,8 +42,9 @@ public class FilterNode extends TransformNode {
Expression[] outputExpressions,
Expression predicate,
boolean keepNull,
- ZoneId zoneId) {
- super(id, childPlanNode, outputExpressions, keepNull, zoneId);
+ ZoneId zoneId,
+ OrderBy scanOrder) {
+ super(id, childPlanNode, outputExpressions, keepNull, zoneId, scanOrder);
this.predicate = predicate;
}
@@ -51,8 +53,9 @@ public class FilterNode extends TransformNode {
Expression[] outputExpressions,
Expression predicate,
boolean keepNull,
- ZoneId zoneId) {
- super(id, outputExpressions, keepNull, zoneId);
+ ZoneId zoneId,
+ OrderBy scanOrder) {
+ super(id, outputExpressions, keepNull, zoneId, scanOrder);
this.predicate = predicate;
}
@@ -63,7 +66,8 @@ public class FilterNode extends TransformNode {
@Override
public PlanNode clone() {
- return new FilterNode(getPlanNodeId(), outputExpressions, predicate,
keepNull, zoneId);
+ return new FilterNode(
+ getPlanNodeId(), outputExpressions, predicate, keepNull, zoneId,
scanOrder);
}
@Override
@@ -76,6 +80,7 @@ public class FilterNode extends TransformNode {
Expression.serialize(predicate, byteBuffer);
ReadWriteIOUtils.write(keepNull, byteBuffer);
ReadWriteIOUtils.write(zoneId.getId(), byteBuffer);
+ ReadWriteIOUtils.write(scanOrder.ordinal(), byteBuffer);
}
@Override
@@ -88,6 +93,7 @@ public class FilterNode extends TransformNode {
Expression.serialize(predicate, stream);
ReadWriteIOUtils.write(keepNull, stream);
ReadWriteIOUtils.write(zoneId.getId(), stream);
+ ReadWriteIOUtils.write(scanOrder.ordinal(), stream);
}
public static FilterNode deserialize(ByteBuffer byteBuffer) {
@@ -99,8 +105,9 @@ public class FilterNode extends TransformNode {
Expression predicate = Expression.deserialize(byteBuffer);
boolean keepNull = ReadWriteIOUtils.readBool(byteBuffer);
ZoneId zoneId =
ZoneId.of(Objects.requireNonNull(ReadWriteIOUtils.readString(byteBuffer)));
+ OrderBy scanOrder = OrderBy.values()[ReadWriteIOUtils.readInt(byteBuffer)];
PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
- return new FilterNode(planNodeId, outputExpressions, predicate, keepNull,
zoneId);
+ return new FilterNode(planNodeId, outputExpressions, predicate, keepNull,
zoneId, scanOrder);
}
public Expression getPredicate() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/TransformNode.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/TransformNode.java
index f01ba32dfa..c57fb91317 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/TransformNode.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/TransformNode.java
@@ -24,6 +24,7 @@ import
org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.mpp.plan.statement.component.OrderBy;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import com.google.common.collect.ImmutableList;
@@ -45,6 +46,8 @@ public class TransformNode extends ProcessNode {
protected final boolean keepNull;
protected final ZoneId zoneId;
+ protected final OrderBy scanOrder;
+
private List<String> outputColumnNames;
public TransformNode(
@@ -52,20 +55,27 @@ public class TransformNode extends ProcessNode {
PlanNode childPlanNode,
Expression[] outputExpressions,
boolean keepNull,
- ZoneId zoneId) {
+ ZoneId zoneId,
+ OrderBy scanOrder) {
super(id);
this.childPlanNode = childPlanNode;
this.outputExpressions = outputExpressions;
this.keepNull = keepNull;
this.zoneId = zoneId;
+ this.scanOrder = scanOrder;
}
public TransformNode(
- PlanNodeId id, Expression[] outputExpressions, boolean keepNull, ZoneId
zoneId) {
+ PlanNodeId id,
+ Expression[] outputExpressions,
+ boolean keepNull,
+ ZoneId zoneId,
+ OrderBy scanOrder) {
super(id);
this.outputExpressions = outputExpressions;
this.keepNull = keepNull;
this.zoneId = zoneId;
+ this.scanOrder = scanOrder;
}
@Override
@@ -101,7 +111,7 @@ public class TransformNode extends ProcessNode {
@Override
public PlanNode clone() {
- return new TransformNode(getPlanNodeId(), outputExpressions, keepNull,
zoneId);
+ return new TransformNode(getPlanNodeId(), outputExpressions, keepNull,
zoneId, scanOrder);
}
@Override
@@ -113,6 +123,7 @@ public class TransformNode extends ProcessNode {
}
ReadWriteIOUtils.write(keepNull, byteBuffer);
ReadWriteIOUtils.write(zoneId.getId(), byteBuffer);
+ ReadWriteIOUtils.write(scanOrder.ordinal(), byteBuffer);
}
@Override
@@ -124,6 +135,7 @@ public class TransformNode extends ProcessNode {
}
ReadWriteIOUtils.write(keepNull, stream);
ReadWriteIOUtils.write(zoneId.getId(), stream);
+ ReadWriteIOUtils.write(scanOrder.ordinal(), stream);
}
public static TransformNode deserialize(ByteBuffer byteBuffer) {
@@ -134,8 +146,9 @@ public class TransformNode extends ProcessNode {
}
boolean keepNull = ReadWriteIOUtils.readBool(byteBuffer);
ZoneId zoneId =
ZoneId.of(Objects.requireNonNull(ReadWriteIOUtils.readString(byteBuffer)));
+ OrderBy scanOrder = OrderBy.values()[ReadWriteIOUtils.readInt(byteBuffer)];
PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
- return new TransformNode(planNodeId, outputExpressions, keepNull, zoneId);
+ return new TransformNode(planNodeId, outputExpressions, keepNull, zoneId,
scanOrder);
}
public final Expression[] getOutputExpressions() {
@@ -150,6 +163,10 @@ public class TransformNode extends ProcessNode {
return zoneId;
}
+ public OrderBy getScanOrder() {
+ return scanOrder;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -165,12 +182,13 @@ public class TransformNode extends ProcessNode {
return keepNull == that.keepNull
&& childPlanNode.equals(that.childPlanNode)
&& Arrays.equals(outputExpressions, that.outputExpressions)
- && zoneId.equals(that.zoneId);
+ && zoneId.equals(that.zoneId)
+ && scanOrder == that.scanOrder;
}
@Override
public int hashCode() {
- int result = Objects.hash(super.hashCode(), childPlanNode, keepNull,
zoneId);
+ int result = Objects.hash(super.hashCode(), childPlanNode, keepNull,
zoneId, scanOrder);
result = 31 * result + Arrays.hashCode(outputExpressions);
return result;
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/QueryLogicalPlanUtil.java
b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/QueryLogicalPlanUtil.java
index c5f7e05b19..c62da3bc79 100644
---
a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/QueryLogicalPlanUtil.java
+++
b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/QueryLogicalPlanUtil.java
@@ -231,7 +231,8 @@ public class QueryLogicalPlanUtil {
new Expression[] {new
TimeSeriesOperand(schemaMap.get("root.sg.d2.s1"))},
predicate,
false,
- ZonedDateTime.now().getOffset());
+ ZonedDateTime.now().getOffset(),
+ OrderBy.TIMESTAMP_DESC);
FilterNullNode filterNullNode =
new FilterNullNode(
@@ -296,7 +297,8 @@ public class QueryLogicalPlanUtil {
},
predicate1,
false,
- ZonedDateTime.now().getOffset());
+ ZonedDateTime.now().getOffset(),
+ OrderBy.TIMESTAMP_DESC);
List<PlanNode> sourceNodeList2 = new ArrayList<>();
sourceNodeList2.add(
@@ -337,7 +339,8 @@ public class QueryLogicalPlanUtil {
},
predicate2,
false,
- ZonedDateTime.now().getOffset());
+ ZonedDateTime.now().getOffset(),
+ OrderBy.TIMESTAMP_DESC);
Map<String, List<Integer>> deviceToMeasurementIndexesMap = new HashMap<>();
deviceToMeasurementIndexesMap.put("root.sg.d1", Arrays.asList(1, 2, 3));
@@ -794,7 +797,8 @@ public class QueryLogicalPlanUtil {
},
predicate,
false,
- ZonedDateTime.now().getOffset());
+ ZonedDateTime.now().getOffset(),
+ OrderBy.TIMESTAMP_DESC);
AggregationNode aggregationNode =
new AggregationNode(
@@ -915,7 +919,8 @@ public class QueryLogicalPlanUtil {
},
predicate1,
false,
- ZonedDateTime.now().getOffset());
+ ZonedDateTime.now().getOffset(),
+ OrderBy.TIMESTAMP_DESC);
AggregationNode aggregationNode1 =
new AggregationNode(
@@ -973,7 +978,8 @@ public class QueryLogicalPlanUtil {
},
predicate2,
false,
- ZonedDateTime.now().getOffset());
+ ZonedDateTime.now().getOffset(),
+ OrderBy.TIMESTAMP_DESC);
AggregationNode aggregationNode2 =
new AggregationNode(
diff --git
a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/FilterNodeSerdeTest.java
b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/FilterNodeSerdeTest.java
index 43310e8f47..bac6d75cd7 100644
---
a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/FilterNodeSerdeTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/FilterNodeSerdeTest.java
@@ -53,7 +53,8 @@ public class FilterNodeSerdeTest {
new TimeSeriesOperand(new PartialPath("root.sg.d1.s1")),
new ConstantOperand(TSDataType.INT64, "100")),
false,
- ZoneId.systemDefault());
+ ZoneId.systemDefault(),
+ OrderBy.TIMESTAMP_ASC);
ByteBuffer byteBuffer = ByteBuffer.allocate(1024);
filterNode.serialize(byteBuffer);
