This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d7844931a87 Fixed the issue that non aligned tree view query used too
much memory
d7844931a87 is described below
commit d7844931a87a9478c5ada93276202e13565cae22
Author: shuwenwei <[email protected]>
AuthorDate: Tue Jul 15 08:47:13 2025 +0800
Fixed the issue that non aligned tree view query used too much memory
---
.../plan/planner/TableOperatorGenerator.java | 4 +-
...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 69 ++++++++++++++--------
2 files changed, 46 insertions(+), 27 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index b3ed766069a..75e2275275b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -647,9 +647,7 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
SeriesScanOptions.Builder builder = new
SeriesScanOptions.Builder();
// time filter may be stateful, so we need to copy it
builder.withGlobalTimeFilter(timeFilter == null ? null :
timeFilter.copy());
- builder
- .withIsTableViewForTreeModel(true)
- .withAllSensors(new HashSet<>(measurementColumnNames));
+
builder.withIsTableViewForTreeModel(true).withAllSensors(allSensors);
if (pushDownPredicateForCurrentMeasurement != null) {
builder.withPushDownFilter(
convertPredicateToFilter(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
index 2188a04479b..01a00529503 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
@@ -126,28 +126,20 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest
{
new Symbol("time"),
new ColumnSchema(
"time", TypeFactory.getType(TSDataType.INT64), false,
TsTableColumnCategory.TIME));
- columnSchemaMap.put(
- new Symbol("sensor0"),
- new ColumnSchema(
- "sensor0", TypeFactory.getType(TSDataType.INT32), false,
TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor1"),
- new ColumnSchema(
- "sensor1", TypeFactory.getType(TSDataType.INT32), false,
TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor2"),
- new ColumnSchema(
- "sensor2", TypeFactory.getType(TSDataType.INT32), false,
TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor3"),
- new ColumnSchema(
- "sensor3", TypeFactory.getType(TSDataType.INT32), false,
TsTableColumnCategory.FIELD));
+ for (int i = 0; i < 10000; i++) {
+ columnSchemaMap.put(
+ new Symbol("sensor" + i),
+ new ColumnSchema(
+ "sensor" + i,
+ TypeFactory.getType(TSDataType.INT32),
+ false,
+ TsTableColumnCategory.FIELD));
+ }
Map<Symbol, Type> symbolTSDataTypeMap = new HashMap<>();
- symbolTSDataTypeMap.put(new Symbol("sensor0"),
TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor1"),
TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor2"),
TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor3"),
TypeFactory.getType(TSDataType.INT32));
+ for (int i = 0; i < 10000; i++) {
+ symbolTSDataTypeMap.put(new Symbol("sensor" + i),
TypeFactory.getType(TSDataType.INT32));
+ }
symbolTSDataTypeMap.put(new Symbol("time"),
TypeFactory.getType(TypeEnum.INT64));
symbolTSDataTypeMap.put(new Symbol("tag1"),
TypeFactory.getType(TSDataType.TEXT));
typeProvider = new TypeProvider(symbolTSDataTypeMap);
@@ -158,10 +150,9 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
tsTable.addColumnSchema(new TagColumnSchema("id_column",
TSDataType.STRING));
tsTable.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64));
tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.TEXT));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor0",
TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor1",
TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor2",
TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor3",
TSDataType.INT32));
+ for (int i = 0; i < 10000; i++) {
+ tsTable.addColumnSchema(new FieldColumnSchema("sensor" + i,
TSDataType.INT32));
+ }
tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + "");
tsTable.addProp(
TreeViewSchema.TREE_PATH_PATTERN,
@@ -176,6 +167,36 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
SeriesReaderTestUtil.tearDown(seqResources, unSeqResources);
}
+ @Test
+ public void testQueryManyDevices() throws Exception {
+ List<String> outputColumnList = new ArrayList<>(10000 + 2);
+ TreeNonAlignedDeviceViewScanNode node =
getTreeNonAlignedDeviceViewScanNode(outputColumnList);
+ node.setPushDownOffset(500);
+ node.setPushDownLimit(500);
+ node.setPushDownPredicate(
+ new ComparisonExpression(
+ ComparisonExpression.Operator.GREATER_THAN,
+ new Symbol("sensor1").toSymbolReference(),
+ new LongLiteral("1000")));
+ for (int i = 0; i < 10000; i++) {
+ outputColumnList.add("sensor" + i);
+ }
+ outputColumnList.add("time");
+ outputColumnList.add("tag1");
+ ExecutorService instanceNotificationExecutor =
+ IoTDBThreadPoolFactory.newFixedThreadPool(1,
"test-instance-notification");
+ Operator operator = getOperator(node, instanceNotificationExecutor);
+ try {
+ assertTrue(operator instanceof DeviceIteratorScanOperator);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ } finally {
+ operator.close();
+ instanceNotificationExecutor.shutdown();
+ }
+ }
+
@Test
public void testScanWithPushDownPredicateAndLimitAndOffset() throws
Exception {
List<String> outputColumnList = Arrays.asList("sensor0", "sensor1",
"sensor2", "time", "tag1");
