This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 291f898 [To rel/0.12] [IOTDB-1975] OOM caused by that
MaxQueryDeduplicatedPathNum doesn't take effect (#4344)
291f898 is described below
commit 291f898b87f215f519b26a2fe09c5d69f91ff72c
Author: Xiangwei Wei <[email protected]>
AuthorDate: Wed Nov 10 09:23:52 2021 +0800
[To rel/0.12] [IOTDB-1975] OOM caused by that MaxQueryDeduplicatedPathNum
doesn't take effect (#4344)
---
RELEASE_NOTES.md | 1 +
.../exception/query/PathNumOverLimitException.java | 6 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 15 +++--
.../db/integration/IoTDBPathNumOverLimitIT.java | 72 ++++++++++++++++++++++
4 files changed, 86 insertions(+), 8 deletions(-)
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index efc072a..29f597b 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -62,6 +62,7 @@
* [IOTDB-1879] Fix some Unsequence files never be merged to higher level or
Sequence folder
* [ISSUE-3945] Fix Fuzzy query not support multiDevices and alignByDevice
Dataset
* [IOTDB-1872] Fix data increases abnormally after IoTDB restarts
+* [IOTDB-1975] OOM caused by that MaxQueryDeduplicatedPathNum doesn't take
effect
* fix merge ClassCastException: MeasurementMNode
* change sync version check to major version
* init dummyIndex after restart cluster
diff --git
a/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
b/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
index c4eb10f..c9efadf 100644
---
a/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
+++
b/server/src/main/java/org/apache/iotdb/db/exception/query/PathNumOverLimitException.java
@@ -19,12 +19,14 @@
package org.apache.iotdb.db.exception.query;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
public class PathNumOverLimitException extends QueryProcessException {
- public PathNumOverLimitException(long maxDeduplicatedPathNum) {
+ public PathNumOverLimitException() {
super(
String.format(
"Too many paths in one query! Currently allowed max deduplicated
path number is %d. Please use slimit or adjust max_deduplicated_path_num in
iotdb-engine.properties.",
- maxDeduplicatedPathNum));
+
IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum()));
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 0a03ea6..4432eb5 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -57,6 +57,9 @@ public class ConcatPathOptimizer implements ILogicalOptimizer
{
private static final String WARNING_NO_PREFIX_PATHS =
"given SFWOperator doesn't have prefix paths, cannot concat seriesPath";
+ private final int MAX_QUERY_PATH_NUM =
+
IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum();
+
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
public Operator transform(Operator operator)
@@ -372,7 +375,8 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
int finalOffset)
throws LogicalOptimizeException, PathNumOverLimitException {
int offset = finalOffset;
- int limit = finalLimit == 0 ? Integer.MAX_VALUE : finalLimit;
+ int limit =
+ finalLimit == 0 ? MAX_QUERY_PATH_NUM + 1 : Math.min(finalLimit,
MAX_QUERY_PATH_NUM + 1);
int consumed = 0;
List<PartialPath> newSuffixPathList = new ArrayList<>();
@@ -445,14 +449,13 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
}
}
+ if (newSuffixPathList.size() > MAX_QUERY_PATH_NUM) {
+ throw new PathNumOverLimitException();
+ }
if (limit == 0) {
- int maxDeduplicatedPathNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum();
- if (maxDeduplicatedPathNum < newSuffixPathList.size()) {
- throw new PathNumOverLimitException(maxDeduplicatedPathNum);
- }
break;
}
+
} catch (MetadataException e) {
throw new LogicalOptimizeException("error when remove star: " +
e.getMessage());
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPathNumOverLimitIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPathNumOverLimitIT.java
new file mode 100644
index 0000000..13a012c
--- /dev/null
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPathNumOverLimitIT.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class IoTDBPathNumOverLimitIT {
+
+ @Before
+ public void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+
IoTDBDescriptor.getInstance().getConfig().setMaxQueryDeduplicatedPathNum(2);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+
IoTDBDescriptor.getInstance().getConfig().setMaxQueryDeduplicatedPathNum(1000);
+ }
+
+ /** Test path num over limit, there is supposed to throw
pathNumOverLimitException. */
+ @Test
+ public void pathNumOverLimitTest() {
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/",
"root", "root");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("insert into root.sg.d1(time, s1, s2, s3) values(1, 1,
1, 1)");
+
+ // fail
+ statement.execute("select * from root");
+ fail();
+ } catch (Exception e) {
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "Too many paths in one query! Currently allowed max
deduplicated path number is 2."));
+ }
+ }
+}
