This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix-delete-dnf-limit in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0dd9b6d60d324e12c1b402b32af7011c62d4101c Author: Caideyipi <[email protected]> AuthorDate: Wed Jul 8 12:10:01 2026 +0800 Limit delete predicate DNF expansion --- .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++ .../db/queryengine/plan/analyze/AnalyzeUtils.java | 40 +++++++++++++++++----- .../queryengine/plan/analyze/AnalyzeUtilsTest.java | 36 +++++++++++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 7978fce5113..4ea569ad799 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -335,6 +335,10 @@ public final class DataNodeQueryMessages { public static final String TOO_MANY_DEVICES_MATCHED_BY_ATTRIBUTE_FILTERS_IN_DELETION = "Too many devices (%d) matched by attribute filters in deletion, limit is %d. " + "Please remove all attribute filters (%s) or add more attribute filters."; + public static final String + EXCEPTION_DELETION_PREDICATE_EXPANDS_TO_TOO_MANY_DISJUNCTIVE_TERMS_ARG_LIMIT_IS_ARG_PLEASE_SIMPLIFY_THE_OR_CONDITIONS_BA138588 = + "Deletion predicate expands to too many disjunctive terms (%d), limit is %d. " + + "Please simplify the OR conditions."; public static final String ONLY_TIME_FILTERS_ARE_SUPPORTED_IN_LAST_QUERY = "Only time filters are supported in LAST query"; public static final String VIEWS_CANNOT_BE_USED_IN_GROUP_BY_TAGS = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 34a7fa084bf..adcfc5b252a 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -319,6 +319,10 @@ public final class DataNodeQueryMessages { "列 '%s' 不存在或不是标签列"; public static final String TOO_MANY_DEVICES_MATCHED_BY_ATTRIBUTE_FILTERS_IN_DELETION = "删除语句中的属性过滤条件匹配了过多设备（%d），限制为 %d。请移除所有属性过滤条件（%s），或进一步增加属性过滤条件。"; + public static final String + EXCEPTION_DELETION_PREDICATE_EXPANDS_TO_TOO_MANY_DISJUNCTIVE_TERMS_ARG_LIMIT_IS_ARG_PLEASE_SIMPLIFY_THE_OR_CONDITIONS_BA138588 = + "删除谓词会展开为过多析取项（%d），限制为 %d。" + + "请简化 OR 条件。"; public static final String ONLY_TIME_FILTERS_ARE_SUPPORTED_IN_LAST_QUERY = "LAST 查询中仅支持时间过滤器"; public static final String VIEWS_CANNOT_BE_USED_IN_GROUP_BY_TAGS = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 81849ff9dfb..65fbc06b214 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -96,6 +96,7 @@ import static org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigT public class AnalyzeUtils { private static final int ATTRIBUTE_FILTER_DELETE_DEVICE_IN_LIMIT = 1000; + private static final int DELETE_DNF_EXPRESSION_LIMIT = 1000; private static final CommonQueryAstVisitor<PredicateParseResult, PredicateParseContext> DELETION_PREDICATE_PARSE_VISITOR = new CommonQueryAstVisitor<>() { @@ -455,14 +456,18 @@ public class AnalyzeUtils { } /** - * Convert to a disjunctive normal forms. + * Convert to disjunctive normal forms. * * <p>For example: ( A | B ) & ( C | D ) => ( A & C ) | ( A & D ) | ( B & C ) | ( B & D) * - * <p>Returns the original expression if the expression is null or if the distribution will expand - * the expression by too much. + * <p>Throws SemanticException if the distribution will expand the expression by too much. */ public static List<Expression> toDisjunctiveNormalForms(Expression expression) { + return toDisjunctiveNormalForms(expression, DELETE_DNF_EXPRESSION_LIMIT); + } + + private static List<Expression> toDisjunctiveNormalForms( + Expression expression, int expressionLimit) { if (!(expression instanceof LogicalExpression)) { return Collections.singletonList(expression); } @@ -472,20 +477,24 @@ public class AnalyzeUtils { // ( A | B ) & ( C | D ) => ( A & C ) | ( A & D ) | ( B & C ) | ( B & D) List<Expression> results = null; for (Expression term : logicalExpression.getTerms()) { + List<Expression> termResults = toDisjunctiveNormalForms(term, expressionLimit); if (results == null) { - results = toDisjunctiveNormalForms(term); + results = termResults; } else { results = crossProductOfDisjunctiveNormalForms( - results, toDisjunctiveNormalForms(term), Operator.AND); + results, termResults, Operator.AND, expressionLimit); } + ensureDnfExpressionLimit(results.size(), expressionLimit); } return results; } else if (logicalExpression.getOperator() == Operator.OR) { // ( A | B ) | ( C | D ) => A | B | C | D List<Expression> results = new ArrayList<>(); for (Expression term : logicalExpression.getTerms()) { - results.addAll(toDisjunctiveNormalForms(term)); + List<Expression> termResults = toDisjunctiveNormalForms(term, expressionLimit); + ensureDnfExpressionLimit((long) results.size() + termResults.size(), expressionLimit); + results.addAll(termResults); } return results; } else { @@ -495,8 +504,12 @@ public class AnalyzeUtils { } private static List<Expression> crossProductOfDisjunctiveNormalForms( - List<Expression> leftList, List<Expression> rightList, Operator operator) { - List<Expression> results = new ArrayList<>(); + List<Expression> leftList, + List<Expression> rightList, + Operator operator, + int expressionLimit) { + ensureDnfExpressionLimit((long) leftList.size() * rightList.size(), expressionLimit); + List<Expression> results = new ArrayList<>(leftList.size() * rightList.size()); for (Expression leftExp : leftList) { for (Expression rightExp : rightList) { List<Expression> terms = new ArrayList<>(); @@ -518,6 +531,17 @@ public class AnalyzeUtils { return results; } + private static void ensureDnfExpressionLimit(final long expressionSize, final int limit) { + if (expressionSize > limit) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_DELETION_PREDICATE_EXPANDS_TO_TOO_MANY_DISJUNCTIVE_TERMS_ARG_LIMIT_IS_ARG_PLEASE_SIMPLIFY_THE_OR_CONDITIONS_BA138588, + expressionSize, + limit)); + } + } + private static TableDeletionEntry parsePredicate( Expression expression, TsTable table, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java index 915006a8286..8ecf2b49787 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java @@ -23,11 +23,14 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; @@ -47,6 +50,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -54,6 +58,7 @@ import java.util.List; import java.util.Set; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class AnalyzeUtilsTest { @@ -77,6 +82,24 @@ public class AnalyzeUtilsTest { assertEquals(100, entries.get(0).getEndTime()); } + @Test + public void testDeletePredicateDnfExpansionWithinLimit() { + List<Expression> result = + AnalyzeUtils.toDisjunctiveNormalForms(buildConjunctiveOrExpression(9)); + + assertEquals(512, result.size()); + } + + @Test + public void testDeletePredicateDnfExpansionRejectsTooManyTerms() { + SemanticException exception = + assertThrows( + SemanticException.class, + () -> AnalyzeUtils.toDisjunctiveNormalForms(buildConjunctiveOrExpression(10))); + + assertTrue(exception.getMessage().contains("too many disjunctive terms (1024)")); + } + @Test public void testFetchDeleteReplicaSetsOnlyQueriesTargetDatabaseRegions() throws Exception { final Delete delete = new Delete(new Table(QualifiedName.of("table1"))); @@ -118,6 +141,19 @@ public class AnalyzeUtilsTest { assertEquals(40, requests.get(1).getEndTime()); } + private static Expression buildConjunctiveOrExpression(final int termCount) { + final List<Expression> terms = new ArrayList<>(); + for (int i = 0; i < termCount; i++) { + terms.add(LogicalExpression.or(equalTag("tag" + i, "left"), equalTag("tag" + i, "right"))); + } + return new LogicalExpression(LogicalExpression.Operator.AND, terms); + } + + private static Expression equalTag(final String column, final String value) { + return new ComparisonExpression( + ComparisonExpression.Operator.EQUAL, new Identifier(column), new StringLiteral(value)); + } + private static TGetRegionGroupsByTimeResp successRegionGroupsResp( final Set<TRegionReplicaSet> replicaSets) { final TGetRegionGroupsByTimeResp resp =
