This is an automated email from the ASF dual-hosted git repository. nic pushed a commit to branch 2.6.x in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 26b0588fd290bb0ee0fb49f4dc9b7b0931da31aa Author: Ma,Gang <ga...@ebay.com> AuthorDate: Wed Jan 30 15:40:11 2019 +0800 KYLIN-3797 Too many or filters may break Kylin server when flatting filter --- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++ .../apache/kylin/metadata/filter/TupleFilter.java | 30 ++++++++-- .../kylin/metadata/filter/TupleFilterTest.java | 64 ++++++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 91c86b5..aa8626f 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1706,6 +1706,10 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(this.getOptional("kylin.query.cache-signature-enabled", FALSE)); } + public int getFlatFilterMaxChildrenSize() { + return Integer.parseInt(this.getOptional("kylin.query.flat-filter-max-children", "500000")); + } + // ============================================================================ // SERVER // ============================================================================ diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java index 672aba0..019960e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; import org.slf4j.Logger; @@ -215,10 +216,19 @@ public abstract class TupleFilter { * @return */ public TupleFilter flatFilter() { - return flattenInternal(this); + return flatFilter(KylinConfig.getInstanceFromEnv().getFlatFilterMaxChildrenSize()); } - private TupleFilter flattenInternal(TupleFilter filter) { + /** + * throws IllegalStateException when the flat children exceed the maxFlatChildrenSize + * @param maxFlatChildrenSize + * @return + */ + public TupleFilter flatFilter(int maxFlatChildrenSize) { + return flattenInternal(this, maxFlatChildrenSize); + } + + private TupleFilter flattenInternal(TupleFilter filter, int maxFlatChildrenSize) { TupleFilter flatFilter = null; if (!(filter instanceof LogicalTupleFilter)) { flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND); @@ -231,7 +241,7 @@ public abstract class TupleFilter { List<TupleFilter> andChildren = new LinkedList<TupleFilter>(); List<TupleFilter> orChildren = new LinkedList<TupleFilter>(); for (TupleFilter child : filter.getChildren()) { - TupleFilter flatChild = flattenInternal(child); + TupleFilter flatChild = flattenInternal(child, maxFlatChildrenSize); FilterOperatorEnum childOp = flatChild.getOperator(); if (childOp == FilterOperatorEnum.AND) { andChildren.add(flatChild); @@ -249,7 +259,7 @@ public abstract class TupleFilter { flatFilter.addChildren(andChild.getChildren()); } if (!orChildren.isEmpty()) { - List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter); + List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter, maxFlatChildrenSize); flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR); flatFilter.addChildren(fullAndFilters); } @@ -262,14 +272,18 @@ public abstract class TupleFilter { } else if (op == FilterOperatorEnum.NOT) { assert (filter.children.size() == 1); TupleFilter reverse = filter.children.get(0).reverse(); - flatFilter = flattenInternal(reverse); + flatFilter = flattenInternal(reverse, maxFlatChildrenSize); } else { throw new IllegalStateException("Filter is " + filter); } + if (flatFilter.getChildren() != null && flatFilter.getChildren().size() > maxFlatChildrenSize) { + throw new IllegalStateException("the filter is too large after do the flat, size=" + + flatFilter.getChildren().size()); + } return flatFilter; } - private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter) { + private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter, int maxFlatChildrenSize) { List<TupleFilter> oldProductFilters = new LinkedList<TupleFilter>(); oldProductFilters.add(partialAndFilter); for (TupleFilter orFilter : leftOrFilters) { @@ -279,6 +293,10 @@ public abstract class TupleFilter { TupleFilter fullAndFilter = productFilter.copy(); fullAndFilter.addChildren(orChildFilter.getChildren()); newProductFilters.add(fullAndFilter); + if (newProductFilters.size() > maxFlatChildrenSize) { + throw new IllegalStateException("the filter is too large after do the flat, size=" + + newProductFilters.size()); + } } } oldProductFilters = newProductFilters; diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java index 95609eb..6a3843b 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java @@ -28,6 +28,7 @@ import java.util.Set; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum; import org.junit.Assert; import org.junit.Test; @@ -138,5 +139,68 @@ public class TupleFilterTest { Assert.assertEquals(Sets.newHashSet(compFilter2), compFilter2.findMustTrueCompareFilters()); } + + @Test + public void flatFilterTest() { + TupleFilter topAndFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND); + topAndFilter.addChild(createEQFilter("c1", "v1")); + TupleFilter orFilter1 = new LogicalTupleFilter(FilterOperatorEnum.OR); + TupleFilter andFilter11 = new LogicalTupleFilter(FilterOperatorEnum.AND); + andFilter11.addChild(createEQFilter("c2", "v2")); + andFilter11.addChild(createEQFilter("c3", "v3")); + + TupleFilter andFilter12 = new LogicalTupleFilter(FilterOperatorEnum.AND); + andFilter12.addChild(createEQFilter("c2", "v21")); + andFilter12.addChild(createEQFilter("c3", "v31")); + + orFilter1.addChild(andFilter11); + orFilter1.addChild(andFilter12); + + TupleFilter orFilter2 = new LogicalTupleFilter(FilterOperatorEnum.OR); + TupleFilter andFilter21 = new LogicalTupleFilter(FilterOperatorEnum.AND); + andFilter21.addChild(createEQFilter("c4", "v4")); + andFilter21.addChild(createEQFilter("c5", "v5")); + + TupleFilter andFilter22 = new LogicalTupleFilter(FilterOperatorEnum.AND); + andFilter22.addChild(createEQFilter("c4", "v41")); + andFilter22.addChild(createEQFilter("c5", "v51")); + + TupleFilter andFilter23 = new LogicalTupleFilter(FilterOperatorEnum.AND); + andFilter23.addChild(createEQFilter("c4", "v42")); + andFilter23.addChild(createEQFilter("c5", "v52")); + + orFilter2.addChild(andFilter21); + orFilter2.addChild(andFilter22); + orFilter2.addChild(andFilter23); + + topAndFilter.addChild(orFilter1); + topAndFilter.addChild(orFilter2); + + TupleFilter flatFilter = topAndFilter.flatFilter(500000); + Assert.assertEquals(6, flatFilter.children.size()); + } + + @Test(expected = IllegalStateException.class) + public void flatFilterTooFatTest() { + TupleFilter topAndFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND); + for (int i = 0; i < 3; i++) { + TupleFilter orFilter = new LogicalTupleFilter(FilterOperatorEnum.OR); + String col = "col-" + i; + for (int j = 0; j < 100; j++) { + orFilter.addChild(createEQFilter(col, String.valueOf(j))); + } + topAndFilter.addChild(orFilter); + } + TupleFilter flatFilter = topAndFilter.flatFilter(500000); + System.out.println(flatFilter); + } + + private TupleFilter createEQFilter(String colName, String colVal) { + CompareTupleFilter compareTupleFilter = new CompareTupleFilter(FilterOperatorEnum.EQ); + compareTupleFilter + .addChild(new ColumnTupleFilter(TblColRef.newInnerColumn(colName, InnerDataTypeEnum.LITERAL))); + compareTupleFilter.addChild(new ConstantTupleFilter(colVal)); + return compareTupleFilter; + } }