This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 24f1987 KYLIN-3797 Too many or filters may break Kylin server when
flatting filter
24f1987 is described below
commit 24f1987d71df1c51f9e1fca5885a2bfa4fca501d
Author: Ma,Gang <ga...@ebay.com>
AuthorDate: Wed Jan 30 15:40:11 2019 +0800
KYLIN-3797 Too many or filters may break Kylin server when flatting filter
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/metadata/filter/TupleFilter.java | 30 ++++++++--
.../kylin/metadata/filter/TupleFilterTest.java | 64 ++++++++++++++++++++++
3 files changed, 92 insertions(+), 6 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ab6d1a9..ecd663e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1673,6 +1673,10 @@ abstract public class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(this.getOptional("kylin.query.cache-signature-enabled",
FALSE));
}
+ public int getFlatFilterMaxChildrenSize() {
+ return
Integer.parseInt(this.getOptional("kylin.query.flat-filter-max-children",
"500000"));
+ }
+
//
============================================================================
// SERVER
//
============================================================================
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 672aba0..019960e 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.slf4j.Logger;
@@ -215,10 +216,19 @@ public abstract class TupleFilter {
* @return
*/
public TupleFilter flatFilter() {
- return flattenInternal(this);
+ return
flatFilter(KylinConfig.getInstanceFromEnv().getFlatFilterMaxChildrenSize());
}
- private TupleFilter flattenInternal(TupleFilter filter) {
+ /**
+ * throws IllegalStateException when the flat children exceed the
maxFlatChildrenSize
+ * @param maxFlatChildrenSize
+ * @return
+ */
+ public TupleFilter flatFilter(int maxFlatChildrenSize) {
+ return flattenInternal(this, maxFlatChildrenSize);
+ }
+
+ private TupleFilter flattenInternal(TupleFilter filter, int
maxFlatChildrenSize) {
TupleFilter flatFilter = null;
if (!(filter instanceof LogicalTupleFilter)) {
flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
@@ -231,7 +241,7 @@ public abstract class TupleFilter {
List<TupleFilter> andChildren = new LinkedList<TupleFilter>();
List<TupleFilter> orChildren = new LinkedList<TupleFilter>();
for (TupleFilter child : filter.getChildren()) {
- TupleFilter flatChild = flattenInternal(child);
+ TupleFilter flatChild = flattenInternal(child,
maxFlatChildrenSize);
FilterOperatorEnum childOp = flatChild.getOperator();
if (childOp == FilterOperatorEnum.AND) {
andChildren.add(flatChild);
@@ -249,7 +259,7 @@ public abstract class TupleFilter {
flatFilter.addChildren(andChild.getChildren());
}
if (!orChildren.isEmpty()) {
- List<TupleFilter> fullAndFilters =
cartesianProduct(orChildren, flatFilter);
+ List<TupleFilter> fullAndFilters =
cartesianProduct(orChildren, flatFilter, maxFlatChildrenSize);
flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
flatFilter.addChildren(fullAndFilters);
}
@@ -262,14 +272,18 @@ public abstract class TupleFilter {
} else if (op == FilterOperatorEnum.NOT) {
assert (filter.children.size() == 1);
TupleFilter reverse = filter.children.get(0).reverse();
- flatFilter = flattenInternal(reverse);
+ flatFilter = flattenInternal(reverse, maxFlatChildrenSize);
} else {
throw new IllegalStateException("Filter is " + filter);
}
+ if (flatFilter.getChildren() != null &&
flatFilter.getChildren().size() > maxFlatChildrenSize) {
+ throw new IllegalStateException("the filter is too large after do
the flat, size="
+ + flatFilter.getChildren().size());
+ }
return flatFilter;
}
- private List<TupleFilter> cartesianProduct(List<TupleFilter>
leftOrFilters, TupleFilter partialAndFilter) {
+ private List<TupleFilter> cartesianProduct(List<TupleFilter>
leftOrFilters, TupleFilter partialAndFilter, int maxFlatChildrenSize) {
List<TupleFilter> oldProductFilters = new LinkedList<TupleFilter>();
oldProductFilters.add(partialAndFilter);
for (TupleFilter orFilter : leftOrFilters) {
@@ -279,6 +293,10 @@ public abstract class TupleFilter {
TupleFilter fullAndFilter = productFilter.copy();
fullAndFilter.addChildren(orChildFilter.getChildren());
newProductFilters.add(fullAndFilter);
+ if (newProductFilters.size() > maxFlatChildrenSize) {
+ throw new IllegalStateException("the filter is too
large after do the flat, size="
+ + newProductFilters.size());
+ }
}
}
oldProductFilters = newProductFilters;
diff --git
a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
index 95609eb..6a3843b 100644
---
a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
+++
b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;
import org.junit.Assert;
import org.junit.Test;
@@ -138,5 +139,68 @@ public class TupleFilterTest {
Assert.assertEquals(Sets.newHashSet(compFilter2),
compFilter2.findMustTrueCompareFilters());
}
+
+ @Test
+ public void flatFilterTest() {
+ TupleFilter topAndFilter = new
LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ topAndFilter.addChild(createEQFilter("c1", "v1"));
+ TupleFilter orFilter1 = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ TupleFilter andFilter11 = new
LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter11.addChild(createEQFilter("c2", "v2"));
+ andFilter11.addChild(createEQFilter("c3", "v3"));
+
+ TupleFilter andFilter12 = new
LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter12.addChild(createEQFilter("c2", "v21"));
+ andFilter12.addChild(createEQFilter("c3", "v31"));
+
+ orFilter1.addChild(andFilter11);
+ orFilter1.addChild(andFilter12);
+
+ TupleFilter orFilter2 = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ TupleFilter andFilter21 = new
LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter21.addChild(createEQFilter("c4", "v4"));
+ andFilter21.addChild(createEQFilter("c5", "v5"));
+
+ TupleFilter andFilter22 = new
LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter22.addChild(createEQFilter("c4", "v41"));
+ andFilter22.addChild(createEQFilter("c5", "v51"));
+
+ TupleFilter andFilter23 = new
LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter23.addChild(createEQFilter("c4", "v42"));
+ andFilter23.addChild(createEQFilter("c5", "v52"));
+
+ orFilter2.addChild(andFilter21);
+ orFilter2.addChild(andFilter22);
+ orFilter2.addChild(andFilter23);
+
+ topAndFilter.addChild(orFilter1);
+ topAndFilter.addChild(orFilter2);
+
+ TupleFilter flatFilter = topAndFilter.flatFilter(500000);
+ Assert.assertEquals(6, flatFilter.children.size());
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void flatFilterTooFatTest() {
+ TupleFilter topAndFilter = new
LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ for (int i = 0; i < 3; i++) {
+ TupleFilter orFilter = new
LogicalTupleFilter(FilterOperatorEnum.OR);
+ String col = "col-" + i;
+ for (int j = 0; j < 100; j++) {
+ orFilter.addChild(createEQFilter(col, String.valueOf(j)));
+ }
+ topAndFilter.addChild(orFilter);
+ }
+ TupleFilter flatFilter = topAndFilter.flatFilter(500000);
+ System.out.println(flatFilter);
+ }
+
+ private TupleFilter createEQFilter(String colName, String colVal) {
+ CompareTupleFilter compareTupleFilter = new
CompareTupleFilter(FilterOperatorEnum.EQ);
+ compareTupleFilter
+ .addChild(new
ColumnTupleFilter(TblColRef.newInnerColumn(colName,
InnerDataTypeEnum.LITERAL)));
+ compareTupleFilter.addChild(new ConstantTupleFilter(colVal));
+ return compareTupleFilter;
+ }
}
