[
https://issues.apache.org/jira/browse/DRILL-5796?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16512638#comment-16512638
]
ASF GitHub Bot commented on DRILL-5796:
---------------------------------------
arina-ielchiieva closed pull request #1298: DRILL-5796: Filter pruning for
multi rowgroup parquet file
URL: https://github.com/apache/drill/pull/1298
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetBooleanPredicate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetBooleanPredicate.java
index fa5c4672a8..e156754f0d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetBooleanPredicate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetBooleanPredicate.java
@@ -27,8 +27,7 @@
/**
* Boolean predicates for parquet filter pushdown.
*/
-public abstract class ParquetBooleanPredicate<C extends Comparable<C>> extends
BooleanOperator
- implements ParquetFilterPredicate<C> {
+public abstract class ParquetBooleanPredicate<C extends Comparable<C>> extends
BooleanOperator implements ParquetFilterPredicate<C> {
private ParquetBooleanPredicate(String name, List<LogicalExpression> args,
ExpressionPosition pos) {
super(name, args, pos);
@@ -40,51 +39,72 @@ private ParquetBooleanPredicate(String name,
List<LogicalExpression> args, Expre
}
@SuppressWarnings("unchecked")
- private static <C extends Comparable<C>> LogicalExpression
createAndPredicate(
- String name,
- List<LogicalExpression> args,
- ExpressionPosition pos
- ) {
+ private static <C extends Comparable<C>> LogicalExpression
createAndPredicate(String name, List<LogicalExpression> args,
ExpressionPosition pos) {
return new ParquetBooleanPredicate<C>(name, args, pos) {
+ /**
+ * Evaluates a compound "AND" filter on the statistics of a RowGroup
(the filter reads "filterA and filterB").
+ * Return value :<ul>
+ * <li>ALL : only if all filters return ALL
+ * <li>NONE : if one filter at least returns NONE
+ * <li>INAPPLICABLE : all other cases
+ * </ul>
+ */
@Override
- public boolean canDrop(RangeExprEvaluator<C> evaluator) {
- // "and" : as long as one branch is OK to drop, we can drop it.
+ public ROWS_MATCH matches(RangeExprEvaluator<C> evaluator) {
+ ROWS_MATCH childMatch, lastMatch = ROWS_MATCH.NONE;
for (LogicalExpression child : this) {
- if (child instanceof ParquetFilterPredicate &&
((ParquetFilterPredicate)child).canDrop(evaluator)) {
- return true;
+ if (child instanceof ParquetFilterPredicate) {
+ childMatch = ((ParquetFilterPredicate) child).matches(evaluator);
+ if (childMatch != ROWS_MATCH.ALL) {
+ if (childMatch == ROWS_MATCH.NONE) {
+ return ROWS_MATCH.NONE; // No row comply to 1 filter part =>
can drop RG
+ }
+ lastMatch = childMatch;
+ }
}
}
- return false;
+ if (lastMatch != ROWS_MATCH.NONE) {
+ return ROWS_MATCH.INAPPLICABLE; // One at least was INAPPLICABLE or
SOME => can not drop RG nor filter
+ }
+ return ROWS_MATCH.ALL; // Comply to full filter => can drop filter but
not RG
}
};
}
@SuppressWarnings("unchecked")
- private static <C extends Comparable<C>> LogicalExpression createOrPredicate(
- String name,
- List<LogicalExpression> args,
- ExpressionPosition pos
- ) {
+ private static <C extends Comparable<C>> LogicalExpression
createOrPredicate(String name, List<LogicalExpression> args, ExpressionPosition
pos) {
return new ParquetBooleanPredicate<C>(name, args, pos) {
+ /**
+ * Evaluates a compound "OR" filter on the statistics of a RowGroup (the
filter reads "filterA or filterB").
+ * Return value :<ul>
+ * <li>NONE : only if all filters return NONE
+ * <li>ALL : if one filter at least returns ALL
+ * <li>INAPPLICABLE : all other cases
+ * </ul>
+ */
@Override
- public boolean canDrop(RangeExprEvaluator<C> evaluator) {
+ public ROWS_MATCH matches(RangeExprEvaluator<C> evaluator) {
+ ROWS_MATCH childMatch, lastMatch = ROWS_MATCH.NONE;
for (LogicalExpression child : this) {
- // "or" : as long as one branch is NOT ok to drop, we can NOT drop
it.
- if (!(child instanceof ParquetFilterPredicate) ||
!((ParquetFilterPredicate)child).canDrop(evaluator)) {
- return false;
+ if (child instanceof ParquetFilterPredicate) {
+ childMatch = ((ParquetFilterPredicate) child).matches(evaluator);
+ if (childMatch != ROWS_MATCH.NONE) {
+ if (childMatch == ROWS_MATCH.ALL) {
+ return ROWS_MATCH.ALL; // One at least is ALL => can drop
filter but not RG
+ }
+ lastMatch = childMatch;
+ }
}
}
- return true;
+ if (lastMatch != ROWS_MATCH.NONE) {
+ return ROWS_MATCH.INAPPLICABLE; // One at least was INAPPLICABLE or
SOME => can not drop RG nor filter
+ }
+ return ROWS_MATCH.NONE; // All are NONE => can drop RG
}
};
}
- public static <C extends Comparable<C>> LogicalExpression
createBooleanPredicate(
- String function,
- String name,
- List<LogicalExpression> args,
- ExpressionPosition pos
- ) {
+ public static <C extends Comparable<C>> LogicalExpression
createBooleanPredicate(String function, String name, List<LogicalExpression>
args, ExpressionPosition pos) {
switch (function) {
case "booleanOr":
return ParquetBooleanPredicate.<C>createOrPredicate(name, args, pos);
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetComparisonPredicate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetComparisonPredicate.java
index ebceefb435..292267b8b9 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetComparisonPredicate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetComparisonPredicate.java
@@ -26,7 +26,6 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.function.BiPredicate;
import static
org.apache.drill.exec.expr.stat.ParquetPredicatesHelper.isNullOrEmpty;
import static
org.apache.drill.exec.expr.stat.ParquetPredicatesHelper.isAllNulls;
@@ -34,21 +33,15 @@
/**
* Comparison predicates for parquet filter pushdown.
*/
-public class ParquetComparisonPredicate<C extends Comparable<C>> extends
LogicalExpressionBase
- implements ParquetFilterPredicate<C> {
+public abstract class ParquetComparisonPredicate<C extends Comparable<C>>
extends LogicalExpressionBase implements ParquetFilterPredicate<C> {
private final LogicalExpression left;
+
private final LogicalExpression right;
- private final BiPredicate<Statistics<C>, Statistics<C>> predicate;
- private ParquetComparisonPredicate(
- LogicalExpression left,
- LogicalExpression right,
- BiPredicate<Statistics<C>, Statistics<C>> predicate
- ) {
+ private ParquetComparisonPredicate(LogicalExpression left, LogicalExpression
right) {
super(left.getPosition());
this.left = left;
this.right = right;
- this.predicate = predicate;
}
@Override
@@ -65,10 +58,10 @@ private ParquetComparisonPredicate(
}
/**
- * Semantics of canDrop() is very similar to what is implemented in Parquet
library's
+ * Semantics of matches() is very similar to what is implemented in Parquet
library's
* {@link org.apache.parquet.filter2.statisticslevel.StatisticsFilter} and
* {@link org.apache.parquet.filter2.predicate.FilterPredicate}
- *
+ * <p>
* Main difference :
* 1. A RangeExprEvaluator is used to compute the min/max of an expression,
such as CAST function
* of a column. CAST function could be explicitly added by Drill user (It's
recommended to use CAST
@@ -83,38 +76,53 @@ private ParquetComparisonPredicate(
* where Column1 and Column2 are from same parquet table.
*/
@Override
- public boolean canDrop(RangeExprEvaluator<C> evaluator) {
+ public ROWS_MATCH matches(RangeExprEvaluator<C> evaluator) {
Statistics<C> leftStat = left.accept(evaluator, null);
if (isNullOrEmpty(leftStat)) {
- return false;
+ return ROWS_MATCH.INAPPLICABLE;
}
-
Statistics<C> rightStat = right.accept(evaluator, null);
if (isNullOrEmpty(rightStat)) {
- return false;
+ return ROWS_MATCH.INAPPLICABLE;
}
-
- // if either side is ALL null, = is evaluated to UNKNOWN -> canDrop
if (isAllNulls(leftStat, evaluator.getRowCount()) || isAllNulls(rightStat,
evaluator.getRowCount())) {
- return true;
+ return ROWS_MATCH.NONE;
}
+ return apply(leftStat, rightStat);
+ }
- return (leftStat.hasNonNullValue() && rightStat.hasNonNullValue()) &&
predicate.test(leftStat, rightStat);
+ /**
+ * If the tested rowgroup contains some null values, change the
ROWS_MATCH.ALL into ROWS_MATCH.SOME (null values should be discarded by filter)
+ *
+ * @param leftStat the rowgroup statistics to check
+ * @return the corresponding ROWS_MATCH
+ */
+ ROWS_MATCH checkNull(Statistics leftStat) {
+ return leftStat.getNumNulls() > 0 ? ROWS_MATCH.SOME : ROWS_MATCH.ALL;
}
+ /**
+ * Apply the filter condition to the statistics of the leftstat rowgroup.
+ *
+ * @param leftStat the statistics to test against filter
+ * @param rightStat a representation of the filter condition
+ * @return the corresponding ROWS_MATCH case
+ */
+ abstract ROWS_MATCH apply(Statistics leftStat, Statistics rightStat);
+
/**
* EQ (=) predicate
*/
- private static <C extends Comparable<C>> LogicalExpression
createEqualPredicate(
- LogicalExpression left,
- LogicalExpression right
- ) {
- return new ParquetComparisonPredicate<C>(left, right, (leftStat,
rightStat) -> {
- // can drop when left's max < right's min, or right's max < left's min
- final C leftMin = leftStat.genericGetMin();
- final C rightMin = rightStat.genericGetMin();
- return (leftStat.compareMaxToValue(rightMin) < 0) ||
(rightStat.compareMaxToValue(leftMin) < 0);
- }) {
+ private static <C extends Comparable<C>> LogicalExpression
createEqualPredicate(LogicalExpression left, LogicalExpression right) {
+ return new ParquetComparisonPredicate<C>(left, right) {
+ @Override
+ protected ROWS_MATCH apply(Statistics leftStat, Statistics rightStat) {
+ if ((leftStat.genericGetMax().compareTo(rightStat.genericGetMin()) < 0
|| rightStat.genericGetMax().compareTo(leftStat.genericGetMin()) < 0)) {
+ return ROWS_MATCH.NONE;
+ }
+ return ROWS_MATCH.SOME;
+ }
+
@Override
public String toString() {
return left + " = " + right;
@@ -125,79 +133,74 @@ public String toString() {
/**
* GT (>) predicate.
*/
- private static <C extends Comparable<C>> LogicalExpression createGTPredicate(
- LogicalExpression left,
- LogicalExpression right
- ) {
- return new ParquetComparisonPredicate<C>(left, right, (leftStat,
rightStat) -> {
- // can drop when left's max <= right's min.
- final C rightMin = rightStat.genericGetMin();
- return leftStat.compareMaxToValue(rightMin) <= 0;
- });
+ private static <C extends Comparable<C>> LogicalExpression
createGTPredicate(LogicalExpression left, LogicalExpression right) {
+ return new ParquetComparisonPredicate<C>(left, right) {
+ protected ROWS_MATCH apply(Statistics leftStat, Statistics rightStat) {
+ if (leftStat.genericGetMax().compareTo(rightStat.genericGetMin()) <=
0) {
+ return ROWS_MATCH.NONE;
+ }
+ return leftStat.genericGetMin().compareTo(rightStat.genericGetMax()) >
0 ? checkNull(leftStat) : ROWS_MATCH.SOME;
+ }
+ };
}
/**
* GE (>=) predicate.
*/
- private static <C extends Comparable<C>> LogicalExpression createGEPredicate(
- LogicalExpression left,
- LogicalExpression right
- ) {
- return new ParquetComparisonPredicate<C>(left, right, (leftStat,
rightStat) -> {
- // can drop when left's max < right's min.
- final C rightMin = rightStat.genericGetMin();
- return leftStat.compareMaxToValue(rightMin) < 0;
- });
+ private static <C extends Comparable<C>> LogicalExpression
createGEPredicate(LogicalExpression left, LogicalExpression right) {
+ return new ParquetComparisonPredicate<C>(left, right) {
+ protected ROWS_MATCH apply(Statistics leftStat, Statistics rightStat) {
+ if (leftStat.genericGetMax().compareTo(rightStat.genericGetMin()) < 0)
{
+ return ROWS_MATCH.NONE;
+ }
+ return leftStat.genericGetMin().compareTo(rightStat.genericGetMax())
>= 0 ? checkNull(leftStat) : ROWS_MATCH.SOME;
+ }
+ };
}
/**
* LT (<) predicate.
*/
- private static <C extends Comparable<C>> LogicalExpression createLTPredicate(
- LogicalExpression left,
- LogicalExpression right
- ) {
- return new ParquetComparisonPredicate<C>(left, right, (leftStat,
rightStat) -> {
- // can drop when right's max <= left's min.
- final C leftMin = leftStat.genericGetMin();
- return rightStat.compareMaxToValue(leftMin) <= 0;
- });
+ private static <C extends Comparable<C>> LogicalExpression
createLTPredicate(LogicalExpression left, LogicalExpression right) {
+ return new ParquetComparisonPredicate<C>(left, right) {
+ protected ROWS_MATCH apply(Statistics leftStat, Statistics rightStat) {
+ if (rightStat.genericGetMax().compareTo(leftStat.genericGetMin()) <=
0) {
+ return ROWS_MATCH.NONE;
+ }
+ return leftStat.genericGetMax().compareTo(rightStat.genericGetMin()) <
0 ? checkNull(leftStat) : ROWS_MATCH.SOME;
+ }
+ };
}
/**
* LE (<=) predicate.
*/
- private static <C extends Comparable<C>> LogicalExpression createLEPredicate(
- LogicalExpression left, LogicalExpression right
- ) {
- return new ParquetComparisonPredicate<C>(left, right, (leftStat,
rightStat) -> {
- // can drop when right's max < left's min.
- final C leftMin = leftStat.genericGetMin();
- return rightStat.compareMaxToValue(leftMin) < 0;
- });
+ private static <C extends Comparable<C>> LogicalExpression
createLEPredicate(LogicalExpression left, LogicalExpression right) {
+ return new ParquetComparisonPredicate<C>(left, right) {
+ protected ROWS_MATCH apply(Statistics leftStat, Statistics rightStat) {
+ if (rightStat.genericGetMax().compareTo(leftStat.genericGetMin()) < 0)
{
+ return ROWS_MATCH.NONE;
+ }
+ return leftStat.genericGetMax().compareTo(rightStat.genericGetMin())
<= 0 ? checkNull(leftStat) : ROWS_MATCH.SOME;
+ }
+ };
}
/**
* NE (!=) predicate.
*/
- private static <C extends Comparable<C>> LogicalExpression createNEPredicate(
- LogicalExpression left,
- LogicalExpression right
- ) {
- return new ParquetComparisonPredicate<C>(left, right, (leftStat,
rightStat) -> {
- // can drop when there is only one unique value.
- final C leftMax = leftStat.genericGetMax();
- final C rightMax = rightStat.genericGetMax();
- return leftStat.compareMinToValue(leftMax) == 0 &&
rightStat.compareMinToValue(rightMax) == 0 &&
- leftStat.compareMaxToValue(rightMax) == 0;
- });
+ private static <C extends Comparable<C>> LogicalExpression
createNEPredicate(LogicalExpression left, LogicalExpression right) {
+ return new ParquetComparisonPredicate<C>(left, right) {
+ protected ROWS_MATCH apply(Statistics leftStat, Statistics rightStat) {
+ if (leftStat.genericGetMax().compareTo(rightStat.genericGetMin()) < 0
|| rightStat.genericGetMax().compareTo(leftStat.genericGetMin()) < 0) {
+ return checkNull(leftStat);
+ }
+ return leftStat.genericGetMax().compareTo(rightStat.genericGetMax())
== 0 && leftStat.genericGetMin().compareTo(rightStat.genericGetMin()) == 0 ?
ROWS_MATCH.NONE : ROWS_MATCH.INAPPLICABLE;
+ }
+ };
}
- public static <C extends Comparable<C>> LogicalExpression
createComparisonPredicate(
- String function,
- LogicalExpression left,
- LogicalExpression right
- ) {
+ public static <C extends Comparable<C>> LogicalExpression
createComparisonPredicate(String function, LogicalExpression left,
LogicalExpression right) {
switch (function) {
case FunctionGenerationHelper.EQ:
return ParquetComparisonPredicate.<C>createEqualPredicate(left, right);
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetFilterPredicate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetFilterPredicate.java
index 1b7e9e5934..ff6a7ab420 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetFilterPredicate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetFilterPredicate.java
@@ -18,5 +18,17 @@
package org.apache.drill.exec.expr.stat;
public interface ParquetFilterPredicate<T extends Comparable<T>> {
- boolean canDrop(RangeExprEvaluator<T> evaluator);
+
+ /**
+ * Define the validity of a row group against a filter
+ * <ul>
+ * <li>ALL : all rows match the filter (can not drop the row group and
prune filter)
+ * <li>NONE : no row matches the filter (can drop the row group)
+ * <li>SOME : some rows only match the filter (can not drop the row group
not the filter)
+ * <li>INAPPLICABLE : filter can not be applied
+ * </ul>
+ */
+ enum ROWS_MATCH {ALL, NONE, SOME, INAPPLICABLE}
+
+ ROWS_MATCH matches(RangeExprEvaluator<T> evaluator);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicate.java
index 547dc06704..2a22fcd35e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicate.java
@@ -19,35 +19,28 @@
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.LogicalExpressionBase;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.parquet.column.statistics.BooleanStatistics;
import org.apache.parquet.column.statistics.Statistics;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.function.BiPredicate;
import static
org.apache.drill.exec.expr.stat.ParquetPredicatesHelper.hasNoNulls;
import static
org.apache.drill.exec.expr.stat.ParquetPredicatesHelper.isAllNulls;
-import static
org.apache.drill.exec.expr.stat.ParquetPredicatesHelper.isNullOrEmpty;
/**
* IS predicates for parquet filter pushdown.
*/
-public class ParquetIsPredicate<C extends Comparable<C>> extends
LogicalExpressionBase
- implements ParquetFilterPredicate<C> {
+public abstract class ParquetIsPredicate<C extends Comparable<C>> extends
LogicalExpressionBase implements ParquetFilterPredicate<C> {
- private final LogicalExpression expr;
- private final BiPredicate<Statistics<C>, RangeExprEvaluator<C>> predicate;
+ private LogicalExpression expr;
- private ParquetIsPredicate(LogicalExpression expr,
BiPredicate<Statistics<C>, RangeExprEvaluator<C>> predicate) {
+ private ParquetIsPredicate(LogicalExpression expr) {
super(expr.getPosition());
this.expr = expr;
- this.predicate = predicate;
}
@Override
@@ -62,39 +55,62 @@ private ParquetIsPredicate(LogicalExpression expr,
BiPredicate<Statistics<C>, Ra
return visitor.visitUnknown(this, value);
}
- @Override
- public boolean canDrop(RangeExprEvaluator<C> evaluator) {
+ /**
+ * Apply the filter condition against the meta of the rowgroup.
+ */
+ public ROWS_MATCH matches(RangeExprEvaluator<C> evaluator) {
Statistics<C> exprStat = expr.accept(evaluator, null);
- if (isNullOrEmpty(exprStat)) {
- return false;
- }
+ return ParquetPredicatesHelper.isNullOrEmpty(exprStat) ?
ROWS_MATCH.INAPPLICABLE : apply(exprStat, evaluator);
+ }
+
+ /**
+ * Apply the filter condition to the statistics of the leftstat rowgroup.
+ *
+ * @param exprStat the statistics to test against filter
+ * @param evaluator a representation of the filter condition
+ * @return the corresponding ROWS_MATCH case
+ */
+ abstract ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator);
- return predicate.test(exprStat, evaluator);
+ /**
+ * After the applying of the filter against the statistics of the rowgroup,
if the result is ROWS_MATCH.ALL,
+ * then we still must know if the rowgroup contains some null values,
because it can change the filter result.
+ * If it contains some null values, then we change the ROWS_MATCH.ALL into
ROWS_MATCH.SOME, which sya that maybe
+ * some values (the null ones) should be disgarded.
+ */
+ ROWS_MATCH checkNull(Statistics<C> exprStat) {
+ return hasNoNulls(exprStat) ? ROWS_MATCH.ALL : ROWS_MATCH.SOME;
}
/**
* IS NULL predicate.
*/
private static <C extends Comparable<C>> LogicalExpression
createIsNullPredicate(LogicalExpression expr) {
- return new ParquetIsPredicate<C>(expr,
- //if there are no nulls -> canDrop
- (exprStat, evaluator) -> hasNoNulls(exprStat)) {
- private final boolean isArray = isArray(expr);
-
+ return new ParquetIsPredicate<C>(expr) {
+ /**
+ * Check if the given expression concerns an array (repeated) or not.
+ *
+ * @return true if it is an array, false otherwise.
+ */
private boolean isArray(LogicalExpression expression) {
if (expression instanceof TypedFieldExpr) {
TypedFieldExpr typedFieldExpr = (TypedFieldExpr) expression;
- SchemaPath schemaPath = typedFieldExpr.getPath();
- return schemaPath.isArray();
+ return typedFieldExpr.getPath().isArray();
}
return false;
}
@Override
- public boolean canDrop(RangeExprEvaluator<C> evaluator) {
+ public ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator) {
// for arrays we are not able to define exact number of nulls
// [1,2,3] vs [1,2] -> in second case 3 is absent and thus it's null
but statistics shows no nulls
- return !isArray && super.canDrop(evaluator);
+ if (isArray(expr)) {
+ return ROWS_MATCH.INAPPLICABLE;
+ }
+ if (hasNoNulls(exprStat)) {
+ return ROWS_MATCH.NONE;
+ }
+ return isAllNulls(exprStat, evaluator.getRowCount()) ? ROWS_MATCH.ALL
: ROWS_MATCH.SOME;
}
};
}
@@ -103,50 +119,63 @@ public boolean canDrop(RangeExprEvaluator<C> evaluator) {
* IS NOT NULL predicate.
*/
private static <C extends Comparable<C>> LogicalExpression
createIsNotNullPredicate(LogicalExpression expr) {
- return new ParquetIsPredicate<C>(expr,
- //if there are all nulls -> canDrop
- (exprStat, evaluator) -> isAllNulls(exprStat, evaluator.getRowCount())
- );
+ return new ParquetIsPredicate<C>(expr) {
+ @Override
+ public ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator) {
+ return isAllNulls(exprStat, evaluator.getRowCount()) ? ROWS_MATCH.NONE
: checkNull(exprStat);
+ }
+ };
}
/**
* IS TRUE predicate.
*/
- private static LogicalExpression createIsTruePredicate(LogicalExpression
expr) {
- return new ParquetIsPredicate<Boolean>(expr,
- //if max value is not true or if there are all nulls -> canDrop
- (exprStat, evaluator) -> !((BooleanStatistics)exprStat).getMax() ||
isAllNulls(exprStat, evaluator.getRowCount())
- );
+ private static <C extends Comparable<C>> LogicalExpression
createIsTruePredicate(LogicalExpression expr) {
+ return new ParquetIsPredicate<C>(expr) {
+ @Override
+ public ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator) {
+ if (isAllNulls(exprStat, evaluator.getRowCount()) ||
(exprStat.genericGetMin().equals(Boolean.FALSE) &&
exprStat.genericGetMax().equals(Boolean.FALSE))) {
+ return ROWS_MATCH.NONE;
+ }
+ return exprStat.genericGetMin().equals(Boolean.TRUE) &&
exprStat.genericGetMax().equals(Boolean.TRUE) ? checkNull(exprStat) :
ROWS_MATCH.SOME;
+ }
+ };
}
/**
* IS FALSE predicate.
*/
- private static LogicalExpression createIsFalsePredicate(LogicalExpression
expr) {
- return new ParquetIsPredicate<Boolean>(expr,
- //if min value is not false or if there are all nulls -> canDrop
- (exprStat, evaluator) -> ((BooleanStatistics)exprStat).getMin() ||
isAllNulls(exprStat, evaluator.getRowCount())
- );
+ private static <C extends Comparable<C>> LogicalExpression
createIsFalsePredicate(LogicalExpression expr) {
+ return new ParquetIsPredicate<C>(expr) {
+ @Override
+ public ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator) {
+ return !exprStat.genericGetMin().equals(Boolean.FALSE) ||
isAllNulls(exprStat, evaluator.getRowCount()) ? ROWS_MATCH.NONE :
checkNull(exprStat);
+ }
+ };
}
/**
* IS NOT TRUE predicate.
*/
- private static LogicalExpression createIsNotTruePredicate(LogicalExpression
expr) {
- return new ParquetIsPredicate<Boolean>(expr,
- //if min value is not false or if there are no nulls -> canDrop
- (exprStat, evaluator) -> ((BooleanStatistics)exprStat).getMin() &&
hasNoNulls(exprStat)
- );
+ private static <C extends Comparable<C>> LogicalExpression
createIsNotTruePredicate(LogicalExpression expr) {
+ return new ParquetIsPredicate<C>(expr) {
+ @Override
+ public ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator) {
+ return !exprStat.genericGetMin().equals(Boolean.FALSE) &&
hasNoNulls(exprStat) ? ROWS_MATCH.NONE : checkNull(exprStat);
+ }
+ };
}
/**
* IS NOT FALSE predicate.
*/
- private static LogicalExpression createIsNotFalsePredicate(LogicalExpression
expr) {
- return new ParquetIsPredicate<Boolean>(expr,
- //if max value is not true or if there are no nulls -> canDrop
- (exprStat, evaluator) -> !((BooleanStatistics)exprStat).getMax() &&
hasNoNulls(exprStat)
- );
+ private static <C extends Comparable<C>> LogicalExpression
createIsNotFalsePredicate(LogicalExpression expr) {
+ return new ParquetIsPredicate<C>(expr) {
+ @Override
+ public ROWS_MATCH apply(Statistics<C> exprStat, RangeExprEvaluator<C>
evaluator) {
+ return !exprStat.genericGetMax().equals(Boolean.TRUE) &&
hasNoNulls(exprStat) ? ROWS_MATCH.NONE : checkNull(exprStat);
+ }
+ };
}
public static <C extends Comparable<C>> LogicalExpression
createIsPredicate(String function, LogicalExpression expr) {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
index f127f0b5b9..89ce62e19b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
@@ -259,17 +259,26 @@ public long getRowCount() {
final ValueHolder maxFuncHolder =
InterpreterEvaluator.evaluateFunction(interpreter, args2, holderExpr.getName());
switch (destType) {
- //TODO : need handle # of nulls.
case INT:
- return getStatistics( ((IntHolder)minFuncHolder).value,
((IntHolder)maxFuncHolder).value);
+ Statistics si = getStatistics( ((IntHolder)minFuncHolder).value,
((IntHolder)maxFuncHolder).value);
+ si.setNumNulls(input.getNumNulls());
+ return si;
case BIGINT:
- return getStatistics( ((BigIntHolder)minFuncHolder).value,
((BigIntHolder)maxFuncHolder).value);
+ Statistics sbi = getStatistics( ((BigIntHolder)minFuncHolder).value,
((BigIntHolder)maxFuncHolder).value);
+ sbi.setNumNulls(input.getNumNulls());
+ return sbi;
case FLOAT4:
- return getStatistics( ((Float4Holder)minFuncHolder).value,
((Float4Holder)maxFuncHolder).value);
+ Statistics sf4 = getStatistics( ((Float4Holder)minFuncHolder).value,
((Float4Holder)maxFuncHolder).value);
+ sf4.setNumNulls(input.getNumNulls());
+ return sf4;
case FLOAT8:
- return getStatistics( ((Float8Holder)minFuncHolder).value,
((Float8Holder)maxFuncHolder).value);
+ Statistics sf8 = getStatistics( ((Float8Holder)minFuncHolder).value,
((Float8Holder)maxFuncHolder).value);
+ sf8.setNumNulls(input.getNumNulls());
+ return sf8;
case TIMESTAMP:
- return getStatistics(((TimeStampHolder) minFuncHolder).value,
((TimeStampHolder) maxFuncHolder).value);
+ Statistics sts = getStatistics(((TimeStampHolder)
minFuncHolder).value, ((TimeStampHolder) maxFuncHolder).value);
+ sts.setNumNulls(input.getNumNulls());
+ return sts;
default:
return null;
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
index 33472bb654..11af7439d1 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
@@ -33,6 +33,7 @@
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.stat.ParquetFilterPredicate;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate.ROWS_MATCH;
import org.apache.drill.exec.ops.UdfUtilities;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.base.AbstractFileGroupScan;
@@ -261,13 +262,17 @@ public GroupScan applyFilter(LogicalExpression
filterExpr, UdfUtilities udfUtili
return null;
}
}
-
- if (ParquetRGFilterEvaluator.canDrop(filterPredicate,
columnStatisticsMap, rowGroup.getRowCount())) {
+ rowGroup.setRowsMatch(ParquetRGFilterEvaluator.matches(filterPredicate,
columnStatisticsMap, rowGroup.getRowCount()));
+ if (rowGroup.getRowsMatch() == ROWS_MATCH.NONE) {// No row comply to the
filter => drop the row group
continue;
}
+ // DRILL_6259 : in case of array, we can't know if the real value of a
repeated column exists until it is really filtered, hence we can't prune the
filter
+ if (rowGroup.getRowsMatch() == ROWS_MATCH.ALL &&
statCollector.isRepeated(schemaPathsInExpr)) {
+ rowGroup.setRowsMatch(ROWS_MATCH.SOME);
+ }
qualifiedRGs.add(rowGroup);
- qualifiedFilePath.add(rowGroup.getPath()); // TODO : optimize when 1
file contains m row groups.
+ qualifiedFilePath.add(rowGroup.getPath());
}
if (qualifiedRGs.size() == rowGroupInfos.size() ) {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
index 83ce4d2454..38f622a582 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
@@ -29,6 +29,7 @@
import org.apache.calcite.rex.RexUtil;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate.ROWS_MATCH;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.planner.common.DrillRelOptUtil;
@@ -165,12 +166,32 @@ protected void doOnMatch(RelOptRuleCall call, FilterPrel
filter, ProjectPrel pro
return;
}
-
- RelNode newScan = ScanPrel.create(scan, scan.getTraitSet(), newGroupScan,
scan.getRowType());;
+ RelNode newScan = ScanPrel.create(scan, scan.getTraitSet(), newGroupScan,
scan.getRowType());
if (project != null) {
newScan = project.copy(project.getTraitSet(), ImmutableList.of(newScan));
}
+
+ ROWS_MATCH matchAll = ROWS_MATCH.ALL;
+ if (newGroupScan instanceof AbstractParquetGroupScan) {
+ List<RowGroupInfo> rowGroupInfos = ((AbstractParquetGroupScan)
newGroupScan).rowGroupInfos;
+ for (RowGroupInfo rowGroup : rowGroupInfos) {
+ if (rowGroup.getRowsMatch() != ROWS_MATCH.ALL) {
+ matchAll = ROWS_MATCH.INAPPLICABLE;
+ break;
+ }
+ }
+ } else {
+ matchAll = ROWS_MATCH.INAPPLICABLE;
+ }
+ // If all groups are in ROWS_MATCH.ALL, no need to apply the filter again
to their rows => prune the filter
+ if (matchAll == ROWS_MATCH.ALL) {
+ call.transformTo(newScan);
+ } else {
+ final RelNode newFilter = filter.copy(filter.getTraitSet(),
ImmutableList.<RelNode>of(newScan));
+ call.transformTo(newFilter);
+ }
+
final RelNode newFilter = filter.copy(filter.getTraitSet(),
ImmutableList.<RelNode>of(newScan));
call.transformTo(newFilter);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRGFilterEvaluator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRGFilterEvaluator.java
index 370988b1c6..ab97a390db 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRGFilterEvaluator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRGFilterEvaluator.java
@@ -27,6 +27,7 @@
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionLookupContext;
import org.apache.drill.exec.expr.stat.ParquetFilterPredicate;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate.ROWS_MATCH;
import org.apache.drill.exec.expr.stat.RangeExprEvaluator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.UdfUtilities;
@@ -43,13 +44,13 @@
public class ParquetRGFilterEvaluator {
static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(ParquetRGFilterEvaluator.class);
- public static boolean evalFilter(LogicalExpression expr, ParquetMetadata
footer, int rowGroupIndex,
+ public static ROWS_MATCH evalFilter(LogicalExpression expr, ParquetMetadata
footer, int rowGroupIndex,
OptionManager options, FragmentContext fragmentContext) {
final HashMap<String, String> emptyMap = new HashMap<String, String>();
return evalFilter(expr, footer, rowGroupIndex, options, fragmentContext,
emptyMap);
}
- public static boolean evalFilter(LogicalExpression expr, ParquetMetadata
footer, int rowGroupIndex,
+ public static ROWS_MATCH evalFilter(LogicalExpression expr, ParquetMetadata
footer, int rowGroupIndex,
OptionManager options, FragmentContext fragmentContext, Map<String,
String> implicitColValues) {
// figure out the set of columns referenced in expression.
final Set<SchemaPath> schemaPathsInExpr = expr.accept(new
FieldReferenceFinder(), null);
@@ -57,23 +58,19 @@ public static boolean evalFilter(LogicalExpression expr,
ParquetMetadata footer,
Map<SchemaPath, ColumnStatistics> columnStatisticsMap =
columnStatCollector.collectColStat(schemaPathsInExpr);
- boolean canDrop = canDrop(expr, columnStatisticsMap,
footer.getBlocks().get(rowGroupIndex).getRowCount(), fragmentContext,
fragmentContext.getFunctionRegistry());
- return canDrop;
+ return matches(expr, columnStatisticsMap,
footer.getBlocks().get(rowGroupIndex).getRowCount(), fragmentContext,
fragmentContext.getFunctionRegistry());
}
-
- public static boolean canDrop(ParquetFilterPredicate parquetPredicate,
Map<SchemaPath,
+ public static ROWS_MATCH matches(ParquetFilterPredicate parquetPredicate,
Map<SchemaPath,
ColumnStatistics> columnStatisticsMap, long rowCount) {
- boolean canDrop = false;
if (parquetPredicate != null) {
RangeExprEvaluator rangeExprEvaluator = new
RangeExprEvaluator(columnStatisticsMap, rowCount);
- canDrop = parquetPredicate.canDrop(rangeExprEvaluator);
+ return parquetPredicate.matches(rangeExprEvaluator);
}
- return canDrop;
+ return ROWS_MATCH.INAPPLICABLE;
}
-
- public static boolean canDrop(LogicalExpression expr, Map<SchemaPath,
ColumnStatistics> columnStatisticsMap,
+ public static ROWS_MATCH matches(LogicalExpression expr, Map<SchemaPath,
ColumnStatistics> columnStatisticsMap,
long rowCount, UdfUtilities udfUtilities, FunctionLookupContext
functionImplementationRegistry) {
ErrorCollector errorCollector = new ErrorCollectorImpl();
LogicalExpression materializedFilter =
ExpressionTreeMaterializer.materializeFilterExpr(
@@ -82,14 +79,14 @@ public static boolean canDrop(LogicalExpression expr,
Map<SchemaPath, ColumnStat
if (errorCollector.hasErrors()) {
logger.error("{} error(s) encountered when materialize filter expression
: {}",
errorCollector.getErrorCount(), errorCollector.toErrorString());
- return false;
+ return ROWS_MATCH.INAPPLICABLE;
}
Set<LogicalExpression> constantBoundaries =
ConstantExpressionIdentifier.getConstantExpressionSet(materializedFilter);
ParquetFilterPredicate parquetPredicate = (ParquetFilterPredicate)
ParquetFilterBuilder.buildParquetFilterPredicate(
materializedFilter, constantBoundaries, udfUtilities);
- return canDrop(parquetPredicate, columnStatisticsMap, rowCount);
+ return matches(parquetPredicate, columnStatisticsMap, rowCount);
}
/**
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupInfo.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupInfo.java
index af436d8fc9..3df5bdaf60 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupInfo.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupInfo.java
@@ -19,6 +19,7 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate.ROWS_MATCH;
import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS;
import org.apache.drill.exec.store.dfs.easy.FileWork;
import org.apache.drill.exec.store.schedule.CompleteWork;
@@ -35,6 +36,7 @@
private List<? extends ColumnMetadata> columns;
private long rowCount; // rowCount = -1 indicates to include all rows.
private long numRecordsToRead;
+ private ROWS_MATCH rowsMatch = ROWS_MATCH.INAPPLICABLE;
@JsonCreator
public RowGroupInfo(@JsonProperty("path") String path,
@@ -95,4 +97,7 @@ public void setColumns(List<? extends ColumnMetadata>
columns) {
this.columns = columns;
}
+ public ROWS_MATCH getRowsMatch() { return rowsMatch; }
+
+ public void setRowsMatch(ROWS_MATCH rowsMatch) { this.rowsMatch = rowsMatch;
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetFooterStatCollector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetFooterStatCollector.java
index ac63bda1db..ef7d8faab7 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetFooterStatCollector.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetFooterStatCollector.java
@@ -59,7 +59,7 @@ public ParquetFooterStatCollector(ParquetMetadata footer, int
rowGroupIndex, Map
// Reasons to pass implicit columns and their values:
// 1. Differentiate implicit columns from regular non-exist columns.
Implicit columns do not
// exist in parquet metadata. Without such knowledge, implicit columns
is treated as non-exist
- // column. A condition on non-exist column would lead to canDrop =
true, which is not the
+ // column. A condition on non-exist column would lead to matches =
true, which is not the
// right behavior for condition on implicit columns.
// 2. Pass in the implicit column name with corresponding values, and wrap
them in Statistics with
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
index 933d8ee0eb..1791a9f12f 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
@@ -59,7 +59,7 @@ public ParquetMetaStatCollector(ParquetTableMetadataBase
parquetTableMetadata,
// Reasons to pass implicit columns and their values:
// 1. Differentiate implicit columns from regular non-exist columns.
Implicit columns do not
// exist in parquet metadata. Without such knowledge, implicit columns
is treated as non-exist
- // column. A condition on non-exist column would lead to canDrop =
true, which is not the
+ // column. A condition on non-exist column would lead to matches =
true, which is not the
// right behavior for condition on implicit columns.
// 2. Pass in the implicit column name with corresponding values, and wrap
them in Statistics with
@@ -71,6 +71,26 @@ public ParquetMetaStatCollector(ParquetTableMetadataBase
parquetTableMetadata,
this.implicitColValues = implicitColValues;
}
+ /**
+ * DRILL_6259 : check if one of the fields involved in the filter is an
array.
+ * @param fields columns used in the filter.
+ * @return true if one at least is an array, false otherwise.
+ */
+ public boolean isRepeated(Set<SchemaPath> fields) {
+ final Map<SchemaPath, ColumnMetadata> columnMetadataMap = new HashMap<>();
+ for (final ColumnMetadata columnMetadata : columnMetadataList) {
+ SchemaPath schemaPath =
SchemaPath.getCompoundPath(columnMetadata.getName());
+ columnMetadataMap.put(schemaPath, columnMetadata);
+ }
+ for (final SchemaPath field : fields) {
+ ColumnMetadata columnMetadata =
columnMetadataMap.get(field.getUnIndexed());
+ if (columnMetadata != null &&
parquetTableMetadata.getRepetitionLevel(columnMetadata.getName()) >= 1) {
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
public Map<SchemaPath, ColumnStatistics> collectColStat(Set<SchemaPath>
fields) {
Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() :
null;
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index 3bbd39713b..820480b785 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -20,6 +20,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.drill.PlanTestBase;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate.ROWS_MATCH;
import org.apache.drill.exec.ops.FragmentContextImpl;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.BitControl;
@@ -57,6 +58,7 @@ public static void initFSAndCreateFragContext() throws
Exception {
dirTestWatcher.copyResourceToRoot(Paths.get("parquetFilterPush"));
dirTestWatcher.copyResourceToRoot(Paths.get("parquet",
"multirowgroup.parquet"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("parquet",
"multirowgroup2.parquet"));
}
@AfterClass
@@ -97,73 +99,74 @@ public void testIntPredicateWithEval() throws Exception {
.toFile();
ParquetMetadata footer = getParquetMetaData(file);
- testParquetRowGroupFilterEval(footer, "intCol = 100", false);
- testParquetRowGroupFilterEval(footer, "intCol = 0", false);
- testParquetRowGroupFilterEval(footer, "intCol = 50", false);
+ testParquetRowGroupFilterEval(footer, "intCol = 100", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol = 0", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol = 50", ROWS_MATCH.SOME);
- testParquetRowGroupFilterEval(footer, "intCol = -1", true);
- testParquetRowGroupFilterEval(footer, "intCol = 101", true);
+ testParquetRowGroupFilterEval(footer, "intCol = -1", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol = 101", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "intCol > 100", true);
- testParquetRowGroupFilterEval(footer, "intCol > 99", false);
+ testParquetRowGroupFilterEval(footer, "intCol > 100", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol > 99", ROWS_MATCH.SOME);
- testParquetRowGroupFilterEval(footer, "intCol >= 100", false);
- testParquetRowGroupFilterEval(footer, "intCol >= 101", true);
+ testParquetRowGroupFilterEval(footer, "intCol >= 100", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol >= 101", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "intCol < 100", false);
- testParquetRowGroupFilterEval(footer, "intCol < 1", false);
- testParquetRowGroupFilterEval(footer, "intCol < 0", true);
+ testParquetRowGroupFilterEval(footer, "intCol < 100", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol < 1", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol < 0", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "intCol <= 100", false);
- testParquetRowGroupFilterEval(footer, "intCol <= 1", false);
- testParquetRowGroupFilterEval(footer, "intCol <= 0", false);
- testParquetRowGroupFilterEval(footer, "intCol <= -1", true);
+ testParquetRowGroupFilterEval(footer, "intCol <= 100", ROWS_MATCH.ALL);
+ testParquetRowGroupFilterEval(footer, "intCol <= 1", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol <= 0", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol <= -1", ROWS_MATCH.NONE);
// "and"
- testParquetRowGroupFilterEval(footer, "intCol > 100 and intCol < 200",
true);
- testParquetRowGroupFilterEval(footer, "intCol > 50 and intCol < 200",
false);
- testParquetRowGroupFilterEval(footer, "intCol > 50 and intCol > 200",
true); // essentially, intCol > 200
+ testParquetRowGroupFilterEval(footer, "intCol > 100 and intCol < 200",
ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol > 50 and intCol < 200",
ROWS_MATCH.INAPPLICABLE);
+ testParquetRowGroupFilterEval(footer, "intCol > 50 and intCol > 200",
ROWS_MATCH.NONE); // essentially, intCol > 200
// "or"
- testParquetRowGroupFilterEval(footer, "intCol = 150 or intCol = 160",
true);
- testParquetRowGroupFilterEval(footer, "intCol = 50 or intCol = 160",
false);
+ testParquetRowGroupFilterEval(footer, "intCol = 150 or intCol = 160",
ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol = 50 or intCol = 160",
ROWS_MATCH.INAPPLICABLE);
//"nonExistCol" does not exist in the table. "AND" with a filter on exist
column
- testParquetRowGroupFilterEval(footer, "intCol > 100 and nonExistCol =
100", true);
- testParquetRowGroupFilterEval(footer, "intCol > 50 and nonExistCol = 100",
true); // since nonExistCol = 100 -> Unknown -> could drop.
- testParquetRowGroupFilterEval(footer, "nonExistCol = 100 and intCol > 50",
true); // since nonExistCol = 100 -> Unknown -> could drop.
- testParquetRowGroupFilterEval(footer, "intCol > 100 and nonExistCol <
'abc'", true);
- testParquetRowGroupFilterEval(footer, "nonExistCol < 'abc' and intCol >
100", true); // nonExistCol < 'abc' hit NumberException and is ignored, but
intCol >100 will say "drop".
- testParquetRowGroupFilterEval(footer, "intCol > 50 and nonExistCol <
'abc'", false); // because nonExistCol < 'abc' hit NumberException and is
ignored.
+ testParquetRowGroupFilterEval(footer, "intCol > 100 and nonExistCol =
100", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol > 50 and nonExistCol = 100",
ROWS_MATCH.NONE); // since nonExistCol = 100 -> Unknown -> could drop.
+ testParquetRowGroupFilterEval(footer, "nonExistCol = 100 and intCol > 50",
ROWS_MATCH.NONE); // since nonExistCol = 100 -> Unknown -> could drop.
+ testParquetRowGroupFilterEval(footer, "intCol > 100 and nonExistCol <
'abc'", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "nonExistCol < 'abc' and intCol >
100", ROWS_MATCH.NONE); // nonExistCol < 'abc' hit NumberException and is
ignored, but intCol >100 will say "drop".
+ testParquetRowGroupFilterEval(footer, "intCol > 50 and nonExistCol <
'abc'", ROWS_MATCH.SOME); // because nonExistCol < 'abc' hit NumberException and
+ // is ignored.
//"nonExistCol" does not exist in the table. "OR" with a filter on exist
column
- testParquetRowGroupFilterEval(footer, "intCol > 100 or nonExistCol = 100",
true); // nonExistCol = 100 -> could drop.
- testParquetRowGroupFilterEval(footer, "nonExistCol = 100 or intCol > 100",
true); // nonExistCol = 100 -> could drop.
- testParquetRowGroupFilterEval(footer, "intCol > 50 or nonExistCol < 100",
false);
- testParquetRowGroupFilterEval(footer, "nonExistCol < 100 or intCol > 50",
false);
+ testParquetRowGroupFilterEval(footer, "intCol > 100 or nonExistCol = 100",
ROWS_MATCH.NONE); // nonExistCol = 100 -> could drop.
+ testParquetRowGroupFilterEval(footer, "nonExistCol = 100 or intCol > 100",
ROWS_MATCH.NONE); // nonExistCol = 100 -> could drop.
+ testParquetRowGroupFilterEval(footer, "intCol > 50 or nonExistCol < 100",
ROWS_MATCH.INAPPLICABLE);
+ testParquetRowGroupFilterEval(footer, "nonExistCol < 100 or intCol > 50",
ROWS_MATCH.INAPPLICABLE);
// cast function on column side (LHS)
- testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 100",
false);
- testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 0", false);
- testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 50",
false);
- testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 101",
true);
- testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = -1", true);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 100",
ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 0",
ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 50",
ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = 101",
ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as bigint) = -1",
ROWS_MATCH.NONE);
// cast function on constant side (RHS)
- testParquetRowGroupFilterEval(footer, "intCol = cast(100 as bigint)",
false);
- testParquetRowGroupFilterEval(footer, "intCol = cast(0 as bigint)", false);
- testParquetRowGroupFilterEval(footer, "intCol = cast(50 as bigint)",
false);
- testParquetRowGroupFilterEval(footer, "intCol = cast(101 as bigint)",
true);
- testParquetRowGroupFilterEval(footer, "intCol = cast(-1 as bigint)", true);
+ testParquetRowGroupFilterEval(footer, "intCol = cast(100 as bigint)",
ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol = cast(0 as bigint)",
ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol = cast(50 as bigint)",
ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "intCol = cast(101 as bigint)",
ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol = cast(-1 as bigint)",
ROWS_MATCH.NONE);
// cast into float4/float8
- testParquetRowGroupFilterEval(footer, "cast(intCol as float4) = cast(101.0
as float4)", true);
- testParquetRowGroupFilterEval(footer, "cast(intCol as float4) = cast(-1.0
as float4)", true);
- testParquetRowGroupFilterEval(footer, "cast(intCol as float4) = cast(1.0
as float4)", false);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as float4) = cast(101.0
as float4)", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as float4) = cast(-1.0
as float4)", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as float4) = cast(1.0
as float4)", ROWS_MATCH.SOME);
- testParquetRowGroupFilterEval(footer, "cast(intCol as float8) = 101.0",
true);
- testParquetRowGroupFilterEval(footer, "cast(intCol as float8) = -1.0",
true);
- testParquetRowGroupFilterEval(footer, "cast(intCol as float8) = 1.0",
false);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as float8) = 101.0",
ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as float8) = -1.0",
ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "cast(intCol as float8) = 1.0",
ROWS_MATCH.SOME);
}
@Test
@@ -176,15 +179,15 @@ public void testIntPredicateAgainstAllNullColWithEval()
throws Exception {
.toFile();
ParquetMetadata footer = getParquetMetaData(file);
- testParquetRowGroupFilterEval(footer, "intCol = 100", true);
- testParquetRowGroupFilterEval(footer, "intCol = 0", true);
- testParquetRowGroupFilterEval(footer, "intCol = -100", true);
+ testParquetRowGroupFilterEval(footer, "intCol = 100", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol = 0", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol = -100", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "intCol > 10", true);
- testParquetRowGroupFilterEval(footer, "intCol >= 10", true);
+ testParquetRowGroupFilterEval(footer, "intCol > 10", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol >= 10", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "intCol < 10", true);
- testParquetRowGroupFilterEval(footer, "intCol <= 10", true);
+ testParquetRowGroupFilterEval(footer, "intCol < 10", ROWS_MATCH.NONE);
+ testParquetRowGroupFilterEval(footer, "intCol <= 10", ROWS_MATCH.NONE);
}
@Test
@@ -216,21 +219,21 @@ public void
testDatePredicateAgainstDrillCTASPost1_8WithEval() throws Exception
}
private void testDatePredicateAgainstDrillCTASHelper(ParquetMetadata footer)
throws Exception{
- testParquetRowGroupFilterEval(footer, "o_orderdate = cast('1992-01-01' as
date)", false);
- testParquetRowGroupFilterEval(footer, "o_orderdate = cast('1991-12-31' as
date)", true);
+ testParquetRowGroupFilterEval(footer, "o_orderdate = cast('1992-01-01' as
date)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_orderdate = cast('1991-12-31' as
date)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_orderdate >= cast('1991-12-31' as
date)", false);
- testParquetRowGroupFilterEval(footer, "o_orderdate >= cast('1992-01-03' as
date)", false);
- testParquetRowGroupFilterEval(footer, "o_orderdate >= cast('1992-01-04' as
date)", true);
+ testParquetRowGroupFilterEval(footer, "o_orderdate >= cast('1991-12-31' as
date)", ROWS_MATCH.ALL);
+ testParquetRowGroupFilterEval(footer, "o_orderdate >= cast('1992-01-03' as
date)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_orderdate >= cast('1992-01-04' as
date)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_orderdate > cast('1992-01-01' as
date)", false);
- testParquetRowGroupFilterEval(footer, "o_orderdate > cast('1992-01-03' as
date)", true);
+ testParquetRowGroupFilterEval(footer, "o_orderdate > cast('1992-01-01' as
date)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_orderdate > cast('1992-01-03' as
date)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_orderdate <= cast('1992-01-01' as
date)", false);
- testParquetRowGroupFilterEval(footer, "o_orderdate <= cast('1991-12-31' as
date)", true);
+ testParquetRowGroupFilterEval(footer, "o_orderdate <= cast('1992-01-01' as
date)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_orderdate <= cast('1991-12-31' as
date)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_orderdate < cast('1992-01-02' as
date)", false);
- testParquetRowGroupFilterEval(footer, "o_orderdate < cast('1992-01-01' as
date)", true);
+ testParquetRowGroupFilterEval(footer, "o_orderdate < cast('1992-01-02' as
date)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_orderdate < cast('1992-01-01' as
date)", ROWS_MATCH.NONE);
}
@Test
@@ -243,24 +246,48 @@ public void testTimeStampPredicateWithEval() throws
Exception {
.toFile();
ParquetMetadata footer = getParquetMetaData(file);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp = cast('1992-01-01
10:20:30' as timestamp)", false);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp = cast('1992-01-01
10:20:29' as timestamp)", true);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp = cast('1992-01-01
10:20:30' as timestamp)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp = cast('1992-01-01
10:20:29' as timestamp)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp >=
cast('1992-01-01 10:20:29' as timestamp)", false);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp >=
cast('1992-01-03 10:20:30' as timestamp)", false);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp >=
cast('1992-01-03 10:20:31' as timestamp)", true);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp >=
cast('1992-01-01 10:20:29' as timestamp)", ROWS_MATCH.ALL);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp >=
cast('1992-01-03 10:20:30' as timestamp)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp >=
cast('1992-01-03 10:20:31' as timestamp)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp > cast('1992-01-03
10:20:29' as timestamp)", false);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp > cast('1992-01-03
10:20:30' as timestamp)", true);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp > cast('1992-01-03
10:20:29' as timestamp)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp > cast('1992-01-03
10:20:30' as timestamp)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp <=
cast('1992-01-01 10:20:30' as timestamp)", false);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp <=
cast('1992-01-01 10:20:29' as timestamp)", true);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp <=
cast('1992-01-01 10:20:30' as timestamp)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp <=
cast('1992-01-01 10:20:29' as timestamp)", ROWS_MATCH.NONE);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp < cast('1992-01-01
10:20:31' as timestamp)", false);
- testParquetRowGroupFilterEval(footer, "o_ordertimestamp < cast('1992-01-01
10:20:30' as timestamp)", true);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp < cast('1992-01-01
10:20:31' as timestamp)", ROWS_MATCH.SOME);
+ testParquetRowGroupFilterEval(footer, "o_ordertimestamp < cast('1992-01-01
10:20:30' as timestamp)", ROWS_MATCH.NONE);
}
+ @Test
+ public void testFilterPruning() throws Exception {
+ // multirowgroup2 is a parquet file with 3 rowgroups inside. One with a=0,
another with a=1 and a=2, and the last with a=3 and a=4;
+ // FilterPushDown should be able to prune the filter from the scan
operator according to the rowgroup statistics.
+ final String sql = "select * from dfs.`parquet/multirowgroup2.parquet`
where ";
+ PlanTestBase.testPlanMatchingPatterns(sql + "a > 1", new
String[]{"numRowGroups=2"}, null); //No filter pruning
+ PlanTestBase.testPlanMatchingPatterns(sql + "a > 2", new
String[]{"numRowGroups=1"}, new String[]{"Filter\\("}); // Filter pruning
+
+ PlanTestBase.testPlanMatchingPatterns(sql + "a < 2", new
String[]{"numRowGroups=2"}, null); // No filter pruning
+ PlanTestBase.testPlanMatchingPatterns(sql + "a < 1", new
String[]{"numRowGroups=1"}, new String[]{"Filter\\("}); // Filter pruning
+
+ PlanTestBase.testPlanMatchingPatterns(sql + "a >= 2", new
String[]{"numRowGroups=2"}, null); // No filter pruning
+ PlanTestBase.testPlanMatchingPatterns(sql + "a >= 1", new
String[]{"numRowGroups=2"}, new String[]{"Filter\\("}); // Filter pruning
+
+ PlanTestBase.testPlanMatchingPatterns(sql + "a <= 1", new
String[]{"numRowGroups=2"}, null); // No filter pruning
+ PlanTestBase.testPlanMatchingPatterns(sql + "a <= 2", new
String[]{"numRowGroups=2"}, new String[]{"Filter\\("}); // Filter pruning
+
+ PlanTestBase.testPlanMatchingPatterns(sql + "a > 0 and a < 2", new
String[]{"numRowGroups=1"}, null); // No filter pruning
+ PlanTestBase.testPlanMatchingPatterns(sql + "a > 0 and a < 3", new
String[]{"numRowGroups=1"}, new String[]{"Filter\\("}); //Filter pruning
+
+ PlanTestBase.testPlanMatchingPatterns(sql + "a < 1 or a > 1", new
String[]{"numRowGroups=3"}, null); // No filter pruning
+ PlanTestBase.testPlanMatchingPatterns(sql + "a < 1 or a > 2", new
String[]{"numRowGroups=2"}, new String[]{"Filter\\("}); //Filter pruning
+ }
+
@Test
// Test against parquet files from Drill CTAS post 1.8.0 release.
public void testDatePredicateAgaistDrillCTASPost1_8() throws Exception {
@@ -467,13 +494,13 @@ private void testParquetFilterPD(final String query, int
expectedRowCount, int e
testPlanMatchingPatterns(query, new String[]{numFilesPattern,
usedMetaPattern});
}
- private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final String exprStr, boolean canDropExpected) throws Exception{
+ private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final String exprStr, ROWS_MATCH canDropExpected) throws Exception{
final LogicalExpression filterExpr = parseExpr(exprStr);
testParquetRowGroupFilterEval(footer, 0, filterExpr, canDropExpected);
}
- private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final int rowGroupIndex, final LogicalExpression filterExpr, boolean
canDropExpected) {
- boolean canDrop = ParquetRGFilterEvaluator.evalFilter(filterExpr, footer,
rowGroupIndex, fragContext.getOptions(), fragContext);
+ private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final int rowGroupIndex, final LogicalExpression filterExpr, ROWS_MATCH
canDropExpected) {
+ ROWS_MATCH canDrop = ParquetRGFilterEvaluator.evalFilter(filterExpr,
footer, rowGroupIndex, fragContext.getOptions(), fragContext);
Assert.assertEquals(canDropExpected, canDrop);
}
diff --git a/exec/java-exec/src/test/resources/parquet/multirowgroup2.parquet
b/exec/java-exec/src/test/resources/parquet/multirowgroup2.parquet
new file mode 100644
index 0000000000..513980277c
Binary files /dev/null and
b/exec/java-exec/src/test/resources/parquet/multirowgroup2.parquet differ
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Filter pruning for multi rowgroup parquet file
> ----------------------------------------------
>
> Key: DRILL-5796
> URL: https://issues.apache.org/jira/browse/DRILL-5796
> Project: Apache Drill
> Issue Type: Improvement
> Components: Storage - Parquet
> Reporter: Damien Profeta
> Assignee: Jean-Blas IMBERT
> Priority: Major
> Fix For: 1.14.0
>
>
> Today, filter pruning use the file name as the partitioning key. This means
> you can remove a partition only if the whole file is for the same partition.
> With parquet, you can prune the filter if the rowgroup make a partition of
> your dataset as the unit of work if the rowgroup not the file.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
