This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5d22d708252bd5a84fe2f277f2b8b9a6abc0d9ce
Author: Arina Ielchiieva <arina.yelchiy...@gmail.com>
AuthorDate: Thu May 17 18:24:35 2018 +0300
DRILL-6413: Update ParquetFilterBuilder.visitBooleanOperator to handle
simplified boolean expression
closes #1269
---
.../store/parquet/AbstractParquetGroupScan.java | 3 +--
.../exec/store/parquet/ParquetFilterBuilder.java | 19 +++++++++++++++----
.../store/parquet/TestParquetFilterPushDown.java | 22 +++++++++++-----------
3 files changed, 27 insertions(+), 17 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
index 1f8c535..33472bb 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
@@ -255,8 +255,7 @@ public abstract class AbstractParquetGroupScan extends
AbstractFileGroupScan {
logger.debug("materializedFilter : {}",
ExpressionStringBuilder.toString(materializedFilter));
Set<LogicalExpression> constantBoundaries =
ConstantExpressionIdentifier.getConstantExpressionSet(materializedFilter);
- filterPredicate = (ParquetFilterPredicate)
ParquetFilterBuilder.buildParquetFilterPredicate(
- materializedFilter, constantBoundaries, udfUtilities);
+ filterPredicate =
ParquetFilterBuilder.buildParquetFilterPredicate(materializedFilter,
constantBoundaries, udfUtilities);
if (filterPredicate == null) {
return null;
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
index e55425e..a8e101d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
@@ -42,6 +42,7 @@ import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.expr.holders.VarDecimalHolder;
import org.apache.drill.exec.expr.stat.ParquetBooleanPredicates;
import org.apache.drill.exec.expr.stat.ParquetComparisonPredicates;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate;
import org.apache.drill.exec.expr.stat.ParquetIsPredicates;
import org.apache.drill.exec.ops.UdfUtilities;
import org.apache.drill.exec.util.DecimalUtility;
@@ -54,7 +55,7 @@ import java.util.Set;
/**
* A visitor which visits a materialized logical expression, and build
ParquetFilterPredicate
- * If a visitXXX method returns null, that means the corresponding filter
branch is not qualified for pushdown.
+ * If a visitXXX method returns null, that means the corresponding filter
branch is not qualified for push down.
*/
public class ParquetFilterBuilder extends
AbstractExprVisitor<LogicalExpression, Set<LogicalExpression>,
RuntimeException> {
static final Logger logger =
LoggerFactory.getLogger(ParquetFilterBuilder.class);
@@ -66,12 +67,18 @@ public class ParquetFilterBuilder extends
AbstractExprVisitor<LogicalExpression,
* @param constantBoundaries set of constant expressions
* @param udfUtilities udf utilities
*
- * @return logical expression
+ * @return parquet filter predicate
*/
- public static LogicalExpression
buildParquetFilterPredicate(LogicalExpression expr, final
Set<LogicalExpression> constantBoundaries, UdfUtilities udfUtilities) {
- return expr.accept(new ParquetFilterBuilder(udfUtilities),
constantBoundaries);
+ public static ParquetFilterPredicate
buildParquetFilterPredicate(LogicalExpression expr, final
Set<LogicalExpression> constantBoundaries, UdfUtilities udfUtilities) {
+ LogicalExpression logicalExpression = expr.accept(new
ParquetFilterBuilder(udfUtilities), constantBoundaries);
+ if (logicalExpression instanceof ParquetFilterPredicate) {
+ return (ParquetFilterPredicate) logicalExpression;
+ }
+ logger.debug("Logical expression {} was not qualified for filter push
down", logicalExpression);
+ return null;
}
+
private ParquetFilterBuilder(UdfUtilities udfUtilities) {
this.udfUtilities = udfUtilities;
}
@@ -150,6 +157,10 @@ public class ParquetFilterBuilder extends
AbstractExprVisitor<LogicalExpression,
return null;
}
} else {
+ if (childPredicate instanceof TypedFieldExpr) {
+ // Calcite simplifies `= true` expression to field name, wrap it
with is true predicate
+ childPredicate = new
ParquetIsPredicates.IsTruePredicate(childPredicate);
+ }
childPredicates.add(childPredicate);
}
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index 83a4e8e..3bbd397 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -26,6 +26,7 @@ import org.apache.drill.exec.proto.BitControl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.junit.Assert;
@@ -424,6 +425,9 @@ public class TestParquetFilterPushDown extends PlanTestBase
{
final String queryNotEqualFalse = "select col_bln from
dfs.`parquetFilterPush/blnTbl` where not col_bln = false";
testParquetFilterPD(queryNotEqualFalse, 4, 2, false);
+
+ final String queryEqualTrueWithAnd = "select col_bln from
dfs.`parquetFilterPush/blnTbl` where col_bln = true and unk_col = 'a'";
+ testParquetFilterPD(queryEqualTrueWithAnd, 0, 2, false);
}
@Test // DRILL-5359
@@ -445,10 +449,9 @@ public class TestParquetFilterPushDown extends
PlanTestBase {
public void testMultiRowGroup() throws Exception {
// multirowgroup is a parquet file with 2 rowgroups inside. One with a = 1
and the other with a = 2;
// FilterPushDown should be able to remove the rowgroup with a = 1 from
the scan operator.
- final String sql = String.format("select * from
dfs.`parquet/multirowgroup.parquet` where a > 1");
+ final String sql = "select * from dfs.`parquet/multirowgroup.parquet`
where a > 1";
final String[] expectedPlan = {"numRowGroups=1"};
- final String[] excludedPlan = {};
- PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan);
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -461,23 +464,20 @@ public class TestParquetFilterPushDown extends
PlanTestBase {
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=" + usedMetadataFile;
- testPlanMatchingPatterns(query, new String[]{numFilesPattern,
usedMetaPattern}, new String[] {});
+ testPlanMatchingPatterns(query, new String[]{numFilesPattern,
usedMetaPattern});
}
- private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final String exprStr,
- boolean canDropExpected) throws Exception{
+ private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final String exprStr, boolean canDropExpected) throws Exception{
final LogicalExpression filterExpr = parseExpr(exprStr);
testParquetRowGroupFilterEval(footer, 0, filterExpr, canDropExpected);
}
- private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final int rowGroupIndex,
- final LogicalExpression filterExpr, boolean canDropExpected) throws
Exception {
- boolean canDrop = ParquetRGFilterEvaluator.evalFilter(filterExpr, footer,
rowGroupIndex,
- fragContext.getOptions(), fragContext);
+ private void testParquetRowGroupFilterEval(final ParquetMetadata footer,
final int rowGroupIndex, final LogicalExpression filterExpr, boolean
canDropExpected) {
+ boolean canDrop = ParquetRGFilterEvaluator.evalFilter(filterExpr, footer,
rowGroupIndex, fragContext.getOptions(), fragContext);
Assert.assertEquals(canDropExpected, canDrop);
}
private ParquetMetadata getParquetMetaData(File file) throws IOException{
- return ParquetFileReader.readFooter(new Configuration(fs.getConf()), new
Path(file.toURI()));
+ return ParquetFileReader.readFooter(new Configuration(fs.getConf()), new
Path(file.toURI()), ParquetMetadataConverter.NO_FILTER);
}
}
--
To stop receiving notification emails like this one, please contact
ar...@apache.org.
