This is an automated email from the ASF dual-hosted git repository. snuyanzin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push: new 4165bac27bd [FLINK-35098][ORC] Fix incorrect results with literal first expressions 4165bac27bd is described below commit 4165bac27bda4457e5940a994d923242d4a271dc Author: Andrey Gaskov <31715230+empath...@users.noreply.github.com> AuthorDate: Tue May 14 14:09:51 2024 +0700 [FLINK-35098][ORC] Fix incorrect results with literal first expressions --- .../main/java/org/apache/flink/orc/OrcFilters.java | 8 +-- .../apache/flink/orc/OrcFileSystemFilterTest.java | 58 +++++++++++++++++++--- .../org/apache/flink/orc/OrcFileSystemITCase.java | 11 ++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcFilters.java b/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcFilters.java index 4393356fc30..46eb9af0927 100644 --- a/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcFilters.java +++ b/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcFilters.java @@ -83,28 +83,28 @@ public class OrcFilters { convertBinary( call, OrcFilters::convertGreaterThan, - OrcFilters::convertLessThanEquals)) + OrcFilters::convertLessThan)) .put( BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL, call -> convertBinary( call, OrcFilters::convertGreaterThanEquals, - OrcFilters::convertLessThan)) + OrcFilters::convertLessThanEquals)) .put( BuiltInFunctionDefinitions.LESS_THAN, call -> convertBinary( call, OrcFilters::convertLessThan, - OrcFilters::convertGreaterThanEquals)) + OrcFilters::convertGreaterThan)) .put( BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL, call -> convertBinary( call, OrcFilters::convertLessThanEquals, - OrcFilters::convertGreaterThan)) + OrcFilters::convertGreaterThanEquals)) .build(); private static boolean isRef(Expression expression) { diff --git a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemFilterTest.java b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemFilterTest.java index cdee400723d..f5d9ed33423 100644 --- a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemFilterTest.java +++ b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemFilterTest.java @@ -25,13 +25,13 @@ import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.expressions.ValueLiteralExpression; import org.apache.flink.table.functions.BuiltInFunctionDefinitions; -import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf.Type.LONG; import static org.assertj.core.api.Assertions.assertThat; /** Unit Tests for {@link OrcFileFormatFactory}. */ @@ -39,7 +39,7 @@ class OrcFileSystemFilterTest { @Test @SuppressWarnings("unchecked") - public void testApplyPredicate() { + void testApplyPredicate() { List<ResolvedExpression> args = new ArrayList<>(); // equal @@ -53,8 +53,7 @@ class OrcFileSystemFilterTest { CallExpression.permanent( BuiltInFunctionDefinitions.EQUALS, args, DataTypes.BOOLEAN()); OrcFilters.Predicate predicate1 = OrcFilters.toOrcPredicate(equalExpression); - OrcFilters.Predicate predicate2 = - new OrcFilters.Equals("long1", PredicateLeaf.Type.LONG, 10); + OrcFilters.Predicate predicate2 = new OrcFilters.Equals("long1", LONG, 10); assertThat(predicate1).hasToString(predicate2.toString()); // greater than @@ -63,8 +62,7 @@ class OrcFileSystemFilterTest { BuiltInFunctionDefinitions.GREATER_THAN, args, DataTypes.BOOLEAN()); OrcFilters.Predicate predicate3 = OrcFilters.toOrcPredicate(greaterExpression); OrcFilters.Predicate predicate4 = - new OrcFilters.Not( - new OrcFilters.LessThanEquals("long1", PredicateLeaf.Type.LONG, 10)); + new OrcFilters.Not(new OrcFilters.LessThanEquals("long1", LONG, 10)); assertThat(predicate3).hasToString(predicate4.toString()); // less than @@ -72,8 +70,7 @@ class OrcFileSystemFilterTest { CallExpression.permanent( BuiltInFunctionDefinitions.LESS_THAN, args, DataTypes.BOOLEAN()); OrcFilters.Predicate predicate5 = OrcFilters.toOrcPredicate(lessExpression); - OrcFilters.Predicate predicate6 = - new OrcFilters.LessThan("long1", PredicateLeaf.Type.LONG, 10); + OrcFilters.Predicate predicate6 = new OrcFilters.LessThan("long1", LONG, 10); assertThat(predicate5).hasToString(predicate6.toString()); // and @@ -86,4 +83,49 @@ class OrcFileSystemFilterTest { OrcFilters.Predicate predicate8 = new OrcFilters.And(predicate4, predicate6); assertThat(predicate7).hasToString(predicate8.toString()); } + + @Test + @SuppressWarnings("unchecked") + void testApplyPredicateReverse() { + List<ResolvedExpression> args = new ArrayList<>(); + + FieldReferenceExpression fieldReferenceExpression = + new FieldReferenceExpression("x", DataTypes.BIGINT(), 0, 0); + ValueLiteralExpression valueLiteralExpression = new ValueLiteralExpression(10); + args.add(valueLiteralExpression); + args.add(fieldReferenceExpression); + + CallExpression expression; + OrcFilters.Predicate predicate; + + // assert that 10 >= x expression is converted to x <= 10 ORC predicate + expression = + CallExpression.permanent( + BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL, + args, + DataTypes.BOOLEAN()); + predicate = new OrcFilters.LessThanEquals("x", LONG, 10); + assertThat(OrcFilters.toOrcPredicate(expression)).hasToString(predicate.toString()); + + // assert that 10 > x expression is converted to x < 10 ORC predicate + expression = + CallExpression.permanent( + BuiltInFunctionDefinitions.GREATER_THAN, args, DataTypes.BOOLEAN()); + predicate = new OrcFilters.LessThan("x", LONG, 10); + assertThat(OrcFilters.toOrcPredicate(expression)).hasToString(predicate.toString()); + + // assert that 10 <= x expression is converted to NOT(x < 10) ORC predicate + expression = + CallExpression.permanent( + BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL, args, DataTypes.BOOLEAN()); + predicate = new OrcFilters.Not(new OrcFilters.LessThan("x", LONG, 10)); + assertThat(OrcFilters.toOrcPredicate(expression)).hasToString(predicate.toString()); + + // assert that 10 < x expression is converted to NOT(x <= 10) ORC predicate + expression = + CallExpression.permanent( + BuiltInFunctionDefinitions.LESS_THAN, args, DataTypes.BOOLEAN()); + predicate = new OrcFilters.Not(new OrcFilters.LessThanEquals("x", LONG, 10)); + assertThat(OrcFilters.toOrcPredicate(expression)).hasToString(predicate.toString()); + } } diff --git a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemITCase.java b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemITCase.java index ec36bdc1d16..2ba3bafb327 100644 --- a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemITCase.java +++ b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcFileSystemITCase.java @@ -208,6 +208,17 @@ public class OrcFileSystemITCase extends BatchFileSystemITCaseBase { Collections.singletonList(Row.of("x10", "10"))); } + @TestTemplate + void testOrcFilterPushDownLiteralFirst() throws ExecutionException, InterruptedException { + super.tableEnv() + .executeSql("insert into orcLimitTable values('a', 10, 10), ('b', 11, 11)") + .await(); + check("select y from orcLimitTable where 10 >= y", Collections.singletonList(Row.of(10))); + check("select y from orcLimitTable where 11 <= y", Collections.singletonList(Row.of(11))); + check("select y from orcLimitTable where 11 > y", Collections.singletonList(Row.of(10))); + check("select y from orcLimitTable where 10 < y", Collections.singletonList(Row.of(11))); + } + @TestTemplate void testNestedTypes() throws Exception { String path = initNestedTypesFile(initNestedTypesData());