This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 671786d58e [hive] Skip predicate pushdown for literals not
representable in the column type (#10403)
671786d58e is described below
commit 671786d58ec61615b10013e69652cbddc4ed8538
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Oct 9 23:01:36 2026 +0900
[hive] Skip predicate pushdown for literals not representable in the column
type (#10403)
---
.../hive/SearchArgumentToPredicateConverter.java | 31 +++++-
.../org/apache/paimon/hive/HiveReadITCaseBase.java | 48 ++++++++++
.../SearchArgumentToPredicateConverterTest.java | 104 +++++++++++++++++++++
3 files changed, 182 insertions(+), 1 deletion(-)
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
index 202cceae29..b33d94b882 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
@@ -18,6 +18,7 @@
package org.apache.paimon.hive;
+import org.apache.paimon.data.Decimal;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.DataType;
@@ -33,6 +34,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
@@ -182,6 +184,33 @@ public class SearchArgumentToPredicateConverter {
if (o instanceof HiveDecimalWritable) {
o = ((HiveDecimalWritable) o).getHiveDecimal().bigDecimalValue();
}
- return convertJavaObject(literalType, o);
+ Object literal = convertJavaObject(literalType, o);
+ // Hive does not cast literals to the column type. A literal that
cannot be represented
+ // exactly in the column type (e.g. 1000 for DECIMAL(5, 2), 1.005 for
DECIMAL(5, 2) or 200
+ // for TINYINT) would be altered here and prune data files wrongly, so
leave this
+ // conjunct to Hive's residual filter instead.
+ if (o != null && !isExact(literalType, o, literal)) {
+ throw new UnsupportedOperationException(
+ "Literal "
+ + o
+ + " cannot be represented exactly in column type "
+ + literalType
+ + ".");
+ }
+ return literal;
+ }
+
+ private static boolean isExact(DataType literalType, Object original,
Object literal) {
+ switch (literalType.getTypeRoot()) {
+ case DECIMAL:
+ return literal != null
+ && ((Decimal)
literal).toBigDecimal().compareTo((BigDecimal) original) == 0;
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ return ((Number) literal).longValue() == ((Number)
original).longValue();
+ default:
+ return true;
+ }
}
}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java
index 2a15ffae34..6c225bafe3 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java
@@ -706,6 +706,54 @@ public abstract class HiveReadITCaseBase extends
HiveTestBase {
.containsExactly("1", "2", "3", "4", "5", "6");
}
+ @Test
+ public void
testPredicatePushDownWithLiteralsNotRepresentableInColumnType() throws
Exception {
+ Options conf = getBasicConf();
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
+ Table table =
+ FileStoreTestUtils.createFileStoreTable(
+ conf,
+ RowType.of(
+ new DataType[] {
+ DataTypes.DECIMAL(5, 2),
DataTypes.TINYINT(), DataTypes.INT()
+ },
+ new String[] {"c", "t", "i"}),
+ Collections.emptyList(),
+ Collections.emptyList());
+
+ // one data file per row, so that file pruning by statistics is
observable
+ StreamWriteBuilder streamWriteBuilder = table.newStreamWriteBuilder();
+ StreamTableWrite write = streamWriteBuilder.newWrite();
+ StreamTableCommit commit = streamWriteBuilder.newCommit();
+ write.write(GenericRow.of(decimal("1.00"), (byte) 5, 5));
+ commit.commit(0, write.prepareCommit(true, 0));
+ write.write(GenericRow.of(decimal("1.01"), (byte) 100, 100));
+ commit.commit(1, write.prepareCommit(true, 1));
+ write.write(GenericRow.of(decimal("999.99"), (byte) 120, 2000000000));
+ commit.commit(2, write.prepareCommit(true, 2));
+ write.close();
+ commit.close();
+ createExternalTable();
+
+ String select = "SELECT t FROM " + externalTable + " WHERE ";
+ assertThat(hiveShell.executeQuery(select + "c < 1000"))
+ .containsExactlyInAnyOrder("5", "100", "120");
+ assertThat(hiveShell.executeQuery(select + "c BETWEEN 0 AND 1000"))
+ .containsExactlyInAnyOrder("5", "100", "120");
+ assertThat(hiveShell.executeQuery(select + "c > 1.005"))
+ .containsExactlyInAnyOrder("100", "120");
+ assertThat(hiveShell.executeQuery(select + "c <
1.004")).containsExactly("5");
+ assertThat(hiveShell.executeQuery(select + "t < 200"))
+ .containsExactlyInAnyOrder("5", "100", "120");
+ assertThat(hiveShell.executeQuery(select + "i < 3000000000"))
+ .containsExactlyInAnyOrder("5", "100", "120");
+ assertThat(hiveShell.executeQuery(select + "c <
1.01")).containsExactly("5");
+ }
+
+ private static Decimal decimal(String value) {
+ return Decimal.fromBigDecimal(new BigDecimal(value), 5, 2);
+ }
+
@Test
public void testDateAndTimestamp() throws Exception {
ThreadLocalRandom random = ThreadLocalRandom.current();
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/SearchArgumentToPredicateConverterTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/SearchArgumentToPredicateConverterTest.java
index d9a6b7c8f1..ad97ecdab7 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/SearchArgumentToPredicateConverterTest.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/SearchArgumentToPredicateConverterTest.java
@@ -421,6 +421,110 @@ public class SearchArgumentToPredicateConverterTest {
assertExpected(sarg, expected);
}
+ private static final List<String> NARROW_COLUMN_NAMES =
+ Arrays.asList("f_decimal", "f_tinyint", "f_int");
+ private static final List<DataType> NARROW_COLUMN_TYPES =
+ Arrays.asList(DataTypes.DECIMAL(5, 2), DataTypes.TINYINT(),
DataTypes.INT());
+ private static final PredicateBuilder NARROW_BUILDER =
+ new PredicateBuilder(
+ RowType.of(
+ NARROW_COLUMN_TYPES.toArray(new DataType[0]),
+ NARROW_COLUMN_NAMES.toArray(new String[0])));
+
+ @Test
+ public void testDecimalLiteralExceedingPrecision() {
+ SearchArgument sarg =
+ SearchArgumentFactory.newBuilder()
+ .lessThan("f_decimal", PredicateLeaf.Type.DECIMAL,
hiveDecimal("1000"))
+ .build();
+ assertNarrowExpected(sarg, null);
+
+ sarg =
+ SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .between(
+ "f_decimal",
+ PredicateLeaf.Type.DECIMAL,
+ hiveDecimal("0"),
+ hiveDecimal("1000"))
+ .in(
+ "f_decimal",
+ PredicateLeaf.Type.DECIMAL,
+ hiveDecimal("1.5"),
+ hiveDecimal("1000"))
+ .lessThan("f_tinyint", PredicateLeaf.Type.LONG, 10L)
+ .end()
+ .build();
+ assertNarrowExpected(sarg, NARROW_BUILDER.lessThan(1, (byte) 10));
+ }
+
+ @Test
+ public void testDecimalLiteralExceedingScale() {
+ SearchArgument sarg =
+ SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .startNot()
+ .lessThanEquals(
+ "f_decimal", PredicateLeaf.Type.DECIMAL,
hiveDecimal("1.005"))
+ .end()
+ .lessThan("f_int", PredicateLeaf.Type.LONG, 100L)
+ .end()
+ .build();
+ assertNarrowExpected(sarg, NARROW_BUILDER.lessThan(2, 100));
+ }
+
+ @Test
+ public void testIntegerLiteralOutOfRange() {
+ SearchArgument sarg =
+ SearchArgumentFactory.newBuilder()
+ .lessThan("f_tinyint", PredicateLeaf.Type.LONG, 200L)
+ .build();
+ assertNarrowExpected(sarg, null);
+
+ sarg =
+ SearchArgumentFactory.newBuilder()
+ .lessThan("f_int", PredicateLeaf.Type.LONG,
3000000000L)
+ .build();
+ assertNarrowExpected(sarg, null);
+ }
+
+ @Test
+ public void testRepresentableNarrowLiterals() {
+ SearchArgument sarg =
+ SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .lessThan("f_decimal", PredicateLeaf.Type.DECIMAL,
hiveDecimal("1.50"))
+ .lessThanEquals(
+ "f_decimal", PredicateLeaf.Type.DECIMAL,
hiveDecimal("999.99"))
+ .lessThanEquals("f_int", PredicateLeaf.Type.LONG,
(long) Integer.MAX_VALUE)
+ .in("f_tinyint", PredicateLeaf.Type.LONG, 0L, null,
127L)
+ .end()
+ .build();
+ Predicate expected =
+ PredicateBuilder.and(
+ NARROW_BUILDER.lessThan(0, decimal("1.50")),
+ NARROW_BUILDER.lessOrEqual(0, decimal("999.99")),
+ NARROW_BUILDER.lessOrEqual(2, Integer.MAX_VALUE),
+ NARROW_BUILDER.in(1, Arrays.asList((byte) 0, null,
(byte) 127)));
+ assertNarrowExpected(sarg, expected);
+ }
+
+ private static HiveDecimalWritable hiveDecimal(String value) {
+ return new HiveDecimalWritable(HiveDecimal.create(value));
+ }
+
+ private static Decimal decimal(String value) {
+ return Decimal.fromBigDecimal(new BigDecimal(value), 5, 2);
+ }
+
+ private void assertNarrowExpected(SearchArgument sarg, Predicate expected)
{
+ SearchArgumentToPredicateConverter converter =
+ new SearchArgumentToPredicateConverter(
+ sarg, NARROW_COLUMN_NAMES, NARROW_COLUMN_TYPES, null);
+ Predicate actual = converter.convert().orElse(null);
+ assertThat(actual).isEqualTo(expected);
+ }
+
private void assertExpected(SearchArgument sarg, Predicate expected) {
SearchArgumentToPredicateConverter converter =
new SearchArgumentToPredicateConverter(sarg, COLUMN_NAMES,
COLUMN_TYPES, null);
