rdblue commented on code in PR #17413:
URL: https://github.com/apache/iceberg/pull/17413#discussion_r3779497865
##########
api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java:
##########
@@ -393,789 +546,525 @@ public void testZeroRecordFile() {
};
for (Expression expr : exprs) {
- boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA,
expr).eval(empty);
+ boolean shouldRead = shouldRead(SCHEMA, expr, emptyFile());
assertThat(shouldRead).as("Should never read 0-record file: " +
expr).isFalse();
}
}
@Test
public void testNot() {
// this test case must use a real predicate, not alwaysTrue(), or binding
will simplify it out
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(lessThan("id", INT_MIN_VALUE
- 25))).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, not(lessThan("id", INT_MIN_VALUE -
25)), file());
assertThat(shouldRead).as("Should read: not(false)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(greaterThan("id",
INT_MIN_VALUE - 25)))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(greaterThan("id", INT_MIN_VALUE -
25)), file());
assertThat(shouldRead).as("Should skip: not(true)").isFalse();
}
@Test
public void testAnd() {
// this test case must use a real predicate, not alwaysTrue(), or binding
will simplify it out
boolean shouldRead =
- new InclusiveMetricsEvaluator(
- SCHEMA,
- and(
- lessThan("id", INT_MIN_VALUE - 25),
- greaterThanOrEqual("id", INT_MIN_VALUE - 30)))
- .eval(FILE);
+ shouldRead(
+ SCHEMA,
+ and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id",
INT_MIN_VALUE - 30)),
+ file());
assertThat(shouldRead).as("Should skip: and(false, true)").isFalse();
shouldRead =
- new InclusiveMetricsEvaluator(
- SCHEMA,
- and(
- lessThan("id", INT_MIN_VALUE - 25),
- greaterThanOrEqual("id", INT_MAX_VALUE + 1)))
- .eval(FILE);
+ shouldRead(
+ SCHEMA,
+ and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id",
INT_MAX_VALUE + 1)),
+ file());
assertThat(shouldRead).as("Should skip: and(false, false)").isFalse();
shouldRead =
- new InclusiveMetricsEvaluator(
- SCHEMA,
- and(greaterThan("id", INT_MIN_VALUE - 25),
lessThanOrEqual("id", INT_MIN_VALUE)))
- .eval(FILE);
+ shouldRead(
+ SCHEMA,
+ and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id",
INT_MIN_VALUE)),
+ file());
assertThat(shouldRead).as("Should read: and(true, true)").isTrue();
}
@Test
public void testOr() {
// this test case must use a real predicate, not alwaysTrue(), or binding
will simplify it out
boolean shouldRead =
- new InclusiveMetricsEvaluator(
- SCHEMA,
- or(lessThan("id", INT_MIN_VALUE - 25),
greaterThanOrEqual("id", INT_MAX_VALUE + 1)))
- .eval(FILE);
+ shouldRead(
+ SCHEMA,
+ or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id",
INT_MAX_VALUE + 1)),
+ file());
assertThat(shouldRead).as("Should skip: or(false, false)").isFalse();
shouldRead =
- new InclusiveMetricsEvaluator(
- SCHEMA,
- or(
- lessThan("id", INT_MIN_VALUE - 25),
- greaterThanOrEqual("id", INT_MAX_VALUE - 19)))
- .eval(FILE);
+ shouldRead(
+ SCHEMA,
+ or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id",
INT_MAX_VALUE - 19)),
+ file());
assertThat(shouldRead).as("Should read: or(false, true)").isTrue();
}
@Test
public void testIntegerLt() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE -
25)).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE -
25), file());
assertThat(shouldRead).as("Should not read: id range below lower bound (5
< 30)").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id",
INT_MIN_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file());
assertThat(shouldRead)
.as("Should not read: id range below lower bound (30 is not < 30)")
.isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE +
1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE + 1), file());
assertThat(shouldRead).as("Should read: one possible id").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id",
INT_MAX_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MAX_VALUE), file());
assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
public void testIntegerLtEq() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id",
INT_MIN_VALUE - 25)).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id",
INT_MIN_VALUE - 25), file());
assertThat(shouldRead).as("Should not read: id range below lower bound (5
< 30)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id",
INT_MIN_VALUE - 1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1),
file());
assertThat(shouldRead).as("Should not read: id range below lower bound (29
< 30)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id",
INT_MIN_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE),
file());
assertThat(shouldRead).as("Should read: one possible id").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id",
INT_MAX_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE),
file());
assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
public void testIntegerGt() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE
+ 6)).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE +
6), file());
assertThat(shouldRead).as("Should not read: id range above upper bound (85
< 79)").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id",
INT_MAX_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE), file());
assertThat(shouldRead)
.as("Should not read: id range above upper bound (79 is not > 79)")
.isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE
- 1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1),
file());
assertThat(shouldRead).as("Should read: one possible id").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE
- 4)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4),
file());
assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
public void testIntegerGtEq() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id",
INT_MAX_VALUE + 6))
- .eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id",
INT_MAX_VALUE + 6), file());
assertThat(shouldRead).as("Should not read: id range above upper bound (85
< 79)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id",
INT_MAX_VALUE + 1))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE +
1), file());
assertThat(shouldRead).as("Should not read: id range above upper bound (80
> 79)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id",
INT_MAX_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE),
file());
assertThat(shouldRead).as("Should read: one possible id").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id",
INT_MAX_VALUE - 4))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE -
4), file());
assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
public void testIntegerEq() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE -
25)).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 25),
file());
assertThat(shouldRead).as("Should not read: id below lower
bound").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id",
INT_MIN_VALUE - 1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 1), file());
assertThat(shouldRead).as("Should not read: id below lower
bound").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id",
INT_MIN_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE), file());
assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id",
INT_MAX_VALUE - 4)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE - 4), file());
assertThat(shouldRead).as("Should read: id between lower and upper
bounds").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id",
INT_MAX_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE), file());
assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id",
INT_MAX_VALUE + 1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 1), file());
assertThat(shouldRead).as("Should not read: id above upper
bound").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id",
INT_MAX_VALUE + 6)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 6), file());
assertThat(shouldRead).as("Should not read: id above upper
bound").isFalse();
}
@Test
public void testIntegerNotEq() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE -
25)).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE -
25), file());
assertThat(shouldRead).as("Should read: id below lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE -
1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 1), file());
assertThat(shouldRead).as("Should read: id below lower bound").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id",
INT_MIN_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE), file());
assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE -
4)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE - 4), file());
assertThat(shouldRead).as("Should read: id between lower and upper
bounds").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id",
INT_MAX_VALUE)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE), file());
assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE +
1)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 1), file());
assertThat(shouldRead).as("Should read: id above upper bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE +
6)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 6), file());
assertThat(shouldRead).as("Should read: id above upper bound").isTrue();
}
@Test
public void testIntegerNotEqRewritten() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE -
25))).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE -
25)), file());
assertThat(shouldRead).as("Should read: id below lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE -
1))).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 1)),
file());
assertThat(shouldRead).as("Should read: id below lower bound").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id",
INT_MIN_VALUE))).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE)), file());
assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE -
4))).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE - 4)),
file());
assertThat(shouldRead).as("Should read: id between lower and upper
bounds").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id",
INT_MAX_VALUE))).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE)), file());
assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE +
1))).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 1)),
file());
assertThat(shouldRead).as("Should read: id above upper bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE +
6))).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 6)),
file());
assertThat(shouldRead).as("Should read: id above upper bound").isTrue();
}
@Test
public void testCaseInsensitiveIntegerNotEqRewritten() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE -
25)), false)
- .eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE -
25)), false, file());
assertThat(shouldRead).as("Should read: id below lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE -
1)), false)
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)),
false, file());
assertThat(shouldRead).as("Should read: id below lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE)),
false).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false,
file());
assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE -
4)), false)
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)),
false, file());
assertThat(shouldRead).as("Should read: id between lower and upper
bounds").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE)),
false).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false,
file());
assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE +
1)), false)
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)),
false, file());
assertThat(shouldRead).as("Should read: id above upper bound").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE +
6)), false)
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)),
false, file());
assertThat(shouldRead).as("Should read: id above upper bound").isTrue();
}
@Test
public void testCaseSensitiveIntegerNotEqRewritten() {
- assertThatThrownBy(
- () -> new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", 5)),
true).eval(FILE))
+ assertThatThrownBy(() -> shouldRead(SCHEMA, not(equal("ID", 5)), true,
file()))
.isInstanceOf(ValidationException.class)
.hasMessageContaining("Cannot find field 'ID'");
}
@Test
public void testStringStartsWith() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"),
true).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true,
file());
assertThat(shouldRead).as("Should read: no stats").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aa"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "aa"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aaa"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "aaa"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1s"),
true).eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "1s"), true,
file3());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required",
"1str1x"), true).eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "1str1x"), true,
file3());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "ff"),
true).eval(FILE_4);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "ff"), true,
file4());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aB"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "aB"), true,
file2());
assertThat(shouldRead).as("Should not read: range doesn't
match").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "dWX"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "dWX"), true,
file2());
assertThat(shouldRead).as("Should not read: range doesn't
match").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "5"),
true).eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "5"), true,
file3());
assertThat(shouldRead).as("Should not read: range doesn't
match").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required",
"3str3x"), true).eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", "3str3x"), true,
file3());
assertThat(shouldRead).as("Should not read: range doesn't
match").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("some_empty",
"房东整租霍"), true).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, startsWith("some_empty", "房东整租霍"), true,
file());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", ""),
true).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", ""), true, file());
assertThat(shouldRead).as("Should not read: range doesn't
match").isFalse();
String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"),
4).value().toString();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, startsWith("required",
aboveMax), true).eval(FILE_4);
+ shouldRead = shouldRead(SCHEMA, startsWith("required", aboveMax), true,
file4());
assertThat(shouldRead).as("Should not read: range doesn't
match").isFalse();
}
@Test
public void testStringNotStartsWith() {
- boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"),
true).eval(FILE);
+ boolean shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"),
true, file());
assertThat(shouldRead).as("Should read: no stats").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aa"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aa"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
"aaa"), true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aaa"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1s"),
true).eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1s"), true,
file3());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
"1str1x"), true)
- .eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1str1x"), true,
file3());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "ff"),
true).eval(FILE_4);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "ff"), true,
file4());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aB"),
true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aB"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
"dWX"), true).eval(FILE_2);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "dWX"), true,
file2());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "5"),
true).eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "5"), true,
file3());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
"3str3x"), true)
- .eval(FILE_3);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "3str3x"), true,
file3());
assertThat(shouldRead).as("Should read: range matches").isTrue();
String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"),
4).value().toString();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
aboveMax), true)
- .eval(FILE_4);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", aboveMax), true,
file4());
assertThat(shouldRead).as("Should read: range matches").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
"abc"), true).eval(FILE_5);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abc"), true,
file5());
assertThat(shouldRead).as("Should not read: all strings start with
prefix").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required",
"abcd"), true).eval(FILE_5);
+ shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abcd"), true,
file5());
assertThat(shouldRead).as("Should not read: lower shorter than prefix,
cannot match").isTrue();
}
@Test
public void testIntegerIn() {
boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 25,
INT_MIN_VALUE - 24))
- .eval(FILE);
+ shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24),
file());
assertThat(shouldRead).as("Should not read: id below lower bound (5 < 30,
6 < 30)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 2,
INT_MIN_VALUE - 1))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE
- 1), file());
assertThat(shouldRead).as("Should not read: id below lower bound (28 < 30,
29 < 30)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 1,
INT_MIN_VALUE))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 1,
INT_MIN_VALUE), file());
assertThat(shouldRead).as("Should read: id equal to lower bound (30 ==
30)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE - 4,
INT_MAX_VALUE - 3))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE
- 3), file());
assertThat(shouldRead)
.as("Should read: id between lower and upper bounds (30 < 75 < 79, 30
< 76 < 79)")
.isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE,
INT_MAX_VALUE + 1))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE +
1), file());
assertThat(shouldRead).as("Should read: id equal to upper bound (79 ==
79)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 1,
INT_MAX_VALUE + 2))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE
+ 2), file());
assertThat(shouldRead).as("Should not read: id above upper bound (80 > 79,
81 > 79)").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 6,
INT_MAX_VALUE + 7))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE
+ 7), file());
assertThat(shouldRead).as("Should not read: id above upper bound (85 > 79,
86 > 79)").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("all_nulls", "abc",
"def")).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("all_nulls", "abc", "def"), file());
assertThat(shouldRead).as("Should skip: in on all nulls column").isFalse();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("some_nulls", "abc",
"def")).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("some_nulls", "abc", "def"), file());
assertThat(shouldRead).as("Should read: in on some nulls column").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("no_nulls", "abc",
"def")).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("no_nulls", "abc", "def"), file());
assertThat(shouldRead).as("Should read: in on no nulls column").isTrue();
// should read as the number of elements in the in expression is too big
List<Integer> ids = Lists.newArrayListWithExpectedSize(400);
for (int id = -400; id <= 0; id++) {
ids.add(id);
}
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("id",
ids)).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, in("id", ids), file());
assertThat(shouldRead).as("Should read: large in expression").isTrue();
}
@Test
public void testIntegerNotIn() {
boolean shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 25,
INT_MIN_VALUE - 24))
- .eval(FILE);
+ shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE -
24), file());
assertThat(shouldRead).as("Should read: id below lower bound (5 < 30, 6 <
30)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 2,
INT_MIN_VALUE - 1))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 2,
INT_MIN_VALUE - 1), file());
assertThat(shouldRead).as("Should read: id below lower bound (28 < 30, 29
< 30)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 1,
INT_MIN_VALUE))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 1,
INT_MIN_VALUE), file());
assertThat(shouldRead).as("Should read: id equal to lower bound (30 ==
30)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE - 4,
INT_MAX_VALUE - 3))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE - 4,
INT_MAX_VALUE - 3), file());
assertThat(shouldRead)
.as("Should read: id between lower and upper bounds (30 < 75 < 79, 30
< 76 < 79)")
.isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE,
INT_MAX_VALUE + 1))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE +
1), file());
assertThat(shouldRead).as("Should read: id equal to upper bound (79 ==
79)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 1,
INT_MAX_VALUE + 2))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 1,
INT_MAX_VALUE + 2), file());
assertThat(shouldRead).as("Should read: id above upper bound (80 > 79, 81
> 79)").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 6,
INT_MAX_VALUE + 7))
- .eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 6,
INT_MAX_VALUE + 7), file());
assertThat(shouldRead).as("Should read: id above upper bound (85 > 79, 86
> 79)").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("all_nulls",
"abc", "def")).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("all_nulls", "abc", "def"), file());
assertThat(shouldRead).as("Should read: notIn on all nulls
column").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(SCHEMA, notIn("some_nulls", "abc",
"def")).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("some_nulls", "abc", "def"), file());
assertThat(shouldRead).as("Should read: notIn on some nulls
column").isTrue();
- shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nulls",
"abc", "def")).eval(FILE);
+ shouldRead = shouldRead(SCHEMA, notIn("no_nulls", "abc", "def"), file());
assertThat(shouldRead).as("Should read: notIn on no nulls
column").isTrue();
}
@Test
public void testIsNullInNestedStruct() {
// read required_address and its nested fields
- boolean shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
isNull("required_address")).eval(FILE_6);
+ boolean shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address"),
file6());
assertThat(shouldRead).as("Should not read: required_address is
required").isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
isNull("required_address.required_street1"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
isNull("required_address.required_street1"), file6());
assertThat(shouldRead)
.as("Should not read: required_address.required_street1 is required")
.isFalse();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
isNull("required_address.optional_street1"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
isNull("required_address.optional_street1"), file6());
assertThat(shouldRead)
.as("Should read: required_address.optional_street1 is optional")
.isTrue();
// read optional_address and its nested fields
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
isNull("optional_address")).eval(FILE_6);
- assertThat(shouldRead).as("Should read: metrics are not tracked for
structs").isTrue();
+ shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address"),
file6());
+ assertThat(shouldRead).as("Should read: optional_address is
optional").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
isNull("optional_address.required_street2"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
isNull("optional_address.required_street2"), file6());
assertThat(shouldRead).as("Should read: optional_address is
optional").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
isNull("optional_address.optional_street2"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
isNull("optional_address.optional_street2"), file6());
assertThat(shouldRead).as("Should read: optional_address is
optional").isTrue();
}
@Test
public void testNotNullInNestedStruct() {
// read required_address and its nested fields
- boolean shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
notNull("required_address")).eval(FILE_6);
+ boolean shouldRead = shouldRead(NESTED_SCHEMA,
notNull("required_address"), file6());
assertThat(shouldRead).as("Should read: required_address is
required").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
notNull("required_address.required_street1"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
notNull("required_address.required_street1"), file6());
assertThat(shouldRead)
.as("Should read: required_address.required_street1 is required")
.isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
notNull("required_address.optional_street1"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
notNull("required_address.optional_street1"), file6());
assertThat(shouldRead)
.as("Should not read: required_address.optional_street1 is optional")
.isFalse();
// read optional_address and its nested fields
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
notNull("optional_address")).eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"),
file6());
assertThat(shouldRead).as("Should read: metrics are not tracked for
structs").isTrue();
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
notNull("optional_address.required_street2"))
- .eval(FILE_6);
- assertThat(shouldRead).as("Should not read: optional_address is
optional").isFalse();
-
- shouldRead =
- new InclusiveMetricsEvaluator(NESTED_SCHEMA,
notNull("optional_address.optional_street2"))
- .eval(FILE_6);
+ shouldRead = shouldRead(NESTED_SCHEMA,
notNull("optional_address.optional_street2"), file6());
assertThat(shouldRead)
.as("Should not read: optional_address.optional_street2 is optional")
.isFalse();
}
+ @Test
+ public void notNullForRequiredFieldInOptionalStruct() {
Review Comment:
Sorry about the confusion, this was my fault. When I looked at what we
currently do, I got it wrong because I used a `float` for the test.
> Having this value align with the parquet stats will be less confusing.
I thought that our stats did align with Parquet, so this statement was a
surprise. After looking into this more, I can confirm that the stats actually
do align with what Parquet produces. The problem above was that I used `float`,
which collects metrics differently to accumulate a NaN count.
If you look at stats for an `int` column that is produced directly from
Parquet, they match:
```
In [8]: %sql select * from nested_test;
Out[8]:
+----+-----------------------------+
| id | point |
+----+-----------------------------+
| 1 | Row(x=1, y=1, z=None) |
| 2 | Row(x=2, y=2, z=0) |
| 3 | Row(x=3, y=3, z=None) |
| 4 | Row(x=None, y=None, z=None) |
| 5 | Row(x=None, y=None, z=None) |
+----+-----------------------------+
```
Parquet:
```
Row group 0: count: 5 42.60 B records start: 4 total(compressed): 213 B
total(uncompressed):165 B
--------------------------------------------------------------------------------
type encodings count avg size nulls min / max
id INT32 Z _ 5 12.20 B 0 "1" / "5"
location.lat FLOAT Z _ 5 10.60 B 2 "1.0" / "3.0"
location.lon FLOAT Z _ 5 10.60 B 2 "1.0" / "3.0"
location.alt FLOAT Z _ 5 9.20 B 4 "-0.0" / "0.0"
```
Iceberg:
```
id=Row(column_size=61, value_count=5, null_value_count=0,
nan_value_count=None, lower_bound=1, upper_bound=5),
point.x=Row(column_size=53, value_count=5, null_value_count=2,
nan_value_count=None, lower_bound=1, upper_bound=3),
point.y=Row(column_size=53, value_count=5, null_value_count=2,
nan_value_count=None, lower_bound=1, upper_bound=3),
point.z=Row(column_size=45, value_count=5, null_value_count=4,
nan_value_count=None, lower_bound=0, upper_bound=0)
```
I also ran both tests for ORC and the counts agree with the Parquet counts.
> Are we able to fix this with the new v4 stats format and just use
`null_value_count` for null detection?
The most important consideration is to be consistent with what we currently
write. Since we are writing counts in metrics maps that include null parents,
we should continue to do that if possible. And luckily, that behavior aligns
with row-level evaluation already.
The existing behavior is:
- There are no null values if the null count is zero
- There is at least one null value if null count is non-zero (including null
parents)
- There is at least one non-null value if the null count is less than the
value count
- There are no non-null values if the null count is equal to the value count
Accessors return `null` when a parent is missing, so this behavior matches
the row-level evaluator. For `float` where we use leaf counts for values and
nulls, the row-level evaluator behavior does not match when a parent is null.
`point.x IS NULL` will return true when `point=null`, but a file containing
only null values for `point` will be skipped because null count is 0.
We will need to address the issue with `float` and `double`. To fix this for
existing files, I think we should check whether the value count is less than
the row count and use that to signal that parents are null. We should also
update how we store stats and use values from Parquet for these types as well.
For v4, we could change to use leaf-based counts rather than including null
parents in the value and null value counts. That would save us from needing an
extra null count column to detect the case where all parents are defined (null
count = 0). It's reasonable to be consistent here.
> Also checking value count == row count won't work as we have array of
struct case.
I don't think this case is relevant to the choice for structs that have at
most 1 value for a field. We also don't currently store counts for array
elements or map key/value pairs, which is a good thing.
Evaluating `point.x IS NULL` returns true when `point` is null, which means
null parents can be handled as null values in stats. (Keep in mind that this
wasn't deliberate, it was a result of null handling in `Accessor`. It just
happens to align with SQL behavior.)
Predicates for arrays and maps would behave differently. For an array `arr`,
you can't directly reference the element like in `arr.element IS NULL`. You'd
have to translate to something like `arr CONTAINS NULL` or `arr[0] IS NULL`.
These aren't necessarily true when the array itself is null because these imply
that the array is not null. In Spark, `arr[0] IS NOT NULL` is false when `arr`
is null: a null array's first element is not null!
Getting back to the point, we could infer the number of missing parents in
structs by comparing row count and value count if we know that there is always
1 value for the field when parents are present. That logic is based on the type
structure and it doesn't hold for arrays, which could contain 0 elements.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
