alamb commented on code in PR #8941:
URL: https://github.com/apache/arrow-datafusion/pull/8941#discussion_r1462347849
##########
datafusion/core/tests/parquet/row_group_pruning.rs:
##########
@@ -215,369 +242,364 @@ async fn prune_disabled() {
#[tokio::test]
async fn prune_int32_lt() {
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where i < 1",
- Some(0),
- Some(1),
- Some(0),
- 11,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where i < 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(11)
+ .test_row_group_prune()
+ .await;
+
// result of sql "SELECT * FROM t where i < 1" is same as
// "SELECT * FROM t where -i > -1"
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where -i > -1",
- Some(0),
- Some(1),
- Some(0),
- 11,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where -i > -1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(11)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_eq() {
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where i = 1",
- Some(0),
- Some(3),
- Some(0),
- 1,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where i = 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(3))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(1)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_scalar_fun_and_eq() {
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where abs(i) = 1 and i = 1",
- Some(0),
- Some(3),
- Some(0),
- 1,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where i = 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(3))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(1)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_scalar_fun() {
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where abs(i) = 1",
- Some(0),
- Some(0),
- Some(0),
- 3,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where abs(i) = 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(3)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_complex_expr() {
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where i+1 = 1",
- Some(0),
- Some(0),
- Some(0),
- 2,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where i+1 = 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(2)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_complex_expr_subtract() {
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where 1-i > 1",
- Some(0),
- Some(0),
- Some(0),
- 9,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where 1-i > 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(9)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_f64_lt() {
- test_row_group_prune(
- Scenario::Float64,
- "SELECT * FROM t where f < 1",
- Some(0),
- Some(1),
- Some(0),
- 11,
- )
- .await;
- test_row_group_prune(
- Scenario::Float64,
- "SELECT * FROM t where -f > -1",
- Some(0),
- Some(1),
- Some(0),
- 11,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Float64)
+ .with_query("SELECT * FROM t where f < 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(11)
+ .test_row_group_prune()
+ .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Float64)
+ .with_query("SELECT * FROM t where -f > -1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(11)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_f64_scalar_fun_and_gt() {
// result of sql "SELECT * FROM t where abs(f - 1) <= 0.000001 and f >=
0.1"
// only use "f >= 0" to prune
- test_row_group_prune(
- Scenario::Float64,
- "SELECT * FROM t where abs(f - 1) <= 0.000001 and f >= 0.1",
- Some(0),
- Some(2),
- Some(0),
- 1,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Float64)
+ .with_query("SELECT * FROM t where abs(f - 1) <= 0.000001 and f >=
0.1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(2))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(1)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_f64_scalar_fun() {
// result of sql "SELECT * FROM t where abs(f-1) <= 0.000001" is not
supported
- test_row_group_prune(
- Scenario::Float64,
- "SELECT * FROM t where abs(f-1) <= 0.000001",
- Some(0),
- Some(0),
- Some(0),
- 1,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Float64)
+ .with_query("SELECT * FROM t where abs(f-1) <= 0.000001")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(1)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_f64_complex_expr() {
// result of sql "SELECT * FROM t where f+1 > 1.1"" is not supported
- test_row_group_prune(
- Scenario::Float64,
- "SELECT * FROM t where f+1 > 1.1",
- Some(0),
- Some(0),
- Some(0),
- 9,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Float64)
+ .with_query("SELECT * FROM t where f+1 > 1.1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(9)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_f64_complex_expr_subtract() {
// result of sql "SELECT * FROM t where 1-f > 1" is not supported
- test_row_group_prune(
- Scenario::Float64,
- "SELECT * FROM t where 1-f > 1",
- Some(0),
- Some(0),
- Some(0),
- 9,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Float64)
+ .with_query("SELECT * FROM t where 1-f > 1")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(9)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_eq_in_list() {
// result of sql "SELECT * FROM t where in (1)"
- test_row_group_prune(
- Scenario::Int32,
- "SELECT * FROM t where i in (1)",
- Some(0),
- Some(3),
- Some(0),
- 1,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where i in (1)")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(3))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(1)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_eq_in_list_2() {
// result of sql "SELECT * FROM t where in (1000)", prune all
// test whether statistics works
- test_prune_verbose(
- Scenario::Int32,
- "SELECT * FROM t where i in (1000)",
- Some(0),
- Some(0),
- Some(4),
- 0,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32)
+ .with_query("SELECT * FROM t where i in (1000)")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(4))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(0)
+ .test_row_group_prune()
+ .await;
}
#[tokio::test]
async fn prune_int32_eq_large_in_list() {
// result of sql "SELECT * FROM t where i in (2050...2582)", prune all
- // test whether sbbf works
- test_prune_verbose(
- Scenario::Int32Range,
- format!(
- "SELECT * FROM t where i in ({})",
- (200050..200082).join(",")
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int32Range)
+ .with_query(
+ format!(
+ "SELECT * FROM t where i in ({})",
+ (200050..200082).join(",")
+ )
+ .as_str(),
)
- .as_str(),
- Some(0),
- Some(1),
- // we don't support pruning by statistics for in_list with more than
20 elements currently
- Some(0),
- 0,
- )
- .await;
+ .with_expected_errors(Some(0))
Review Comment:
This seems like the test is different -- the results are 0, 0, 1 rather than
0,1,0
Perhaps there is a difference between `test_row_group_prune` and
`test_prune_verbose`
##########
datafusion/core/tests/parquet/row_group_pruning.rs:
##########
@@ -586,32 +608,31 @@ async fn prune_periods_in_column_names() {
// name = "HTTP GET / DISPATCH", service.name = ['frontend', 'frontend'],
// name = "HTTP PUT / DISPATCH", service.name = ['backend', 'frontend'],
// name = "HTTP GET / DISPATCH", service.name = ['backend', 'backend' ],
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- // use double quotes to use column named "service.name"
- "SELECT \"name\", \"service.name\" FROM t WHERE \"service.name\" =
'frontend'",
- Some(0),
- Some(1), // prune out last row group
- Some(0),
- 7,
- )
- .await;
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- "SELECT \"name\", \"service.name\" FROM t WHERE \"name\" != 'HTTP GET
/ DISPATCH'",
- Some(0),
- Some(2), // prune out first and last row group
- Some(0),
- 5,
- )
- .await;
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- "SELECT \"name\", \"service.name\" FROM t WHERE \"service.name\" =
'frontend' AND \"name\" != 'HTTP GET / DISPATCH'",
- Some(0),
- Some(2), // prune out middle and last row group
- Some(0),
- 2,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::PeriodsInColumnNames)
+ .with_query( "SELECT \"name\", \"service.name\" FROM t WHERE
\"service.name\" = 'frontend'")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
Review Comment:
```suggestion
// prune out first and last row group
.with_pruned_by_stats(Some(1))
```
##########
datafusion/core/tests/parquet/row_group_pruning.rs:
##########
@@ -586,32 +608,31 @@ async fn prune_periods_in_column_names() {
// name = "HTTP GET / DISPATCH", service.name = ['frontend', 'frontend'],
// name = "HTTP PUT / DISPATCH", service.name = ['backend', 'frontend'],
// name = "HTTP GET / DISPATCH", service.name = ['backend', 'backend' ],
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- // use double quotes to use column named "service.name"
- "SELECT \"name\", \"service.name\" FROM t WHERE \"service.name\" =
'frontend'",
- Some(0),
- Some(1), // prune out last row group
- Some(0),
- 7,
- )
- .await;
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- "SELECT \"name\", \"service.name\" FROM t WHERE \"name\" != 'HTTP GET
/ DISPATCH'",
- Some(0),
- Some(2), // prune out first and last row group
- Some(0),
- 5,
- )
- .await;
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- "SELECT \"name\", \"service.name\" FROM t WHERE \"service.name\" =
'frontend' AND \"name\" != 'HTTP GET / DISPATCH'",
- Some(0),
- Some(2), // prune out middle and last row group
- Some(0),
- 2,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::PeriodsInColumnNames)
+ .with_query( "SELECT \"name\", \"service.name\" FROM t WHERE
\"service.name\" = 'frontend'")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(7)
+ .test_row_group_prune()
+ .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::PeriodsInColumnNames)
+ .with_query( "SELECT \"name\", \"service.name\" FROM t WHERE \"name\"
!= 'HTTP GET / DISPATCH'")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(2))
Review Comment:
```suggestion
// prune out first and last row group
.with_pruned_by_stats(Some(2))
```
##########
datafusion/core/tests/parquet/row_group_pruning.rs:
##########
@@ -586,32 +608,31 @@ async fn prune_periods_in_column_names() {
// name = "HTTP GET / DISPATCH", service.name = ['frontend', 'frontend'],
// name = "HTTP PUT / DISPATCH", service.name = ['backend', 'frontend'],
// name = "HTTP GET / DISPATCH", service.name = ['backend', 'backend' ],
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- // use double quotes to use column named "service.name"
- "SELECT \"name\", \"service.name\" FROM t WHERE \"service.name\" =
'frontend'",
- Some(0),
- Some(1), // prune out last row group
- Some(0),
- 7,
- )
- .await;
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- "SELECT \"name\", \"service.name\" FROM t WHERE \"name\" != 'HTTP GET
/ DISPATCH'",
- Some(0),
- Some(2), // prune out first and last row group
- Some(0),
- 5,
- )
- .await;
- test_row_group_prune(
- Scenario::PeriodsInColumnNames,
- "SELECT \"name\", \"service.name\" FROM t WHERE \"service.name\" =
'frontend' AND \"name\" != 'HTTP GET / DISPATCH'",
- Some(0),
- Some(2), // prune out middle and last row group
- Some(0),
- 2,
- )
- .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::PeriodsInColumnNames)
+ .with_query( "SELECT \"name\", \"service.name\" FROM t WHERE
\"service.name\" = 'frontend'")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(7)
+ .test_row_group_prune()
+ .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::PeriodsInColumnNames)
+ .with_query( "SELECT \"name\", \"service.name\" FROM t WHERE \"name\"
!= 'HTTP GET / DISPATCH'")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(2))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(5)
+ .test_row_group_prune()
+ .await;
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::PeriodsInColumnNames)
+ .with_query( "SELECT \"name\", \"service.name\" FROM t WHERE
\"service.name\" = 'frontend' AND \"name\" != 'HTTP GET / DISPATCH'")
+ .with_expected_errors(Some(0))
+ .with_pruned_by_stats(Some(2))
Review Comment:
```suggestion
// prune out first and last row group
.with_pruned_by_stats(Some(2))
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
