LuciferYang commented on code in PR #60025:
URL: https://github.com/apache/spark/pull/60025#discussion_r4467196702
##########
sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala:
##########
@@ -202,6 +203,59 @@ class StatisticsCollectionSuite extends
StatisticsCollectionTestBase with Shared
}
}
+ test("SPARK-60810: CBO keeps CHAR column statistics through read-side
padding") {
+ withSQLConf(
+ SQLConf.CBO_ENABLED.key -> "true",
+ SQLConf.READ_SIDE_CHAR_PADDING.key -> "true") {
+ val tableName = "char_read_side_padding_stats"
+ withTable(tableName) {
+ sql(s"CREATE TABLE $tableName(c CHAR(5)) USING parquet")
+ sql(s"INSERT INTO $tableName SELECT CAST(id % 10 AS STRING) FROM
range(1000)")
+ sql(s"ANALYZE TABLE $tableName COMPUTE STATISTICS FOR COLUMNS c")
+
+ def estimatedRows(query: String)(
+ pick: PartialFunction[LogicalPlan, LogicalPlan]): Option[BigInt] =
{
+ val plan = sql(query).queryExecution.optimizedPlan
+ plan.collectFirst(pick).getOrElse(fail(s"no matching node
in\n${plan.treeString}"))
+ .stats.rowCount
+ }
+ // Above the padding Project, which carries the column's statistics.
+ assert(estimatedRows(s"SELECT c, count(*) FROM $tableName GROUP BY c")
{
+ case a: Aggregate => a
+ }.contains(BigInt(10)))
+ // Pushed below it, where the predicate compares the padding
expression itself.
+ assert(estimatedRows(s"SELECT * FROM $tableName WHERE c = '3'") {
+ case f: Filter => f
+ }.contains(BigInt(100)))
+ assert(estimatedRows(s"SELECT * FROM $tableName WHERE c IN ('3',
'4')") {
+ case f: Filter => f
+ }.contains(BigInt(200)))
+ }
+ }
+ }
+
+ test("SPARK-60810: partition pruning estimates a filter on a CHAR partition
column") {
Review Comment:
Done in 1d5d0aa1721. `DataSourceV2RelationSuite` sends pushed and inferred
filters on `readSidePadding` of a CHAR column through
`estimateStatsWithFilters`: a pushed equality is estimated at 100 of 1000 rows,
and with a pushed `STARTSWITH`, which cannot be estimated, the inferred `IS NOT
NULL` gives 800.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala:
##########
@@ -366,7 +373,14 @@ case class FilterEstimation(
if (!valueInRange) return Some(0.0)
- val percent = if (colStat.histogram.isDefined) {
+ // Only numeric-like types have equi-height histograms: the histogram path
converts the
+ // literal to a double. A string or binary column uses its distinct count
even if a connector
+ // reported a histogram for it.
+ val histogram = attr.dataType match {
Review Comment:
Done in 1d5d0aa1721, as a Boolean `useHistogram`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
