hemanthboyina commented on code in PR #58272:
URL: https://github.com/apache/spark/pull/58272#discussion_r3861835780
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala:
##########
@@ -85,23 +84,12 @@ abstract class DataSourceV2RelationBase(
s"RelationV2$outputString $nameWithTimeTravelSpec"
}
- override def computeStats(): Statistics = {
- if (Utils.isTesting) {
- // when testing, throw an exception if this computeStats method is
called because stats should
- // not be accessed before pushing the projection and filters to create a
scan. otherwise, the
- // stats are not accurate because they are based on a full table scan of
all columns.
- throw SparkException.internalError(
- s"BUG: computeStats called before pushdown on DSv2 relation: $name")
- } else {
- // when not testing, return stats because bad stats are better than
failing a query
- table.asReadable.newScanBuilder(options).build() match {
- case r: SupportsReportStatistics =>
- val statistics = r.estimateStatistics()
- DataSourceV2Relation.transformV2Stats(statistics,
conf.defaultSizeInBytes, output)
- case _ =>
- Statistics(sizeInBytes = conf.defaultSizeInBytes)
- }
- }
+ override def computeStats(): Statistics = table match {
+ case t: SupportsReportCatalogStatistics =>
+ DataSourceV2Relation.transformV2Stats(
+ t.estimateCatalogStatistics(), conf.defaultSizeInBytes, output)
+ case _ =>
+ Statistics(sizeInBytes = conf.defaultSizeInBytes)
}
Review Comment:
Good catch, thank you @uros-b . Added a secondary fallback. for tables with
BATCH_READ capability that haven't yet implemented
SupportsReportCatalogStatistics, we now fall back to building the scan and
delegating to SupportsReportStatistics if the scan supports it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
