This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 4968f87 [SPARK-23263][TEST] CTAS should update stat if autoUpdate statistics is enabled 4968f87 is described below commit 4968f871685c23b83690a9a1490ed886a3417d93 Author: Yuming Wang <yumw...@ebay.com> AuthorDate: Thu Jun 20 14:19:10 2019 +0900 [SPARK-23263][TEST] CTAS should update stat if autoUpdate statistics is enabled ## What changes were proposed in this pull request? The [SPARK-27403](https://issues.apache.org/jira/browse/SPARK-27403) fixed CTAS cannot update statistics even if `spark.sql.statistics.size.autoUpdate.enabled` is enabled, as mentioned in [SPARK-23263](https://issues.apache.org/jira/browse/SPARK-23263). This pr adds tests for that fix. ## How was this patch tested? N/A Closes #20430 from wangyum/SPARK-23263. Authored-by: Yuming Wang <yumw...@ebay.com> Signed-off-by: HyukjinKwon <gurwls...@apache.org> --- .../apache/spark/sql/hive/StatisticsSuite.scala | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala index 483bd37..7a8e257 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala @@ -1431,4 +1431,26 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto assert(catalogStats.rowCount.isEmpty) } } + + test(s"CTAS should update statistics if ${SQLConf.AUTO_SIZE_UPDATE_ENABLED.key} is enabled") { + val tableName = "SPARK_23263" + Seq(false, true).foreach { isConverted => + Seq(false, true).foreach { updateEnabled => + withSQLConf( + SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> updateEnabled.toString, + HiveUtils.CONVERT_METASTORE_PARQUET.key -> isConverted.toString) { + withTable(tableName) { + sql(s"CREATE TABLE $tableName STORED AS parquet AS SELECT 'a', 'b'") + val catalogTable = getCatalogTable(tableName) + // Hive serde tables always update statistics by Hive metastore + if (!isConverted || updateEnabled) { + assert(catalogTable.stats.nonEmpty) + } else { + assert(catalogTable.stats.isEmpty) + } + } + } + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org