Yuming Wang created SPARK-40303: ----------------------------------- Summary: The performance will be worse after codegen Key: SPARK-40303 URL: https://issues.apache.org/jira/browse/SPARK-40303 Project: Spark Issue Type: Bug Components: SQL Affects Versions: 3.4.0 Reporter: Yuming Wang
{code:scala} import org.apache.spark.benchmark.Benchmark val dir = "/tmp/spark/benchmark" val N = 2000000 val columns = Range(0, 100).map(i => s"id % $i AS id$i") spark.range(N).selectExpr(columns: _*).write.mode("Overwrite").parquet(dir) // Seq(1, 2, 5, 10, 15, 25, 40, 60, 100) Seq(60).foreach{ cnt => val selectExps = columns.take(cnt).map(_.split(" ").last).map(c => s"count(distinct $c)") val benchmark = new Benchmark("Benchmark count distinct", N, minNumIters = 1) benchmark.addCase(s"$cnt count distinct with codegen") { _ => withSQLConf( "spark.sql.codegen.wholeStage" -> "true", "spark.sql.codegen.factoryMode" -> "FALLBACK") { spark.read.parquet(dir).selectExpr(selectExps: _*).write.format("noop").mode("Overwrite").save() } } benchmark.addCase(s"$cnt count distinct without codegen") { _ => withSQLConf( "spark.sql.codegen.wholeStage" -> "false", "spark.sql.codegen.factoryMode" -> "NO_CODEGEN") { spark.read.parquet(dir).selectExpr(selectExps: _*).write.format("noop").mode("Overwrite").save() } } benchmark.run() } {code} {noformat} Java HotSpot(TM) 64-Bit Server VM 1.8.0_281-b09 on Mac OS X 10.15.7 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz Benchmark count distinct: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ 60 count distinct with codegen 628146 628146 0 0.0 314072.8 1.0X 60 count distinct without codegen 147635 147635 0 0.0 73817.5 4.3X {noformat} -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org