sunchao commented on a change in pull request #32473:
URL: https://github.com/apache/spark/pull/32473#discussion_r628909233
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BloomFilterBenchmark.scala
##########
@@ -81,8 +80,51 @@ object BloomFilterBenchmark extends SqlBasedBenchmark {
}
}
+ private def writeParquetBenchmark(): Unit = {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+
+ runBenchmark(s"Parquet Write") {
+ val benchmark = new Benchmark(s"Write ${scaleFactor}M rows", N, output
= output)
+ benchmark.addCase("Without bloom filter") { _ =>
+ df.write.mode("overwrite").parquet(path + "/withoutBF")
+ }
+ benchmark.addCase("With bloom filter") { _ =>
+ df.write.mode("overwrite")
+ .option(ParquetOutputFormat.BLOOM_FILTER_ENABLED + "#value", true)
+ .parquet(path + "/withBF")
+ }
+ benchmark.run()
+ }
+ }
+ }
+
+ private def readParquetBenchmark(): Unit = {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+
+ df.write.parquet(path + "/withoutBF")
+ df.write.option(ParquetOutputFormat.BLOOM_FILTER_ENABLED + "#value",
true)
+ .parquet(path + "/withBF")
Review comment:
Wow that's a great improvement! I wonder if we should make the row group
size a parameter for the benchmark too. There seems to be other related
parameters too such as `DEFAULT_MAX_BLOOM_FILTER_BYTES`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
