wForget commented on code in PR #3708:
URL: https://github.com/apache/datafusion-comet/pull/3708#discussion_r2943886515
##########
spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala:
##########
@@ -77,6 +79,43 @@ class CometPluginsSuite extends CometTestBase {
}
}
+ test("CometSource metrics are recorded") {
+ val nativeBefore = CometSource.NATIVE_OPERATORS.getCount
+ val queriesBefore = CometSource.QUERIES_PLANNED.getCount
+
+ withTempPath { dir =>
+ val path = new File(dir, "test.parquet").toString
+ spark.range(1000).toDF("id").write.mode(SaveMode.Overwrite).parquet(path)
+ spark.read.parquet(path).filter("id > 500").collect()
+ }
+
+ assert(
+ CometSource.QUERIES_PLANNED.getCount > queriesBefore,
+ "queries.planned should increment after query")
+ assert(
+ CometSource.NATIVE_OPERATORS.getCount > nativeBefore,
+ "operators.native should increment for native execution")
+ }
+
+ test("metrics not double counted with AQE") {
+ withSQLConf("spark.sql.adaptive.enabled" -> "true") {
+ withTempPath { dir =>
+ val path = new File(dir, "test.parquet").toString
+
spark.range(10000).toDF("id").write.mode(SaveMode.Overwrite).parquet(path)
+
+ val queriesBefore = CometSource.QUERIES_PLANNED.getCount
+
+ spark.read.parquet(path).filter("id > 100").collect()
+ spark.read.parquet(path).filter("id > 200").collect()
+
+ val queriesAfter = CometSource.QUERIES_PLANNED.getCount
Review Comment:
Add `spark.sparkContext.listenerBus.waitUntilEmpty()` before this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
