izchen commented on a change in pull request #3863:
URL: https://github.com/apache/iceberg/pull/3863#discussion_r781809578
##########
File path:
spark/v3.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala
##########
@@ -41,6 +43,11 @@ abstract class DynamicFileFilterExecBase(
@transient
override lazy val references: AttributeSet =
AttributeSet(fileFilterExec.output)
+ override lazy val metrics = Map(
+ "totalFiles" -> SQLMetrics.createMetric(sparkContext, "total number of
files"),
+ "hitFiles" -> SQLMetrics.createMetric(sparkContext, "number of files hit"),
+ "fileHitRate" -> SQLMetrics.createMetric(sparkContext, "file hit rate %"))
Review comment:
done
##########
File path:
spark/v3.0/spark/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java
##########
@@ -32,5 +32,22 @@
*
* @param locations file locations
*/
- void filterFiles(Set<String> locations);
+ SupportsFileFilter.FileFilterMetric filterFiles(Set<String> locations);
+
+ class FileFilterMetric {
+ private int totalFilesNumber;
+ private int hitFilesNumber;
+
+ public FileFilterMetric(int totalFilesNumber, int hitFilesNumber) {
+ this.totalFilesNumber = totalFilesNumber;
+ this.hitFilesNumber = hitFilesNumber;
+ }
+
+ public int getTotalFilesNumber() {
+ return totalFilesNumber;
+ }
+ public int getHitFilesNumber() {
Review comment:
done
##########
File path:
spark/v3.0/spark/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java
##########
@@ -32,5 +32,22 @@
*
* @param locations file locations
*/
- void filterFiles(Set<String> locations);
+ SupportsFileFilter.FileFilterMetric filterFiles(Set<String> locations);
+
+ class FileFilterMetric {
+ private int totalFilesNumber;
+ private int hitFilesNumber;
+
+ public FileFilterMetric(int totalFilesNumber, int hitFilesNumber) {
+ this.totalFilesNumber = totalFilesNumber;
+ this.hitFilesNumber = hitFilesNumber;
+ }
+
+ public int getTotalFilesNumber() {
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
