(incubator-gluten) branch main updated: [VL] Add more metrics for generate (#5608)

[ulyssesyou]

This is an automated email from the ASF dual-hosted git repository.

ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 939f929af [VL] Add more metrics for generate (#5608)
939f929af is described below

commit 939f929afe9db02e9a342fd8fddf0e43c359ebfc
Author: Xiduo You <ulyssesyo...@gmail.com>
AuthorDate: Mon May 6 16:05:59 2024 +0800

    [VL] Add more metrics for generate (#5608)
---
 .../apache/gluten/execution/GenerateExecTransformer.scala | 12 +++++++++++-
 .../org/apache/gluten/execution/VeloxMetricsSuite.scala   | 15 +++++++++++++++
 .../apache/gluten/metrics/GenerateMetricsUpdater.scala    |  6 ++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
index 08d2937b5..830fe396b 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
@@ -51,7 +51,17 @@ case class GenerateExecTransformer(
 
   @transient
   override lazy val metrics =
-    Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of 
output rows"))
+    Map(
+      "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of 
output rows"),
+      "numOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of 
output vectors"),
+      "numOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of 
output bytes"),
+      "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, 
"totaltime of generate"),
+      "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time 
count"),
+      "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak 
memory bytes"),
+      "numMemoryAllocations" -> SQLMetrics.createMetric(
+        sparkContext,
+        "number of memory allocations")
+    )
 
   override def metricsUpdater(): MetricsUpdater = new 
GenerateMetricsUpdater(metrics)
 
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
index ac9911bba..ce8450fea 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
@@ -128,6 +128,21 @@ class VeloxMetricsSuite extends 
VeloxWholeStageTransformerSuite with AdaptiveSpa
     }
   }
 
+  test("Generate metrics") {
+    runQueryAndCompare("SELECT explode(array(c1, c2, 1)) FROM metrics_t1") {
+      df =>
+        val generate = find(df.queryExecution.executedPlan) {
+          case _: GenerateExecTransformer => true
+          case _ => false
+        }
+        assert(generate.isDefined)
+        val metrics = generate.get.metrics
+        assert(metrics("numOutputRows").value == 300)
+        assert(metrics("numOutputVectors").value > 0)
+        assert(metrics("numOutputBytes").value > 0)
+    }
+  }
+
   test("Write metrics") {
     if (SparkShimLoader.getSparkVersion.startsWith("3.4")) {
       withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) {
diff --git 
a/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
 
b/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
index 670fd1c4d..0a3dccd64 100644
--- 
a/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
+++ 
b/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
@@ -23,6 +23,12 @@ class GenerateMetricsUpdater(val metrics: Map[String, 
SQLMetric]) extends Metric
     if (operatorMetrics != null) {
       val nativeMetrics = operatorMetrics.asInstanceOf[OperatorMetrics]
       metrics("numOutputRows") += nativeMetrics.outputRows
+      metrics("numOutputVectors") += nativeMetrics.outputVectors
+      metrics("numOutputBytes") += nativeMetrics.outputBytes
+      metrics("cpuCount") += nativeMetrics.cpuCount
+      metrics("wallNanos") += nativeMetrics.wallNanos
+      metrics("peakMemoryBytes") += nativeMetrics.peakMemoryBytes
+      metrics("numMemoryAllocations") += nativeMetrics.numMemoryAllocations
     }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org