This is an automated email from the ASF dual-hosted git repository. hvanhovell pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push: new d1c3902921f [SPARK-42564][CONNECT] Implement SparkSession.version and SparkSession.time d1c3902921f is described below commit d1c3902921f6978fbdcda6de5def42bff08c39bd Author: panbingkun <pbk1...@gmail.com> AuthorDate: Sun Feb 26 20:39:12 2023 -0400 [SPARK-42564][CONNECT] Implement SparkSession.version and SparkSession.time ### What changes were proposed in this pull request? The pr aims to implement SparkSession.version and SparkSession.time. ### Why are the changes needed? API coverage. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add new UT. Closes #40176 from panbingkun/SPARK-42564. Authored-by: panbingkun <pbk1...@gmail.com> Signed-off-by: Herman van Hovell <her...@databricks.com> (cherry picked from commit 9c7aa16c9a5ede3a712a49f84118bfff89273f60) Signed-off-by: Herman van Hovell <her...@databricks.com> --- .../scala/org/apache/spark/sql/SparkSession.scala | 20 ++++++++++++++++++++ .../org/apache/spark/sql/ClientE2ETestSuite.scala | 10 ++++++++++ 2 files changed, 30 insertions(+) diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala index 53cd3955232..e39a6779e25 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -17,12 +17,14 @@ package org.apache.spark.sql import java.io.Closeable +import java.util.concurrent.TimeUnit._ import java.util.concurrent.atomic.AtomicLong import scala.collection.JavaConverters._ import org.apache.arrow.memory.RootAllocator +import org.apache.spark.SPARK_VERSION import org.apache.spark.annotation.Experimental import org.apache.spark.connect.proto import org.apache.spark.internal.Logging @@ -59,6 +61,24 @@ class SparkSession( private[this] val allocator = new RootAllocator() + def version: String = SPARK_VERSION + + /** + * Executes some code block and prints to stdout the time taken to execute the block. This is + * available in Scala only and is used primarily for interactive testing and debugging. + * + * @since 3.4.0 + */ + def time[T](f: => T): T = { + val start = System.nanoTime() + val ret = f + val end = System.nanoTime() + // scalastyle:off println + println(s"Time taken: ${NANOSECONDS.toMillis(end - start)} ms") + // scalastyle:on println + ret + } + /** * Executes a SQL query substituting named parameters by the given arguments, returning the * result as a `DataFrame`. This API eagerly runs DDL/DML commands, but not for SELECT queries. diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala index 23a5d5f5e9e..33e9d0756c1 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala @@ -26,6 +26,7 @@ import org.apache.commons.io.FileUtils import org.apache.commons.io.output.TeeOutputStream import org.scalactic.TolerantNumerics +import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.connect.client.util.{IntegrationTestUtils, RemoteSparkSession} import org.apache.spark.sql.functions.{aggregate, array, col, lit, rand, sequence, shuffle, transform, udf} import org.apache.spark.sql.types._ @@ -417,4 +418,13 @@ class ClientE2ETestSuite extends RemoteSparkSession { spark.range(1000).createOrReplaceGlobalTempView("view1") assert(spark.sql("SELECT * FROM global_temp.view1").count() == 1000) } + + test("version") { + assert(spark.version == SPARK_VERSION) + } + + test("time") { + val timeFragments = Seq("Time taken: ", " ms") + testCapturedStdOut(spark.time(spark.sql("select 1").collect()), timeFragments: _*) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org