This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e60d8fc [SPARK-27045][SQL] SQL tab in UI shows actual SQL instead of callsite in case of SparkSQLDriver e60d8fc is described below commit e60d8fce0b0cf2a6d766ea2fc5f994546550570a Author: Ajith <ajith2...@gmail.com> AuthorDate: Tue Mar 12 16:14:29 2019 -0700 [SPARK-27045][SQL] SQL tab in UI shows actual SQL instead of callsite in case of SparkSQLDriver ## What changes were proposed in this pull request? When we run sql in spark via SparkSQLDriver (thrift server, spark-sql), SQL string is siet via ``setJobDescription``. the SparkUI SQL tab must show SQL instead of stacktrace in case ``setJobDescription`` is set which is more useful to end user. Instead it currently shows in description column the callsite shortform which is less useful ![image](https://user-images.githubusercontent.com/22072336/53734682-aaa7d900-3eaa-11e9-957b-0e5006db417e.png) ## How was this patch tested? Manually: ![image](https://user-images.githubusercontent.com/22072336/53734657-9f54ad80-3eaa-11e9-8dc5-2b38f6970f4e.png) Closes #23958 from ajithme/sqlui. Authored-by: Ajith <ajith2...@gmail.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../org/apache/spark/sql/internal/StaticSQLConf.scala | 6 ++++++ .../org/apache/spark/sql/execution/SQLExecution.scala | 15 ++++++++++++++- .../apache/spark/sql/execution/ui/AllExecutionsPage.scala | 5 +++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala index fc07efb..e12f05b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala @@ -132,4 +132,10 @@ object StaticSQLConf { .intConf .createWithDefault(1000) + val SQL_EVENT_TRUNCATE_LENGTH = buildStaticConf("spark.sql.event.truncate.length") + .doc("Threshold of SQL length beyond which it will be truncated before adding to " + + "event. Defaults to no truncation. If set to 0, callsite will be logged instead.") + .intConf + .checkValue(_ >= 0, "Must be set greater or equal to zero") + .createWithDefault(Int.MaxValue) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala index 5b38fe5..ca66337 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala @@ -20,9 +20,12 @@ package org.apache.spark.sql.execution import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong +import org.apache.spark.SparkContext import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.ui.{SparkListenerSQLExecutionEnd, SparkListenerSQLExecutionStart} +import org.apache.spark.sql.internal.StaticSQLConf.SQL_EVENT_TRUNCATE_LENGTH +import org.apache.spark.util.Utils object SQLExecution { @@ -71,13 +74,23 @@ object SQLExecution { // streaming queries would give us call site like "run at <unknown>:0" val callSite = sc.getCallSite() + val truncateLength = sc.conf.get(SQL_EVENT_TRUNCATE_LENGTH) + + val desc = Option(sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION)) + .filter(_ => truncateLength > 0) + .map { sqlStr => + val redactedStr = Utils + .redact(sparkSession.sessionState.conf.stringRedactionPattern, sqlStr) + redactedStr.substring(0, Math.min(truncateLength, redactedStr.length)) + }.getOrElse(callSite.shortForm) + withSQLConfPropagated(sparkSession) { var ex: Option[Exception] = None val startTime = System.nanoTime() try { sc.listenerBus.post(SparkListenerSQLExecutionStart( executionId = executionId, - description = callSite.shortForm, + description = desc, details = callSite.longForm, physicalPlanDescription = queryExecution.toString, // `queryExecution.executedPlan` triggers query planning. If it fails, the exception diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala index 43ff1e1..824c094 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala @@ -388,14 +388,15 @@ private[ui] class ExecutionPagedTable( +details </span> ++ <div class="stage-details collapsed"> - <pre>{execution.details}</pre> + <pre>{execution.description}<br></br>{execution.details}</pre> </div> } else { Nil } val desc = if (execution.description != null && execution.description.nonEmpty) { - <a href={executionURL(execution.executionId)}>{execution.description}</a> + <a href={executionURL(execution.executionId)} class="description-input"> + {execution.description}</a> } else { <a href={executionURL(execution.executionId)}>{execution.executionId}</a> } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org