This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 375aacd2e730 [SPARK-45886][SQL] Output full stack trace in `callSite`
of DataFrame context
375aacd2e730 is described below
commit 375aacd2e7308395b5a5503210cbfc5177be2925
Author: Max Gekk <max.g...@gmail.com>
AuthorDate: Sun Nov 12 14:28:44 2023 -0800
[SPARK-45886][SQL] Output full stack trace in `callSite` of DataFrame
context
### What changes were proposed in this pull request?
In the PR, I propose to include all available stack traces in DataFrame
context to the `callSite` field and apparently to the `summary`. For now,
DataFrame context contains only one item of stack trace, but later we'll add a
config to control the number of items in stack traces (see
https://github.com/apache/spark/pull/43695).
### Why are the changes needed?
To improve user experience with Spark SQL while debugging some issue. Users
can see all available stack trace, and see from where the issue comes in user
code from.
### Does this PR introduce _any_ user-facing change?
No, should not. Even if user's code parses the summary.
### How was this patch tested?
By running new test suite:
```
$ build/sbt "test:testOnly *QueryContextSuite"
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43758 from MaxGekk/output-stack-trace.
Authored-by: Max Gekk <max.g...@gmail.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../spark/sql/catalyst/trees/QueryContexts.scala | 4 +--
.../spark/sql/errors/QueryContextSuite.scala | 39 ++++++++++++++++++++++
2 files changed, 41 insertions(+), 2 deletions(-)
diff --git
a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
index 874c834b7558..57271e535afb 100644
---
a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
+++
b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
@@ -153,7 +153,7 @@ case class DataFrameQueryContext(stackTrace:
Seq[StackTraceElement]) extends Que
}.getOrElse("")
}
- override val callSite: String =
stackTrace.tail.headOption.map(_.toString).getOrElse("")
+ override val callSite: String = stackTrace.tail.mkString("\n")
override lazy val summary: String = {
val builder = new StringBuilder
@@ -162,7 +162,7 @@ case class DataFrameQueryContext(stackTrace:
Seq[StackTraceElement]) extends Que
builder ++= fragment
builder ++= "\""
- builder ++= " was called from "
+ builder ++= " was called from\n"
builder ++= callSite
builder += '\n'
builder.result()
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryContextSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryContextSuite.scala
new file mode 100644
index 000000000000..7d57eeb01bfa
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryContextSuite.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.errors
+
+import org.apache.spark.SparkArithmeticException
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.functions.lit
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class QueryContextSuite extends QueryTest with SharedSparkSession {
+
+ test("summary of DataFrame context") {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ val e = intercept[SparkArithmeticException] {
+ spark.range(1).select(lit(1) / lit(0)).collect()
+ }
+ assert(e.getQueryContext.head.summary() ==
+ """== DataFrame ==
+ |"div" was called from
+
|org.apache.spark.sql.errors.QueryContextSuite.$anonfun$new$3(QueryContextSuite.scala:30)
+ |""".stripMargin)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
