This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 12d6ecc8f702 [SPARK-39174][SQL] Catalogs loading swallows missing classname for ClassNotFoundException 12d6ecc8f702 is described below commit 12d6ecc8f7021a4ed59812eaca16f601ce1ef10c Author: Kent Yao <y...@apache.org> AuthorDate: Sat May 14 19:45:24 2022 +0900 [SPARK-39174][SQL] Catalogs loading swallows missing classname for ClassNotFoundException ### What changes were proposed in this pull request? this PR captures the actual missing classname when catalog loading meets ClassNotFoundException ### Why are the changes needed? ClassNotFoundException can occur when missing dependencies, we shall not always report the catalog class is missing ### Does this PR introduce _any_ user-facing change? yes, when loading catalogs and ClassNotFoundException occurs, it shows the correct missing class. ### How was this patch tested? new test added Closes #36534 from yaooqinn/SPARK-39174. Authored-by: Kent Yao <y...@apache.org> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> (cherry picked from commit 1b37f19876298e995596a30edc322c856ea1bbb4) Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../spark/sql/connector/catalog/Catalogs.scala | 5 ++-- .../spark/sql/errors/QueryExecutionErrors.scala | 5 ++-- .../sql/connector/catalog/CatalogLoadingSuite.java | 28 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala index 9949f45d4833..71b1042ab306 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala @@ -60,8 +60,9 @@ private[sql] object Catalogs { plugin.initialize(name, catalogOptions(name, conf)) plugin } catch { - case _: ClassNotFoundException => - throw QueryExecutionErrors.catalogPluginClassNotFoundForCatalogError(name, pluginClassName) + case e: ClassNotFoundException => + throw QueryExecutionErrors.catalogPluginClassNotFoundForCatalogError( + name, pluginClassName, e) case e: NoSuchMethodException => throw QueryExecutionErrors.catalogFailToFindPublicNoArgConstructorError( name, pluginClassName, e) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index d4fbd38019b0..11dc57cb7813 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1450,8 +1450,9 @@ object QueryExecutionErrors { def catalogPluginClassNotFoundForCatalogError( name: String, - pluginClassName: String): Throwable = { - new SparkException(s"Cannot find catalog plugin class for catalog '$name': $pluginClassName") + pluginClassName: String, + e: Exception): Throwable = { + new SparkException(s"Cannot find catalog plugin class for catalog '$name': $pluginClassName", e) } def catalogFailToFindPublicNoArgConstructorError( diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java index 37f60511cd60..81870508b70a 100644 --- a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java +++ b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java @@ -20,6 +20,8 @@ package org.apache.spark.sql.connector.catalog; import org.apache.spark.SparkException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import org.apache.spark.util.Utils; + import org.junit.Assert; import org.junit.Test; @@ -91,6 +93,19 @@ public class CatalogLoadingSuite { exc.getMessage().contains("com.example.NoSuchCatalogPlugin")); } + @Test + public void testLoadMissingDependentClasses() { + SQLConf conf = new SQLConf(); + String catalogClass = ClassFoundCatalogPlugin.class.getCanonicalName(); + conf.setConfString("spark.sql.catalog.missing", catalogClass); + + SparkException exc = + Assert.assertThrows(SparkException.class, () -> Catalogs.load("missing", conf)); + + Assert.assertTrue(exc.getCause() instanceof ClassNotFoundException); + Assert.assertTrue(exc.getCause().getMessage().contains(catalogClass + "Dep")); + } + @Test public void testLoadNonCatalogPlugin() { SQLConf conf = new SQLConf(); @@ -209,3 +224,16 @@ class InvalidCatalogPlugin { // doesn't implement CatalogPlugin public void initialize(CaseInsensitiveStringMap options) { } } + +class ClassFoundCatalogPlugin implements CatalogPlugin { + + @Override + public void initialize(String name, CaseInsensitiveStringMap options) { + Utils.classForName(this.getClass().getCanonicalName() + "Dep", true, true); + } + + @Override + public String name() { + return null; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org