This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new c0240ad [SPARK-37069][SQL] Properly fallback when Hive.getWithoutRegisterFns is not available c0240ad is described below commit c0240ad792f6a987cd4f774e5b01059da95bad1b Author: Chao Sun <sunc...@apple.com> AuthorDate: Fri Oct 22 13:31:22 2021 +0900 [SPARK-37069][SQL] Properly fallback when Hive.getWithoutRegisterFns is not available Properly fallback to `Hive.get` when `Hive.getWithoutRegisterFns` is unavailable to a Hive version. In SPARK-35321 we switched to use the new method `Hive.getWithoutRegisterFns` introduced by [HIVE-21563](https://issues.apache.org/jira/browse/HIVE-21563). The code path is supposed to only active for Hive versions that are >= 2.3.9. However, due to how `HiveVersion` is initialized in `IsolatedClientLoader`, if users set `spark.sql.hive.metastore.version` to `2.3.8`, Spark will still convert it to "2.3.9" and thus will subsequently fail with `NoSuchMethodError`. This fixes it by always fallback on `NoSuchMethodError`. By doing this we are also able to support other Hive versions with `Hive.getWithoutRegisterFns` implemented. I manually tested via launching a Spark session with custom Hive version: ``` $SPARK_HOME/bin/spark-shell --conf spark.sql.hive.metastore.version=2.3.8 --conf spark.sql.hive.metastore.jars="/tmp/apache-hive-2.3.8-bin/lib/* ``` And then tried this command: ``` import org.apache.spark.sql.SparkSession val spark = SparkSession.builder() .master("local[*]") .enableHiveSupport() .config("spark.sql.hive.metastore.version", "2.3.8") .config("spark.sql.hive.metastore.jars", "/tmp/apache-hive-2.3.8-bin/lib/*") .getOrCreate() spark.sql("show tables").show ``` The command is failing before this PR, but working afterwards. Closes #34360 from sunchao/SPARK-37069. Authored-by: Chao Sun <sunc...@apple.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> (cherry picked from commit 39a0c224581ae5f221522cee1adeea5d2da73a08) Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../apache/spark/sql/hive/client/HiveClientImpl.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index edb1c12..d4362fa 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -57,11 +57,11 @@ import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.SupportsNamespaces._ import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} import org.apache.spark.sql.execution.QueryExecutionException -import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} +import org.apache.spark.sql.hive.HiveExternalCatalog import org.apache.spark.sql.hive.HiveExternalCatalog.DATASOURCE_SCHEMA import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.spark.util.{CircularBuffer, Utils, VersionUtils} +import org.apache.spark.util.{CircularBuffer, Utils} /** * A class that wraps the HiveClient and converts its responses to externally visible classes. @@ -201,12 +201,13 @@ private[hive] class HiveClientImpl( } private def getHive(conf: HiveConf): Hive = { - VersionUtils.majorMinorPatchVersion(version.fullVersion).map { - case (2, 3, v) if v >= 9 => Hive.getWithoutRegisterFns(conf) - case _ => Hive.get(conf) - }.getOrElse { - throw QueryExecutionErrors.unsupportedHiveMetastoreVersionError( - version.fullVersion, HiveUtils.HIVE_METASTORE_VERSION.key) + try { + Hive.getWithoutRegisterFns(conf) + } catch { + // SPARK-37069: not all Hive versions have the above method (e.g., Hive 2.3.9 has it but + // 2.3.8 don't), therefore here we fallback when encountering the exception. + case _: NoSuchMethodError => + Hive.get(conf) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org