Repository: spark
Updated Branches:
refs/heads/branch-2.2 f6730a70c -> 5fdc7d80f
[SPARK-20924][SQL] Unable to call the function registered in the not-current
database
### What changes were proposed in this pull request?
We are unable to call the function registered in the not-current database.
```Scala
sql("CREATE DATABASE dAtABaSe1")
sql(s"CREATE FUNCTION dAtABaSe1.test_avg AS
'${classOf[GenericUDAFAverage].getName}'")
sql("SELECT dAtABaSe1.test_avg(1)")
```
The above code returns an error:
```
Undefined function: 'dAtABaSe1.test_avg'. This function is neither a registered
temporary function nor a permanent function registered in the database
'default'.; line 1 pos 7
```
This PR is to fix the above issue.
### How was this patch tested?
Added test cases.
Author: Xiao Li <gatorsm...@gmail.com>
Closes #18146 from gatorsmile/qualifiedFunction.
(cherry picked from commit 4bb6a53ebd06de3de97139a2dbc7c85fc3aa3e66)
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fdc7d80
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fdc7d80
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fdc7d80
Branch: refs/heads/branch-2.2
Commit: 5fdc7d80f46d51d4a8e49d9390b191fff42ec222
Parents: f6730a7
Author: Xiao Li <gatorsm...@gmail.com>
Authored: Tue May 30 14:06:19 2017 -0700
Committer: Wenchen Fan <wenc...@databricks.com>
Committed: Tue May 30 14:06:26 2017 -0700
----------------------------------------------------------------------
.../sql/catalyst/catalog/SessionCatalog.scala | 17 +++++++++--------
.../spark/sql/hive/HiveSessionCatalog.scala | 6 +++---
.../spark/sql/hive/execution/HiveUDFSuite.scala | 20 ++++++++++++++++++++
3 files changed, 32 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5fdc7d80/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index f6653d3..a78440d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -1105,8 +1105,9 @@ class SessionCatalog(
!hiveFunctions.contains(name.funcName.toLowerCase(Locale.ROOT))
}
- protected def failFunctionLookup(name: String): Nothing = {
- throw new NoSuchFunctionException(db = currentDb, func = name)
+ protected def failFunctionLookup(name: FunctionIdentifier): Nothing = {
+ throw new NoSuchFunctionException(
+ db = name.database.getOrElse(getCurrentDatabase), func = name.funcName)
}
/**
@@ -1128,7 +1129,7 @@ class SessionCatalog(
qualifiedName.database.orNull,
qualifiedName.identifier)
} else {
- failFunctionLookup(name.funcName)
+ failFunctionLookup(name)
}
}
}
@@ -1158,8 +1159,8 @@ class SessionCatalog(
}
// If the name itself is not qualified, add the current database to it.
- val database =
name.database.orElse(Some(currentDb)).map(formatDatabaseName)
- val qualifiedName = name.copy(database = database)
+ val database =
formatDatabaseName(name.database.getOrElse(getCurrentDatabase))
+ val qualifiedName = name.copy(database = Some(database))
if (functionRegistry.functionExists(qualifiedName.unquotedString)) {
// This function has been already loaded into the function registry.
@@ -1172,10 +1173,10 @@ class SessionCatalog(
// in the metastore). We need to first put the function in the
FunctionRegistry.
// TODO: why not just check whether the function exists first?
val catalogFunction = try {
- externalCatalog.getFunction(currentDb, name.funcName)
+ externalCatalog.getFunction(database, name.funcName)
} catch {
- case e: AnalysisException => failFunctionLookup(name.funcName)
- case e: NoSuchPermanentFunctionException =>
failFunctionLookup(name.funcName)
+ case _: AnalysisException => failFunctionLookup(name)
+ case _: NoSuchPermanentFunctionException => failFunctionLookup(name)
}
loadFunctionResources(catalogFunction.resources)
// Please note that qualifiedName is provided by the user. However,
http://git-wip-us.apache.org/repos/asf/spark/blob/5fdc7d80/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
index 377d4f2..6227e78 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
@@ -140,7 +140,7 @@ private[sql] class HiveSessionCatalog(
// Hive is case insensitive.
val functionName = funcName.unquotedString.toLowerCase(Locale.ROOT)
if (!hiveFunctions.contains(functionName)) {
- failFunctionLookup(funcName.unquotedString)
+ failFunctionLookup(funcName)
}
// TODO: Remove this fallback path once we implement the list of
fallback functions
@@ -148,12 +148,12 @@ private[sql] class HiveSessionCatalog(
val functionInfo = {
try {
Option(HiveFunctionRegistry.getFunctionInfo(functionName)).getOrElse(
- failFunctionLookup(funcName.unquotedString))
+ failFunctionLookup(funcName))
} catch {
// If HiveFunctionRegistry.getFunctionInfo throws an exception,
// we are failing to load a Hive builtin function, which means
that
// the given function is not a Hive builtin function.
- case NonFatal(e) => failFunctionLookup(funcName.unquotedString)
+ case NonFatal(e) => failFunctionLookup(funcName)
}
}
val className = functionInfo.getFunctionClass.getName
http://git-wip-us.apache.org/repos/asf/spark/blob/5fdc7d80/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 4446af2..8fcbad5 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.{AnalysisException, QueryTest,
Row}
import org.apache.spark.sql.catalyst.plans.logical.Project
import org.apache.spark.sql.functions.max
import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.util.Utils
@@ -590,6 +591,25 @@ class HiveUDFSuite extends QueryTest with
TestHiveSingleton with SQLTestUtils {
}
}
}
+
+ test("Call the function registered in the not-current database") {
+ Seq("true", "false").foreach { caseSensitive =>
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
+ withDatabase("dAtABaSe1") {
+ sql("CREATE DATABASE dAtABaSe1")
+ withUserDefinedFunction("dAtABaSe1.test_avg" -> false) {
+ sql(s"CREATE FUNCTION dAtABaSe1.test_avg AS
'${classOf[GenericUDAFAverage].getName}'")
+ checkAnswer(sql("SELECT dAtABaSe1.test_avg(1)"), Row(1.0))
+ }
+ val message = intercept[AnalysisException] {
+ sql("SELECT dAtABaSe1.unknownFunc(1)")
+ }.getMessage
+ assert(message.contains("Undefined function: 'unknownFunc'") &&
+ message.contains("nor a permanent function registered in the
database 'dAtABaSe1'"))
+ }
+ }
+ }
+ }
}
class TestPair(x: Int, y: Int) extends Writable with Serializable {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
