This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 017919b [SPARK-27383][SQL][TEST] Avoid using hard-coded jar names in
Hive tests
017919b is described below
commit 017919b636fd3ce43ccf5ec57f1c1842aa2130db
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Sat Apr 6 18:06:52 2019 -0500
[SPARK-27383][SQL][TEST] Avoid using hard-coded jar names in Hive tests
## What changes were proposed in this pull request?
This pr avoid using hard-coded jar names(`hive-contrib-0.13.1.jar` and
`hive-hcatalog-core-0.13.1.jar`) in Hive tests. This change makes it easy to
change when upgrading the built-in Hive to 2.3.4.
## How was this patch tested?
Existing test
Closes #24294 from wangyum/SPARK-27383.
Authored-by: Yuming Wang <yumw...@ebay.com>
Signed-off-by: Sean Owen <sean.o...@databricks.com>
---
.../main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 11 +++++++++++
.../org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala | 12 ++++++------
.../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 6 +++---
.../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 2 +-
4 files changed, 21 insertions(+), 10 deletions(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index c45f3e7..1515807 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -116,6 +116,9 @@ class TestHiveContext(
@transient override val sparkSession: TestHiveSparkSession)
extends SQLContext(sparkSession) {
+ val HIVE_CONTRIB_JAR: String = "hive-contrib-0.13.1.jar"
+ val HIVE_HCATALOG_CORE_JAR: String = "hive-hcatalog-core-0.13.1.jar"
+
/**
* If loadTestTables is false, no test tables are loaded. Note that this
flag can only be true
* when running in the JVM, i.e. it needs to be false when calling from
Python.
@@ -142,6 +145,14 @@ class TestHiveContext(
sparkSession.getHiveFile(path)
}
+ def getHiveContribJar(): File = {
+ sparkSession.getHiveFile(HIVE_CONTRIB_JAR)
+ }
+
+ def getHiveHcatalogCoreJar(): File = {
+ sparkSession.getHiveFile(HIVE_HCATALOG_CORE_JAR)
+ }
+
def loadTestTable(name: String): Unit = {
sparkSession.loadTestTable(name)
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 8a36148..3b64a6b 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -108,8 +108,8 @@ class HiveSparkSubmitSuite
val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
val jar1 = TestUtils.createJarWithClasses(Seq("SparkSubmitClassA"))
val jar2 = TestUtils.createJarWithClasses(Seq("SparkSubmitClassB"))
- val jar3 = TestHive.getHiveFile("hive-contrib-0.13.1.jar").getCanonicalPath
- val jar4 =
TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").getCanonicalPath
+ val jar3 = TestHive.getHiveContribJar().getCanonicalPath
+ val jar4 = TestHive.getHiveHcatalogCoreJar().getCanonicalPath
val jarsString = Seq(jar1, jar2, jar3, jar4).map(j =>
j.toString).mkString(",")
val args = Seq(
"--class", SparkSubmitClassLoaderTest.getClass.getName.stripSuffix("$"),
@@ -315,7 +315,7 @@ class HiveSparkSubmitSuite
"--master", "local-cluster[2,1,1024]",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
- "--jars",
TestHive.getHiveFile("hive-contrib-0.13.1.jar").getCanonicalPath,
+ "--jars", TestHive.getHiveContribJar().getCanonicalPath,
unusedJar.toString)
runSparkSubmit(argsForCreateTable)
@@ -457,7 +457,7 @@ object TemporaryHiveUDFTest extends Logging {
// Load a Hive UDF from the jar.
logInfo("Registering a temporary Hive UDF provided in a jar.")
- val jar =
hiveContext.getHiveFile("hive-contrib-0.13.1.jar").getCanonicalPath
+ val jar = hiveContext.getHiveContribJar().getCanonicalPath
hiveContext.sql(
s"""
|CREATE TEMPORARY FUNCTION example_max
@@ -495,7 +495,7 @@ object PermanentHiveUDFTest1 extends Logging {
// Load a Hive UDF from the jar.
logInfo("Registering a permanent Hive UDF provided in a jar.")
- val jar =
hiveContext.getHiveFile("hive-contrib-0.13.1.jar").getCanonicalPath
+ val jar = hiveContext.getHiveContribJar().getCanonicalPath
hiveContext.sql(
s"""
|CREATE FUNCTION example_max
@@ -532,7 +532,7 @@ object PermanentHiveUDFTest2 extends Logging {
val hiveContext = new TestHiveContext(sc)
// Load a Hive UDF from the jar.
logInfo("Write the metadata of a permanent Hive UDF into metastore.")
- val jar =
hiveContext.getHiveFile("hive-contrib-0.13.1.jar").getCanonicalPath
+ val jar = hiveContext.getHiveContribJar().getCanonicalPath
val function = CatalogFunction(
FunctionIdentifier("example_max"),
"org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax",
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index a5cff35..c35ff80 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -816,7 +816,7 @@ class HiveQuerySuite extends HiveComparisonTest with
SQLTestUtils with BeforeAnd
test("ADD JAR command 2") {
// this is a test case from mapjoin_addjar.q
- val testJar = TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").toURI
+ val testJar = TestHive.getHiveHcatalogCoreJar().toURI
val testData = TestHive.getHiveFile("data/files/sample.json").toURI
sql(s"ADD JAR $testJar")
sql(
@@ -826,9 +826,9 @@ class HiveQuerySuite extends HiveComparisonTest with
SQLTestUtils with BeforeAnd
sql("select * from src join t1 on src.key = t1.a")
sql("DROP TABLE t1")
assert(sql("list jars").
- filter(_.getString(0).contains("hive-hcatalog-core-0.13.1.jar")).count()
> 0)
+ filter(_.getString(0).contains(TestHive.HIVE_HCATALOG_CORE_JAR)).count()
> 0)
assert(sql("list jar").
- filter(_.getString(0).contains("hive-hcatalog-core-0.13.1.jar")).count()
> 0)
+ filter(_.getString(0).contains(TestHive.HIVE_HCATALOG_CORE_JAR)).count()
> 0)
val testJar2 = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath
sql(s"ADD JAR $testJar2")
assert(sql(s"list jar $testJar").count() == 1)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index aad34a3..29de55f 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -1105,7 +1105,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
override def run() {
// To make sure this test works, this jar should not be loaded in
another place.
sql(
- s"ADD JAR
${hiveContext.getHiveFile("hive-contrib-0.13.1.jar").getCanonicalPath()}")
+ s"ADD JAR ${hiveContext.getHiveContribJar().getCanonicalPath}")
try {
sql(
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
