[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user asfgit closed the pull request at: https://github.com/apache/spark/pull/19148 --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user cloud-fan commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137706271 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) +Utils.deleteRecursively(tmpDataDir) +super.afterAll() + } + + private def downloadSpark(version: String): Unit = { +import scala.sys.process._ + +val url = s"https://d3kbcqa49mib13.cloudfront.net/spark-$version-bin-hadoop2.7.tgz"; + +Seq("wget", url, "-q", "-P", sparkTestingDir).! + +val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath +val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath + +Seq("mkdir", targetDir).! + +Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").! + +Seq("rm", downloaded).! + } + + private def genDataDir(name: String): String = { +new File(tmpDataDir, name).getCanonicalPath + } + + override def beforeAll(): Unit = { +super.beforeAll() + +val tempPyFile = File.createTempFile("test", ".py") +Files.write(tempPyFile.toPath, + s""" +|from pyspark.sql import SparkSession +| +|spark = SparkSession.builder.enableHiveSupport().getOrCreate() +|version_index = spark.conf.get("spark.sql.test.version.index", None) +| +|spark.sql("create table data_source_tbl_{} using json as select 1 i".format(version_index)) +| +|spark.sql("create table hive_compatible_data_source_tbl_" + version_index + \\ +| " using parquet as select 1 i") +| +|json_file = "${genDataDir("json_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file) +|spark.sql("create table external_data_source_tbl_" + version_index + \\ +| "(i int) using json options (path '{}')".format(json_file)) +| +|parquet_file = "${genDataDir("parquet_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.parquet(parquet_file) +|spark.sql("create table hive_compatible_external_data_source_tbl_" + version_index + \\ +| "(i int) using parquet options (path '{}')".format(parquet_file)) +| +|json_file2 = "${genDataDir("json2_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file2) +|spark.sql("create table external_tab
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137704899 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) +Utils.deleteRecursively(tmpDataDir) +super.afterAll() + } + + private def downloadSpark(version: String): Unit = { +import scala.sys.process._ + +val url = s"https://d3kbcqa49mib13.cloudfront.net/spark-$version-bin-hadoop2.7.tgz"; + +Seq("wget", url, "-q", "-P", sparkTestingDir).! + +val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath +val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath + +Seq("mkdir", targetDir).! + +Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").! + +Seq("rm", downloaded).! + } + + private def genDataDir(name: String): String = { +new File(tmpDataDir, name).getCanonicalPath + } + + override def beforeAll(): Unit = { +super.beforeAll() + +val tempPyFile = File.createTempFile("test", ".py") +Files.write(tempPyFile.toPath, + s""" +|from pyspark.sql import SparkSession +| +|spark = SparkSession.builder.enableHiveSupport().getOrCreate() +|version_index = spark.conf.get("spark.sql.test.version.index", None) +| +|spark.sql("create table data_source_tbl_{} using json as select 1 i".format(version_index)) +| +|spark.sql("create table hive_compatible_data_source_tbl_" + version_index + \\ +| " using parquet as select 1 i") +| +|json_file = "${genDataDir("json_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file) +|spark.sql("create table external_data_source_tbl_" + version_index + \\ +| "(i int) using json options (path '{}')".format(json_file)) +| +|parquet_file = "${genDataDir("parquet_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.parquet(parquet_file) +|spark.sql("create table hive_compatible_external_data_source_tbl_" + version_index + \\ +| "(i int) using parquet options (path '{}')".format(parquet_file)) +| +|json_file2 = "${genDataDir("json2_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file2) +|spark.sql("create table external_table_
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137704429 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) +Utils.deleteRecursively(tmpDataDir) +super.afterAll() + } + + private def downloadSpark(version: String): Unit = { +import scala.sys.process._ + +val url = s"https://d3kbcqa49mib13.cloudfront.net/spark-$version-bin-hadoop2.7.tgz"; + +Seq("wget", url, "-q", "-P", sparkTestingDir).! + +val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath +val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath + +Seq("mkdir", targetDir).! + +Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").! + +Seq("rm", downloaded).! + } + + private def genDataDir(name: String): String = { +new File(tmpDataDir, name).getCanonicalPath + } + + override def beforeAll(): Unit = { +super.beforeAll() + +val tempPyFile = File.createTempFile("test", ".py") +Files.write(tempPyFile.toPath, + s""" +|from pyspark.sql import SparkSession +| +|spark = SparkSession.builder.enableHiveSupport().getOrCreate() +|version_index = spark.conf.get("spark.sql.test.version.index", None) +| +|spark.sql("create table data_source_tbl_{} using json as select 1 i".format(version_index)) --- End diff -- Instead of only using lowercase column name, should we use mix-case Hive schema for those tables? --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137703092 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { --- End diff -- Ok. After a build clean it works now. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137700913 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { --- End diff -- Let me do build clean and try again. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user cloud-fan commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137700499 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { --- End diff -- Did you try a clean clone? I added the derby dependency to make the test work on jenkins... --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137699853 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { --- End diff -- After removing the added derby dependency, this test can work. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user dongjoon-hyun commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137699802 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/SparkSubmitTestUtils.scala --- @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.sql.Timestamp +import java.util.Date + +import scala.collection.mutable.ArrayBuffer + +import org.scalatest.concurrent.Timeouts +import org.scalatest.exceptions.TestFailedDueToTimeoutException +import org.scalatest.time.SpanSugar._ + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer +import org.apache.spark.util.Utils + +trait SparkSubmitTestUtils extends SparkFunSuite with Timeouts { --- End diff -- nit. Let's use `TimeLimits` instead of `Timeouts`. `Timeouts` is deprecated now. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user cloud-fan commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137699720 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { --- End diff -- can you print `org.apache.derby.tools.sysinfo.getVersionString` in `IsolatedClientLoader.createClient` to see what's your actual derby version? --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137699367 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { --- End diff -- I ran this test locally and encountered the failure like: 2017-09-07 19:28:07.595 - stderr> Caused by: java.sql.SQLException: Database at /root/repos/spark-1/target/tmp/warehouse-66dad501-c743-4ac3-83cc-51451c6d697a/metastore_db has an incompatible format with the current version of the software. The database was created by or upgraded by version 10.12. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user cloud-fan commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137686311 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) --- End diff -- I wanna keep the `sparkTestingDir`, so we don't need to download spark again if this jenkins machine has already run this suite before. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user gatorsmile commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137683665 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) +super.afterAll() + } + + private def downloadSpark(version: String): Unit = { +import scala.sys.process._ + +val url = s"https://d3kbcqa49mib13.cloudfront.net/spark-$version-bin-hadoop2.7.tgz"; + +Seq("wget", url, "-q", "-P", sparkTestingDir).! + +val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath +val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath + +Seq("mkdir", targetDir).! + +Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").! + +Seq("rm", downloaded).! + } + + private def genDataDir(name: String): String = { +new File(tmpDataDir, name).getCanonicalPath + } + + override def beforeAll(): Unit = { +super.beforeAll() + +val tempPyFile = File.createTempFile("test", ".py") +Files.write(tempPyFile.toPath, + s""" +|from pyspark.sql import SparkSession +| +|spark = SparkSession.builder.enableHiveSupport().getOrCreate() +|version_index = spark.conf.get("spark.sql.test.version.index", None) +| +|spark.sql("create table data_source_tbl_{} using json as select 1 i".format(version_index)) +| +|spark.sql("create table hive_compatible_data_source_tbl_" + version_index + \\ +| " using parquet as select 1 i") +| +|json_file = "${genDataDir("json_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file) +|spark.sql("create table external_data_source_tbl_" + version_index + \\ +| "(i int) using json options (path '{}')".format(json_file)) +| +|parquet_file = "${genDataDir("parquet_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.parquet(parquet_file) +|spark.sql("create table hive_compatible_external_data_source_tbl_" + version_index + \\ +| "(i int) using parquet options (path '{}')".format(parquet_file)) +| +|json_file2 = "${genDataDir("json2_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file2) +|spark.sql("create table external_table_without_schema_" + version_index + \\
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user gatorsmile commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137682740 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) +super.afterAll() + } + + private def downloadSpark(version: String): Unit = { +import scala.sys.process._ + +val url = s"https://d3kbcqa49mib13.cloudfront.net/spark-$version-bin-hadoop2.7.tgz"; + +Seq("wget", url, "-q", "-P", sparkTestingDir).! + +val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath +val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath + +Seq("mkdir", targetDir).! + +Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").! + +Seq("rm", downloaded).! + } + + private def genDataDir(name: String): String = { +new File(tmpDataDir, name).getCanonicalPath + } + + override def beforeAll(): Unit = { +super.beforeAll() + +val tempPyFile = File.createTempFile("test", ".py") +Files.write(tempPyFile.toPath, + s""" +|from pyspark.sql import SparkSession +| +|spark = SparkSession.builder.enableHiveSupport().getOrCreate() +|version_index = spark.conf.get("spark.sql.test.version.index", None) +| +|spark.sql("create table data_source_tbl_{} using json as select 1 i".format(version_index)) +| +|spark.sql("create table hive_compatible_data_source_tbl_" + version_index + \\ +| " using parquet as select 1 i") +| +|json_file = "${genDataDir("json_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file) +|spark.sql("create table external_data_source_tbl_" + version_index + \\ +| "(i int) using json options (path '{}')".format(json_file)) +| +|parquet_file = "${genDataDir("parquet_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.parquet(parquet_file) +|spark.sql("create table hive_compatible_external_data_source_tbl_" + version_index + \\ +| "(i int) using parquet options (path '{}')".format(parquet_file)) +| +|json_file2 = "${genDataDir("json2_")}" + str(version_index) +|spark.range(1, 2).selectExpr("cast(id as int) as i").write.json(json_file2) +|spark.sql("create table external_table_without_schema_" + version_index + \\
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user gatorsmile commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137682115 --- Diff: sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala --- @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import java.io.File +import java.nio.file.Files + +import org.apache.spark.TestUtils +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.Utils + +/** + * Test HiveExternalCatalog backward compatibility. + * + * Note that, this test suite will automatically download spark binary packages of different + * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with + * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the + * downloading for this spark version. + */ +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") + private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") + private val sparkTestingDir = "/tmp/spark-test" + private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + override def afterAll(): Unit = { +Utils.deleteRecursively(wareHousePath) --- End diff -- Also delete `tmpDataDir ` and `sparkTestingDir `? --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #19148: [SPARK-21936][SQL] backward compatibility test fr...
Github user dongjoon-hyun commented on a diff in the pull request: https://github.com/apache/spark/pull/19148#discussion_r137663570 --- Diff: sql/hive/pom.xml --- @@ -177,6 +177,10 @@ libfb303 + org.apache.derby + derby --- End diff -- I see. Thank you! --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org