This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 2dbfae8 [SPARK-32049][SQL][TESTS] Upgrade Oracle JDBC Driver 8 2dbfae8 is described below commit 2dbfae8775e00da521f59c6328428ae541396802 Author: Gabor Somogyi <gabor.g.somo...@gmail.com> AuthorDate: Tue Jun 23 03:58:40 2020 -0700 [SPARK-32049][SQL][TESTS] Upgrade Oracle JDBC Driver 8 ### What changes were proposed in this pull request? `OracleIntegrationSuite` is not using the latest oracle JDBC driver. In this PR I've upgraded the driver to the latest which supports JDK8, JDK9, and JDK11. ### Why are the changes needed? Old JDBC driver. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing unit tests. Existing integration tests (especially `OracleIntegrationSuite`) Closes #28893 from gaborgsomogyi/SPARK-32049. Authored-by: Gabor Somogyi <gabor.g.somo...@gmail.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- external/docker-integration-tests/pom.xml | 12 ++------ .../sql/jdbc/DockerJDBCIntegrationSuite.scala | 3 +- .../spark/sql/jdbc/OracleIntegrationSuite.scala | 36 +++++++++++++--------- pom.xml | 6 ++++ 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 298e3d3..b240dd2 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -130,15 +130,9 @@ <artifactId>postgresql</artifactId> <scope>test</scope> </dependency> - <!-- Oracle ojdbc jar, used for oracle integration suite for docker testing. - See https://github.com/apache/spark/pull/11306 for background on why we need - to use a an ojdbc jar for the testcase. The maven dependency here is commented - because currently the maven repository does not contain the ojdbc jar mentioned. - Once the jar is available in maven, this could be uncommented. --> - <dependency> - <groupId>com.oracle</groupId> - <artifactId>ojdbc6</artifactId> - <version>11.2.0.1.0</version> + <dependency> + <groupId>com.oracle.database.jdbc</groupId> + <artifactId>ojdbc8</artifactId> <scope>test</scope> </dependency> diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala index d15b366..6d1a22d 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala @@ -95,6 +95,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu protected val dockerIp = DockerUtils.getDockerIp() val db: DatabaseOnDocker + val connectionTimeout = timeout(2.minutes) private var docker: DockerClient = _ protected var externalPort: Int = _ @@ -155,7 +156,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu docker.startContainer(containerId) jdbcUrl = db.getJdbcUrl(dockerIp, externalPort) var conn: Connection = null - eventually(timeout(2.minutes), interval(1.second)) { + eventually(connectionTimeout, interval(1.second)) { conn = getConnection() } // Run any setup queries: diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 24c3adb..9c59023 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala @@ -21,6 +21,8 @@ import java.math.BigDecimal import java.sql.{Connection, Date, Timestamp} import java.util.{Properties, TimeZone} +import org.scalatest.time.SpanSugar._ + import org.apache.spark.sql.{Row, SaveMode} import org.apache.spark.sql.execution.{RowDataSourceScanExec, WholeStageCodegenExec} import org.apache.spark.sql.execution.datasources.LogicalRelation @@ -31,27 +33,27 @@ import org.apache.spark.sql.types._ import org.apache.spark.tags.DockerTest /** - * This patch was tested using the Oracle docker. Created this integration suite for the same. - * The ojdbc6-11.2.0.2.0.jar was to be downloaded from the maven repository. Since there was - * no jdbc jar available in the maven repository, the jar was downloaded from oracle site - * manually and installed in the local; thus tested. So, for SparkQA test case run, the - * ojdbc jar might be manually placed in the local maven repository(com/oracle/ojdbc6/11.2.0.2.0) - * while Spark QA test run. - * * The following would be the steps to test this * 1. Build Oracle database in Docker, please refer below link about how to. * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md * 2. export ORACLE_DOCKER_IMAGE_NAME=$ORACLE_DOCKER_IMAGE_NAME * Pull oracle $ORACLE_DOCKER_IMAGE_NAME image - docker pull $ORACLE_DOCKER_IMAGE_NAME * 3. Start docker - sudo service docker start - * 4. Download oracle 11g driver jar and put it in maven local repo: - * (com/oracle/ojdbc6/11.2.0.2.0/ojdbc6-11.2.0.2.0.jar) - * 5. The timeout and interval parameter to be increased from 60,1 to a high value for oracle test - * in DockerJDBCIntegrationSuite.scala (Locally tested with 200,200 and executed successfully). - * 6. Run spark test - ./build/sbt "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite" + * 4. Run spark test - ./build/sbt -Pdocker-integration-tests + * "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite" + * + * An actual sequence of commands to run the test is as follows * - * All tests in this suite are ignored because of the dependency with the oracle jar from maven - * repository. + * $ git clone https://github.com/oracle/docker-images.git + * // Head SHA: 3e352a22618070595f823977a0fd1a3a8071a83c + * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles + * $ ./buildDockerImage.sh -v 18.4.0 -x + * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:18.4.0-xe + * $ cd $SPARK_HOME + * $ ./build/sbt -Pdocker-integration-tests + * "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite" + * + * It has been validated with 18.4.0 Express Edition. */ @DockerTest class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSparkSession { @@ -60,7 +62,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark override val db = new DatabaseOnDocker { override val imageName = sys.env("ORACLE_DOCKER_IMAGE_NAME") override val env = Map( - "ORACLE_ROOT_PASSWORD" -> "oracle" + "ORACLE_PWD" -> "oracle" ) override val usesIpc = false override val jdbcPort: Int = 1521 @@ -68,7 +70,11 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark s"jdbc:oracle:thin:system/oracle@//$ip:$port/xe" } + override val connectionTimeout = timeout(7.minutes) + override def dataPreparation(conn: Connection): Unit = { + // In 18.4.0 Express Edition auto commit is enabled by default. + conn.setAutoCommit(false) conn.prepareStatement("CREATE TABLE datetime (id NUMBER(10), d DATE, t TIMESTAMP)") .executeUpdate() conn.prepareStatement( diff --git a/pom.xml b/pom.xml index 7b68465..82c12ae 100644 --- a/pom.xml +++ b/pom.xml @@ -985,6 +985,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.oracle.database.jdbc</groupId> + <artifactId>ojdbc8</artifactId> + <version>19.6.0.0</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> <version>${curator.version}</version> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org