This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 594e5701a49 HIVE-24419: Refactor junit database rules to exploit
testcontainers (#5928)
594e5701a49 is described below
commit 594e5701a49f308b235d4483d760089dc68d14fb
Author: Stamatis Zampetakis <[email protected]>
AuthorDate: Tue Jul 22 10:33:27 2025 +0300
HIVE-24419: Refactor junit database rules to exploit testcontainers (#5928)
---
itests/hive-unit/pom.xml | 2 -
itests/util/pom.xml | 20 +++
.../apache/hadoop/hive/cli/control/CliAdapter.java | 5 +-
.../hadoop/hive/ql/QTestMetaStoreHandler.java | 32 +---
pom.xml | 31 ++++
ql/pom.xml | 25 +++
.../hadoop/hive/ql/lockmgr/ITestDbTxnManager.java | 32 +---
standalone-metastore/metastore-server/pom.xml | 25 +++
.../metastore/dbinstall/rules/DatabaseRule.java | 180 +--------------------
.../hive/metastore/dbinstall/rules/Derby.java | 19 +--
.../hive/metastore/dbinstall/rules/Mariadb.java | 34 ++--
.../dbinstall/rules/MetastoreRuleFactory.java | 48 ++++++
.../hive/metastore/dbinstall/rules/Mssql.java | 46 +++---
.../hive/metastore/dbinstall/rules/Mysql.java | 36 ++---
.../hive/metastore/dbinstall/rules/Oracle.java | 45 +++---
.../hive/metastore/dbinstall/rules/Postgres.java | 43 +++--
.../metastore/dbinstall/rules/PostgresTPCDS.java | 11 +-
.../schematool/TestSchemaToolForMetastore.java | 25 +--
standalone-metastore/pom.xml | 26 +++
19 files changed, 304 insertions(+), 381 deletions(-)
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index b66b4f266d8..66a109a5cf4 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -25,7 +25,6 @@
<name>Hive Integration - Unit Tests</name>
<properties>
<hive.path.to.root>../..</hive.path.to.root>
- <testcontainers.version>1.15.2</testcontainers.version>
<htmlunit.version>2.70.0</htmlunit.version>
</properties>
<dependencies>
@@ -500,7 +499,6 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
- <version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/itests/util/pom.xml b/itests/util/pom.xml
index 63f24d22d55..2322e5ea63c 100644
--- a/itests/util/pom.xml
+++ b/itests/util/pom.xml
@@ -193,6 +193,26 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mariadb</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mysql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>oracle-xe</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mssqlserver</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java
index e85895c29a1..828a75db7e5 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java
@@ -72,11 +72,10 @@ public Statement apply(final Statement base, Description
description) {
return new Statement() {
@Override
public void evaluate() throws Throwable {
- metaStoreHandler.setSystemProperties(); // for QTestUtil
pre-initialization
- CliAdapter.this.beforeClass(); // instantiating QTestUtil
-
metaStoreHandler.getRule().before();
metaStoreHandler.getRule().install();
+ metaStoreHandler.setSystemProperties(); // for QTestUtil
pre-initialization
+ CliAdapter.this.beforeClass(); // instantiating QTestUtil
if (getQt() != null) {
metaStoreHandler.setMetaStoreConfiguration(getQt().getConf());
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java
index f6054d97cbc..48a66b35718 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java
@@ -21,12 +21,7 @@
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.PostgresTPCDS;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.MetastoreRuleFactory;
import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +41,7 @@ public class QTestMetaStoreHandler {
public QTestMetaStoreHandler(String metastore) {
this.metastoreType = Objects.requireNonNull(metastore);
- this.rule = getDatabaseRule(metastoreType).setVerbose(false);
+ this.rule = MetastoreRuleFactory.create(metastoreType).setVerbose(false);
LOG.info(String.format("initialized metastore type '%s' for qtests",
metastoreType));
}
@@ -75,24 +70,6 @@ public QTestMetaStoreHandler
setMetaStoreConfiguration(HiveConf conf) {
return this;
}
- private DatabaseRule getDatabaseRule(String metastoreType) {
- switch (metastoreType) {
- case "postgres":
- return new Postgres();
- case "postgres.tpcds":
- return new PostgresTPCDS();
- case "oracle":
- return new Oracle();
- case "mysql":
- return new Mysql();
- case "mssql":
- case "sqlserver":
- return new Mssql();
- default:
- return new Derby();
- }
- }
-
private String getDbTypeConfString() {// "ORACLE", "MYSQL", "MSSQL",
"POSTGRES"
return "sqlserver".equalsIgnoreCase(metastoreType) ? "MSSQL" :
metastoreType.toUpperCase();
}
@@ -105,6 +82,11 @@ public void truncateDatabase(QTestUtil qt) throws Exception
{
}
}
+ /**
+ * Sets system properties for the metastore connection.
+ * The metastore database must be initialized before calling this method.
Some information, such as port
+ * number (in JDBC URL), is not available until the rule is initialized.
+ */
public void setSystemProperties() {
System.setProperty(MetastoreConf.ConfVars.CONNECT_URL_KEY.getVarname(),
rule.getJdbcUrl());
System.setProperty(MetastoreConf.ConfVars.CONNECTION_DRIVER.getVarname(),
rule.getJdbcDriver());
diff --git a/pom.xml b/pom.xml
index e20f1fce884..bce696397f7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -209,6 +209,7 @@
<tez.version>0.10.5</tez.version>
<super-csv.version>2.2.0</super-csv.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
+ <testcontainers.version>1.21.3</testcontainers.version>
<snappy.version>1.1.10.4</snappy.version>
<jersey-wadl-doclet.version>4.0.0-M2</jersey-wadl-doclet.version>
<jaxb-runtime.version>4.0.0-M2</jaxb-runtime.version>
@@ -1457,6 +1458,36 @@
<version>${postgres.version}</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mariadb</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mysql</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>oracle-xe</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mssqlserver</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
diff --git a/ql/pom.xml b/ql/pom.xml
index b4fea3627e9..9aba7488fca 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -574,6 +574,31 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mariadb</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mysql</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>oracle-xe</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mssqlserver</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- test inter-project -->
<dependency>
<groupId>org.apache.parquet</groupId>
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
index 17328b1281e..b3a805eaa29 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
@@ -21,12 +21,7 @@
import org.apache.hadoop.hive.metastore.DatabaseProduct;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mariadb;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.MetastoreRuleFactory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@@ -48,7 +43,10 @@ public static void setupDb() throws Exception {
String metastoreType =
System.getProperty(SYS_PROP_METASTORE_DB) == null ? "derby" :
System.getProperty(SYS_PROP_METASTORE_DB)
.toLowerCase();
- rule = getDatabaseRule(metastoreType).setVerbose(false);
+ rule = MetastoreRuleFactory.create(metastoreType).setVerbose(false);
+ // Start the docker container and create the hive user
+ rule.before();
+ rule.install();
conf.setVar(HiveConf.ConfVars.METASTORE_DB_TYPE,
metastoreType.toUpperCase());
@@ -62,30 +60,10 @@ public static void setupDb() throws Exception {
LOG.info("Set metastore connection to url: {}",
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY));
- // Start the docker container and create the hive user
- rule.before();
- rule.install();
}
@AfterClass
public static void tearDownDb() {
rule.after();
}
-
- private static DatabaseRule getDatabaseRule(String metastoreType) {
- switch (metastoreType) {
- case "postgres":
- return new Postgres();
- case "oracle":
- return new Oracle();
- case "mysql":
- return new Mysql();
- case "mariadb":
- return new Mariadb();
- case "mssql":
- return new Mssql();
- default:
- return new Derby();
- }
- }
}
diff --git a/standalone-metastore/metastore-server/pom.xml
b/standalone-metastore/metastore-server/pom.xml
index 2014f10cfdd..eb329c44100 100644
--- a/standalone-metastore/metastore-server/pom.xml
+++ b/standalone-metastore/metastore-server/pom.xml
@@ -435,6 +435,31 @@
<version>2.1.8</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mariadb</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mysql</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>postgresql</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>oracle-xe</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mssqlserver</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
index 23115ab916a..a5da05ed6fe 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
@@ -17,38 +17,20 @@
*/
package org.apache.hadoop.hive.metastore.dbinstall.rules;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.metastore.tools.schematool.MetastoreSchemaTool;
import org.junit.rules.ExternalResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Abstract JUnit TestRule for different RDMBS types.
*/
public abstract class DatabaseRule extends ExternalResource {
- private static final Logger LOG =
LoggerFactory.getLogger(DatabaseRule.class);
-
protected static final String HIVE_USER = "hiveuser";
// used in most of the RDBMS configs, except MSSQL
protected static final String HIVE_PASSWORD = "hivepassword";
protected static final String HIVE_DB = "hivedb";
- private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000;
public abstract String getHivePassword();
- public abstract String getDockerImageName();
-
- public abstract String[] getDockerAdditionalArgs();
-
public abstract String getDbType();
public abstract String getDbRootUser();
@@ -57,25 +39,13 @@ public abstract class DatabaseRule extends ExternalResource
{
public abstract String getJdbcDriver();
- public abstract String getJdbcUrl(String hostAddress);
-
- public final String getJdbcUrl() {
- return getJdbcUrl(getContainerHostAddress());
- }
-
- public final String getContainerHostAddress() {
- String hostAddress = System.getenv("HIVE_TEST_DOCKER_HOST");
- if (hostAddress != null) {
- return hostAddress;
- } else {
- return "localhost";
- }
- }
+ public abstract String getJdbcUrl();
private boolean verbose;
public DatabaseRule() {
verbose = System.getProperty("verbose.schematool") != null;
+ MetastoreSchemaTool.setHomeDirForTesting();
}
public DatabaseRule setVerbose(boolean verbose) {
@@ -92,151 +62,15 @@ public String getDb() {
*
* @return URL
*/
- public abstract String getInitialJdbcUrl(String hostAddress);
-
- public final String getInitialJdbcUrl() {
- return getInitialJdbcUrl(getContainerHostAddress());
- }
-
- /**
- * Determine if the docker container is ready to use.
- *
- * @param pr output of docker logs command
- * @return true if ready, false otherwise
- */
- public abstract boolean isContainerReady(ProcessResults pr);
-
- protected String[] buildArray(String... strs) {
- return strs;
- }
-
- public static class ProcessResults {
- final String stdout;
- final String stderr;
- final int rc;
-
- public ProcessResults(String stdout, String stderr, int rc) {
- this.stdout = stdout;
- this.stderr = stderr;
- this.rc = rc;
- }
- }
+ public abstract String getInitialJdbcUrl();
@Override
- public void before() throws Exception { //runDockerContainer
- runCmdAndPrintStreams(buildRmCmd(), 600);
- if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) {
- printDockerEvents();
- throw new RuntimeException("Unable to start docker container");
- }
- long startTime = System.currentTimeMillis();
- ProcessResults pr;
- do {
- Thread.sleep(1000);
- pr = runCmdAndPrintStreams(buildLogCmd(), 30);
- if (pr.rc != 0) {
- printDockerEvents();
- throw new RuntimeException("Failed to get docker logs");
- }
- } while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() &&
!isContainerReady(pr));
- if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) {
- printDockerEvents();
- throw new RuntimeException(
- String.format("Container initialization failed within %d seconds.
Please check the hive logs.",
- MAX_STARTUP_WAIT / 1000));
- }
- MetastoreSchemaTool.setHomeDirForTesting();
- }
-
+ public abstract void before() throws Exception;
@Override
- public void after() { // stopAndRmDockerContainer
- if
("true".equalsIgnoreCase(System.getProperty("metastore.itest.no.stop.container")))
{
- LOG.warn("Not stopping container " + getDockerContainerName() + " at
user request, please "
- + "be sure to shut it down before rerunning the test.");
- return;
- }
- try {
- if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) {
- throw new RuntimeException("Unable to remove docker container");
- }
- } catch (InterruptedException | IOException e) {
- e.printStackTrace();
- }
- }
-
- protected String getDockerContainerName() {
- String suffix = System.getenv("HIVE_TEST_DOCKER_CONTAINER_SUFFIX");
- if (suffix == null) {
- suffix = "";
- } else {
- suffix = "-" + suffix;
- }
- return String.format("metastore-test-%s-install%s", getDbType(), suffix);
- }
-
- private ProcessResults runCmd(String[] cmd, long secondsToWait)
- throws IOException, InterruptedException {
- LOG.info("Going to run: " + StringUtils.join(cmd, " "));
- Process proc = Runtime.getRuntime().exec(cmd);
- if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) {
- throw new RuntimeException("Process " + cmd[0] + " failed to run in " +
secondsToWait + " seconds");
- }
- BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
- final StringBuilder lines = new StringBuilder();
- reader.lines().forEach(s -> lines.append(s).append('\n'));
-
- reader = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
- final StringBuilder errLines = new StringBuilder();
- reader.lines().forEach(s -> errLines.append(s).append('\n'));
- LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(),
errLines.length());
- return new ProcessResults(lines.toString(), errLines.toString(),
proc.exitValue());
- }
+ public abstract void after();
- private ProcessResults runCmdAndPrintStreams(String[] cmd, long
secondsToWait)
- throws InterruptedException, IOException {
- ProcessResults results = runCmd(cmd, secondsToWait);
- LOG.info("Stdout from proc: " + results.stdout);
- LOG.info("Stderr from proc: " + results.stderr);
- return results;
- }
-
- protected void printDockerEvents() {
- try {
- runCmdAndPrintStreams(new String[] { "docker", "events", "--since",
"24h", "--until", "0s" }, 3);
- } catch (Exception e) {
- LOG.warn("A problem was encountered while attempting to retrieve Docker
events (the system made an analytical"
- + " best effort to list the events to reveal the root cause). No
further actions are necessary.", e);
- }
- }
-
- private String[] buildRunCmd() {
- List<String> cmd = new ArrayList<>(4 + getDockerAdditionalArgs().length);
- cmd.add("docker");
- cmd.add("run");
- cmd.add("--rm");
- cmd.add("--name");
- cmd.add(getDockerContainerName());
- cmd.addAll(Arrays.asList(getDockerAdditionalArgs()));
- cmd.add(getDockerImageName());
- return cmd.toArray(new String[cmd.size()]);
- }
-
- private String[] buildRmCmd() {
- return buildArray(
- "docker",
- "rm",
- "-f",
- "-v",
- getDockerContainerName()
- );
- }
-
- private String[] buildLogCmd() {
- return buildArray(
- "docker",
- "logs",
- getDockerContainerName()
- );
+ protected String[] buildArray(String... strs) {
+ return strs;
}
public String getHiveUser(){
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Derby.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Derby.java
index 07d9679f4d9..fdf59d87b78 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Derby.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Derby.java
@@ -36,16 +36,6 @@ public Derby(boolean purgeInAfter) {
this.purgeInAfter = purgeInAfter;
}
- @Override
- public String getDockerImageName() {
- return null;
- }
-
- @Override
- public String[] getDockerAdditionalArgs() {
- return null;
- }
-
@Override
public String getDbType() {
return "derby";
@@ -77,13 +67,13 @@ public String getJdbcDriver() {
}
@Override
- public String getJdbcUrl(String hostAddress) {
+ public String getJdbcUrl() {
return String.format("jdbc:derby:memory:%s;create=true", getDb());
}
@Override
- public String getInitialJdbcUrl(String hostAddress) {
+ public String getInitialJdbcUrl() {
return String.format("jdbc:derby:memory:%s;create=true", getDb());
}
@@ -91,11 +81,6 @@ public String getDb() {
return MetaStoreServerUtils.JUNIT_DATABASE_PREFIX;
};
- @Override
- public boolean isContainerReady(ProcessResults pr) {
- return true;
- }
-
@Override
public void before() throws Exception {
MetastoreSchemaTool.setHomeDirForTesting();
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mariadb.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mariadb.java
index f32d6730a7b..b2f63789fd3 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mariadb.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mariadb.java
@@ -18,19 +18,22 @@
package org.apache.hadoop.hive.metastore.dbinstall.rules;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.IOException;
public class Mariadb extends DatabaseRule {
+ private final MariaDBContainer<?> container = new
MariaDBContainer<>(DockerImageName.parse("mariadb:11.4"));
@Override
- public String getDockerImageName() {
- return "mariadb:11.4";
+ public void before() throws IOException, InterruptedException {
+ container.start();
}
@Override
- public String[] getDockerAdditionalArgs() {
- return buildArray("-p", "3306:3306", "-e", "MYSQL_ROOT_PASSWORD=" +
getDbRootPassword(), "-d");
+ public void after() {
+ container.stop();
}
@Override
@@ -45,29 +48,22 @@ public String getDbRootUser() {
@Override
public String getDbRootPassword() {
- return "its-a-secret";
+ return container.getPassword();
}
@Override
public String getJdbcDriver() {
- return "org.mariadb.jdbc.Driver";
- }
-
- @Override
- public String getJdbcUrl(String hostAddress) {
- return "jdbc:mariadb://" + hostAddress + ":3306/" + HIVE_DB;
+ return container.getDriverClassName();
}
@Override
- public String getInitialJdbcUrl(String hostAddress) {
- return "jdbc:mariadb://" + hostAddress +
":3306/?allowPublicKeyRetrieval=true";
+ public String getJdbcUrl() {
+ return container.withDatabaseName(HIVE_DB).getJdbcUrl();
}
@Override
- public boolean isContainerReady(ProcessResults pr) {
- Pattern pat = Pattern.compile("ready for connections");
- Matcher m = pat.matcher(pr.stderr);
- return m.find() && m.find();
+ public String getInitialJdbcUrl() {
+ return container.withDatabaseName("").getJdbcUrl();
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/MetastoreRuleFactory.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/MetastoreRuleFactory.java
new file mode 100644
index 00000000000..c010847120a
--- /dev/null
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/MetastoreRuleFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.dbinstall.rules;
+
+/**
+ * A factory for creating a Metastore database rule for use in tests.
+ */
+public final class MetastoreRuleFactory {
+ private MetastoreRuleFactory() {
+ throw new IllegalStateException("Factory class should not be
instantiated");
+ }
+
+ /**
+ * Creates a new Metastore rule based on the provided database type.
+ *
+ * @param dbType the type of database (e.g., "mysql", "postgres", etc.)
+ * @return a DatabaseRule instance for the specified database type
+ * @throws IllegalArgumentException if the database type is unsupported
+ */
+ public static DatabaseRule create(String dbType) {
+ return switch (dbType.toLowerCase()) {
+ case "mysql" -> new Mysql();
+ case "postgres" -> new Postgres();
+ case "postgres.tpcds" -> new PostgresTPCDS();
+ case "mariadb" -> new Mariadb();
+ case "derby" -> new Derby();
+ case "derby.clean" -> new Derby(true);
+ case "mssql", "sqlserver" -> new Mssql();
+ case "oracle" -> new Oracle();
+ default -> throw new IllegalArgumentException("Unsupported database
type: " + dbType);
+ };
+ }
+}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mssql.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mssql.java
index 3eafaa637d6..3a9c2b30295 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mssql.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mssql.java
@@ -17,27 +17,27 @@
*/
package org.apache.hadoop.hive.metastore.dbinstall.rules;
+import org.testcontainers.containers.JdbcDatabaseContainer;
+import org.testcontainers.containers.MSSQLServerContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.IOException;
+
/**
* JUnit TestRule for Mssql.
*/
public class Mssql extends DatabaseRule {
+ private final JdbcDatabaseContainer<?> container =
+ new
MSSQLServerContainer<>(DockerImageName.parse("mcr.microsoft.com/mssql/server:2019-latest")).acceptLicense();
@Override
- public String getDockerImageName() {
- return "mcr.microsoft.com/mssql/server:2019-latest";
+ public void before() throws IOException, InterruptedException {
+ container.start();
}
@Override
- public String[] getDockerAdditionalArgs() {
- return buildArray(
- "-p",
- "1433:1433",
- "-e",
- "ACCEPT_EULA=Y",
- "-e",
- "SA_PASSWORD=" + getDbRootPassword(),
- "-d"
- );
+ public void after() {
+ container.stop();
}
@Override
@@ -47,35 +47,27 @@ public String getDbType() {
@Override
public String getDbRootUser() {
- return "SA";
+ return container.getUsername();
}
@Override
public String getDbRootPassword() {
- return "Its-a-s3cret";
+ return container.getPassword();
}
@Override
public String getJdbcDriver() {
- return com.microsoft.sqlserver.jdbc.SQLServerDriver.class.getName();
- // return "com.microsoft.sqlserver.jdbc.SQLServerDriver";
- }
-
- @Override
- public String getJdbcUrl(String hostAddress) {
- return "jdbc:sqlserver://" + hostAddress + ":1433;DatabaseName=" + HIVE_DB
+ ";";
+ return container.getDriverClassName();
}
@Override
- public String getInitialJdbcUrl(String hostAddress) {
- return "jdbc:sqlserver://" + hostAddress + ":1433";
+ public String getJdbcUrl() {
+ return container.withUrlParam("DatabaseName", HIVE_DB).getJdbcUrl();
}
@Override
- public boolean isContainerReady(ProcessResults pr) {
- return pr.stdout
- .contains(
- "Recovery is complete. This is an informational message only. No user
action is required.");
+ public String getInitialJdbcUrl() {
+ return container.withUrlParam("DatabaseName", "master").getJdbcUrl();
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java
index b4dba4dbbac..4adbcc30588 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java
@@ -17,22 +17,25 @@
*/
package org.apache.hadoop.hive.metastore.dbinstall.rules;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.testcontainers.containers.MySQLContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.IOException;
/**
* JUnit TestRule for MySql.
*/
public class Mysql extends DatabaseRule {
-
+ private final MySQLContainer<?> container =
+ new
MySQLContainer<>(DockerImageName.parse("mysql:8.4.3")).withConfigurationOverride("");
@Override
- public String getDockerImageName() {
- return "mysql:8.4.3";
+ public void before() throws IOException, InterruptedException {
+ container.start();
}
@Override
- public String[] getDockerAdditionalArgs() {
- return buildArray("-p", "3306:3306", "-e", "MYSQL_ROOT_PASSWORD=" +
getDbRootPassword(), "-d");
+ public void after() {
+ container.stop();
}
@Override
@@ -47,29 +50,22 @@ public String getDbRootUser() {
@Override
public String getDbRootPassword() {
- return "its-a-secret";
+ return container.getPassword();
}
@Override
public String getJdbcDriver() {
- return "com.mysql.jdbc.Driver";
- }
-
- @Override
- public String getJdbcUrl(String hostAddress) {
- return "jdbc:mysql://" + hostAddress + ":3306/" + HIVE_DB;
+ return container.getDriverClassName();
}
@Override
- public String getInitialJdbcUrl(String hostAddress) {
- return "jdbc:mysql://" + hostAddress +
":3306/?allowPublicKeyRetrieval=true";
+ public String getJdbcUrl() {
+ return container.withDatabaseName(HIVE_DB).getJdbcUrl();
}
@Override
- public boolean isContainerReady(ProcessResults pr) {
- Pattern pat = Pattern.compile("mysqld.*ready for connections.*port.*3306");
- Matcher m = pat.matcher(pr.stderr);
- return m.find();
+ public String getInitialJdbcUrl() {
+ return
container.withDatabaseName("").withUrlParam("allowPublicKeyRetrieval",
"true").getJdbcUrl();
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Oracle.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Oracle.java
index d743bc79f28..a36612cc8ff 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Oracle.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Oracle.java
@@ -17,25 +17,26 @@
*/
package org.apache.hadoop.hive.metastore.dbinstall.rules;
+import org.testcontainers.containers.OracleContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.IOException;
+
/**
* JUnit TestRule for Oracle.
*/
public class Oracle extends DatabaseRule {
-
+ private final DockerImageName name =
+
DockerImageName.parse("abstractdog/oracle-xe:18.4.0-slim").asCompatibleSubstituteFor("gvenzl/oracle-xe");
+ private final OracleContainer container = new
OracleContainer(name).withEnv("ORACLE_PASSWORD", "oracle");
@Override
- public String getDockerImageName() {
- return "abstractdog/oracle-xe:18.4.0-slim";
+ public void before() throws IOException, InterruptedException {
+ container.start();
}
@Override
- public String[] getDockerAdditionalArgs() {
- return buildArray(
- "-p",
- "1521:1521",
- "-d",
- "-e",
- "ORACLE_PASSWORD=" + getDbRootPassword()
- );
+ public void after() {
+ container.stop();
}
@Override
@@ -50,36 +51,26 @@ public String getDbRootUser() {
@Override
public String getDbRootPassword() {
- return "oracle";
+ return container.getPassword();
}
@Override
public String getJdbcDriver() {
- return "oracle.jdbc.OracleDriver";
- }
-
- @Override
- public String getJdbcUrl(String hostAddress) {
- return "jdbc:oracle:thin:@//" + hostAddress + ":1521/xe";
+ return container.getDriverClassName();
}
@Override
- public String getInitialJdbcUrl(String hostAddress) {
- return "jdbc:oracle:thin:@//" + hostAddress + ":1521/xe";
+ public String getJdbcUrl() {
+ return container.getJdbcUrl();
}
@Override
- public boolean isContainerReady(ProcessResults pr) {
- return pr.stdout.contains("DATABASE IS READY TO USE!");
+ public String getInitialJdbcUrl() {
+ return container.getJdbcUrl();
}
@Override
public String getHivePassword() {
return HIVE_PASSWORD;
}
-
- @Override
- public String getHiveUser() {
- return "c##"+ super.getHiveUser();
- }
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Postgres.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Postgres.java
index eda1f972a2f..8844898ee70 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Postgres.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Postgres.java
@@ -17,18 +17,33 @@
*/
package org.apache.hadoop.hive.metastore.dbinstall.rules;
+import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.IOException;
+
/**
* JUnit TestRule for Postgres.
*/
public class Postgres extends DatabaseRule {
+ protected final PostgreSQLContainer<?> container;
+
+ public Postgres() {
+ this(DockerImageName.parse("postgres:11.4"));
+ }
+
+ protected Postgres(DockerImageName dockerImageName) {
+ this.container = new PostgreSQLContainer<>(dockerImageName);
+ }
+
@Override
- public String getDockerImageName() {
- return "postgres:11.6";
+ public void before() throws IOException, InterruptedException {
+ container.start();
}
@Override
- public String[] getDockerAdditionalArgs() {
- return buildArray("-p", "5432:5432", "-e", "POSTGRES_PASSWORD=" +
getDbRootPassword(), "-d");
+ public void after() {
+ container.stop();
}
@Override
@@ -38,33 +53,27 @@ public String getDbType() {
@Override
public String getDbRootUser() {
- return "postgres";
+ return container.getUsername();
}
@Override
public String getDbRootPassword() {
- return "its-a-secret";
+ return container.getPassword();
}
@Override
public String getJdbcDriver() {
- return org.postgresql.Driver.class.getName();
- }
-
- @Override
- public String getJdbcUrl(String hostAddress) {
- return "jdbc:postgresql://" + hostAddress + ":5432/" + HIVE_DB;
+ return container.getDriverClassName();
}
@Override
- public String getInitialJdbcUrl(String hostAddress) {
- return "jdbc:postgresql://" + hostAddress + ":5432/postgres";
+ public String getJdbcUrl() {
+ return container.withDatabaseName(HIVE_DB).getJdbcUrl();
}
@Override
- public boolean isContainerReady(ProcessResults pr) {
- return pr.stdout.contains("database system is ready to accept
connections") &&
- pr.stderr.contains("database system is ready to accept connections");
+ public String getInitialJdbcUrl() {
+ return container.withDatabaseName("postgres").getJdbcUrl();
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java
index c1bfb7a3eba..b0a692d82ca 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
import org.apache.hadoop.hive.metastore.tools.schematool.MetastoreSchemaTool;
+import org.testcontainers.utility.DockerImageName;
import java.io.IOException;
import java.io.InputStream;
@@ -29,14 +30,14 @@
* JUnit TestRule for Postgres metastore with TPCDS schema and stat
information.
*/
public class PostgresTPCDS extends Postgres {
- @Override
- public String getDockerImageName() {
- return "zabetak/postgres-tpcds-metastore:1.3";
+ public PostgresTPCDS() {
+
super(DockerImageName.parse("zabetak/postgres-tpcds-metastore:1.3").asCompatibleSubstituteFor("postgres"));
+ container.withUsername("postgres");
}
@Override
- public String getJdbcUrl(String hostAddress) {
- return "jdbc:postgresql://" + hostAddress + ":5432/metastore";
+ public String getJdbcUrl() {
+ return container.withDatabaseName("metastore").getJdbcUrl();
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
index 804bf156dc6..e302d24a95e 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
@@ -29,10 +29,9 @@
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
@@ -45,12 +44,7 @@
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mariadb;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle;
-import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.MetastoreRuleFactory;
import org.junit.After;
import org.junit.Assert;
@@ -75,20 +69,13 @@ public class TestSchemaToolForMetastore {
private PrintStream outStream;
private SchemaToolTaskValidate validator;
- public TestSchemaToolForMetastore(DatabaseRule dbms){
- this.dbms = dbms;
+ public TestSchemaToolForMetastore(String dbType){
+ this.dbms = MetastoreRuleFactory.create(dbType);
}
@Parameterized.Parameters(name = "{0}")
- public static Collection<Object[]> databases() {
- List<Object[]> dbs = new ArrayList<>();
- dbs.add(new Object[] { new Derby(true) });
- dbs.add(new Object[] { new Mysql() });
- dbs.add(new Object[] { new Oracle() });
- dbs.add(new Object[] { new Postgres() });
- dbs.add(new Object[] { new Mariadb() });
- dbs.add(new Object[] { new Mssql() });
- return dbs;
+ public static Collection<String> databases() {
+ return Arrays.asList("derby.clean", "mysql", "oracle", "postgres",
"mariadb", "mssql");
}
@Before
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 7c137624b5b..14baad61338 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -121,6 +121,7 @@
<!-- If upgrading, upgrade atlas as well in ql/pom.xml, which brings in
some springframework dependencies transitively -->
<spring.version>5.3.39</spring.version>
<spring.ldap.version>2.4.4</spring.ldap.version>
+ <testcontainers.version>1.21.3</testcontainers.version>
<!-- Thrift properties -->
<thrift.home>you-must-set-this-to-run-thrift</thrift.home>
<thrift.gen.dir>${basedir}/src/gen/thrift</thrift.gen.dir>
@@ -508,6 +509,31 @@
<version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mariadb</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mysql</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>oracle-xe</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mssqlserver</artifactId>
+ <version>${testcontainers.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
