This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f7002a193b [test][hive] Upgrade hiverunner usage to support tests in
jdk11 (#5220)
f7002a193b is described below
commit f7002a193b5a91e6f62de369148dd25205c67afb
Author: Yubin Li <[email protected]>
AuthorDate: Thu Mar 6 11:55:52 2025 +0800
[test][hive] Upgrade hiverunner usage to support tests in jdk11 (#5220)
---
paimon-hive/paimon-hive-connector-2.3/pom.xml | 2 +-
paimon-hive/paimon-hive-connector-3.1/pom.xml | 7 +-
paimon-hive/paimon-hive-connector-common/pom.xml | 93 +++++++++++++++++++++-
.../org/apache/paimon/hive/HiveLocationTest.java | 6 +-
.../hive/runner/PaimonEmbeddedHiveRunner.java | 11 ++-
.../runner/PaimonEmbeddedHiveServerContext.java | 27 ++++---
paimon-hive/pom.xml | 3 +-
7 files changed, 125 insertions(+), 24 deletions(-)
diff --git a/paimon-hive/paimon-hive-connector-2.3/pom.xml
b/paimon-hive/paimon-hive-connector-2.3/pom.xml
index c0660fc573..5a7ce1d013 100644
--- a/paimon-hive/paimon-hive-connector-2.3/pom.xml
+++ b/paimon-hive/paimon-hive-connector-2.3/pom.xml
@@ -156,7 +156,7 @@ under the License.
</dependency>
<dependency>
- <groupId>com.klarna</groupId>
+ <groupId>io.github.hiverunner</groupId>
<artifactId>hiverunner</artifactId>
<version>${hiverunner.version}</version>
<scope>test</scope>
diff --git a/paimon-hive/paimon-hive-connector-3.1/pom.xml
b/paimon-hive/paimon-hive-connector-3.1/pom.xml
index ef700d8c0b..5a08634f01 100644
--- a/paimon-hive/paimon-hive-connector-3.1/pom.xml
+++ b/paimon-hive/paimon-hive-connector-3.1/pom.xml
@@ -35,6 +35,7 @@ under the License.
<properties>
<hive.version>3.1.2</hive.version>
+ <hiverunner.version>6.1.0</hiverunner.version>
<jackson.version>2.13.3</jackson.version>
</properties>
@@ -170,7 +171,7 @@ under the License.
</dependency>
<dependency>
- <groupId>com.klarna</groupId>
+ <groupId>io.github.hiverunner</groupId>
<artifactId>hiverunner</artifactId>
<version>${hiverunner.version}</version>
<scope>test</scope>
@@ -228,6 +229,10 @@ under the License.
<artifactId>hadoop-mapreduce-client-core</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
<exclusion>
<artifactId>hadoop-yarn-api</artifactId>
<groupId>org.apache.hadoop</groupId>
diff --git a/paimon-hive/paimon-hive-connector-common/pom.xml
b/paimon-hive/paimon-hive-connector-common/pom.xml
index a79f2002ea..43fd4d538c 100644
--- a/paimon-hive/paimon-hive-connector-common/pom.xml
+++ b/paimon-hive/paimon-hive-connector-common/pom.xml
@@ -165,7 +165,7 @@ under the License.
</dependency>
<dependency>
- <groupId>com.klarna</groupId>
+ <groupId>io.github.hiverunner</groupId>
<artifactId>hiverunner</artifactId>
<version>${hiverunner.version}</version>
<scope>test</scope>
@@ -255,6 +255,97 @@ under the License.
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>tez-dag</artifactId>
+ <groupId>org.apache.tez</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>tez-common</artifactId>
+ <groupId>org.apache.tez</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>tez-mapreduce</artifactId>
+ <groupId>org.apache.tez</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <artifactId>tez-dag</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>${tez.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <artifactId>tez-common</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>${tez.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <artifactId>tez-mapreduce</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>${tez.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ </exclusion>
</exclusions>
</dependency>
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java
index f3fe03fbba..073c119729 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java
@@ -92,9 +92,7 @@ public class HiveLocationTest {
options.set(CatalogOptions.METASTORE, "hive");
options.set(CatalogOptions.URI, "");
options.set(CatalogOptions.LOCK_ENABLED, false);
- options.set(
- HiveCatalogOptions.HIVE_CONF_DIR,
- hiveShell.getBaseDir().getRoot().getPath() + HIVE_CONF);
+ options.set(HiveCatalogOptions.HIVE_CONF_DIR, hiveShell.getBaseDir() +
HIVE_CONF);
options.set(HiveCatalogOptions.LOCATION_IN_PROPERTIES, true);
for (Map.Entry<String, String> stringStringEntry :
@@ -248,7 +246,7 @@ public class HiveLocationTest {
String[][] params =
new String[][] {
{"table1", objectStorePath},
- {"table2",
hiveShell.getBaseDir().getRoot().getAbsolutePath()},
+ {"table2",
hiveShell.getBaseDir().toAbsolutePath().toString()},
};
for (String[] param : params) {
String tableName = param[0];
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
index fbb29cb3c5..2a7ed597fd 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
@@ -40,7 +40,6 @@ import org.junit.Ignore;
import org.junit.internal.AssumptionViolatedException;
import org.junit.internal.runners.model.EachTestNotifier;
import org.junit.rules.ExternalResource;
-import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runner.notification.RunNotifier;
@@ -84,7 +83,12 @@ public class PaimonEmbeddedHiveRunner extends
BlockJUnit4ClassRunner {
protected List<TestRule> classRules() {
// need to load hive runner config before the context is inited
loadAnnotatesHiveRunnerConfig(getTestClass().getJavaClass());
- final TemporaryFolder temporaryFolder = new TemporaryFolder();
+ final Path testBaseDir;
+ try {
+ testBaseDir = Files.createTempDirectory(null);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
List<TestRule> rules = super.classRules();
ExternalResource hiveShell =
new ExternalResource() {
@@ -93,7 +97,7 @@ public class PaimonEmbeddedHiveRunner extends
BlockJUnit4ClassRunner {
@Override
protected void before() throws Throwable {
PaimonEmbeddedHiveServerContext
paimonEmbeddedHiveServerContext =
- new
PaimonEmbeddedHiveServerContext(temporaryFolder, config);
+ new
PaimonEmbeddedHiveServerContext(testBaseDir, config);
innerContainer =
createHiveServerContainer(
getTestClass().getJavaClass(),
@@ -125,7 +129,6 @@ public class PaimonEmbeddedHiveRunner extends
BlockJUnit4ClassRunner {
rules.add(minio);
rules.add(hiveShell);
- rules.add(temporaryFolder);
return rules;
}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
index ba472d5ba4..744bd6d5b2 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
@@ -23,11 +23,13 @@ import com.klarna.hiverunner.config.HiveRunnerConfig;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Map;
import java.util.UUID;
@@ -57,11 +59,11 @@ public class PaimonEmbeddedHiveServerContext implements
HiveServerContext {
private final HiveConf hiveConf = new HiveConf();
- private final TemporaryFolder basedir;
+ private final Path basedir;
private final HiveRunnerConfig hiveRunnerConfig;
private boolean inited = false;
- PaimonEmbeddedHiveServerContext(TemporaryFolder basedir, HiveRunnerConfig
hiveRunnerConfig) {
+ PaimonEmbeddedHiveServerContext(Path basedir, HiveRunnerConfig
hiveRunnerConfig) {
this.basedir = basedir;
this.hiveRunnerConfig = hiveRunnerConfig;
}
@@ -95,7 +97,7 @@ public class PaimonEmbeddedHiveServerContext implements
HiveServerContext {
// such code can
// read the configurations we set here.
private void setHiveSitePath() {
- File hiveSite = new File(newFolder(basedir, "hive-conf"),
"hive-site.xml");
+ File hiveSite = new File(newFolder(basedir, "hive-conf").toFile(),
"hive-site.xml");
try (FileOutputStream outputStream = new FileOutputStream(hiveSite)) {
hiveConf.writeXml(outputStream);
HiveConf.setHiveSiteLocation(hiveSite.toURI().toURL());
@@ -205,10 +207,11 @@ public class PaimonEmbeddedHiveServerContext implements
HiveServerContext {
createAndSetFolderProperty("test.log.dir", "logs", hiveConf, basedir);
}
- private File newFolder(TemporaryFolder basedir, String folder) {
+ private Path newFolder(Path basedir, String folder) {
try {
- File newFolder = basedir.newFolder(folder);
- FileUtil.setPermission(newFolder, FsPermission.getDirDefault());
+ Path subDirectoryPath = Paths.get(basedir.toString(), folder);
+ Path newFolder = Files.createDirectory(subDirectoryPath);
+ FileUtil.setPermission(newFolder.toFile(),
FsPermission.getDirDefault());
return newFolder;
} catch (IOException e) {
throw new IllegalStateException("Failed to create tmp dir: " +
e.getMessage(), e);
@@ -220,17 +223,17 @@ public class PaimonEmbeddedHiveServerContext implements
HiveServerContext {
}
@Override
- public TemporaryFolder getBaseDir() {
+ public Path getBaseDir() {
return basedir;
}
private void createAndSetFolderProperty(
- HiveConf.ConfVars var, String folder, HiveConf conf,
TemporaryFolder basedir) {
- conf.setVar(var, newFolder(basedir, folder).getAbsolutePath());
+ HiveConf.ConfVars var, String folder, HiveConf conf, Path basedir)
{
+ conf.setVar(var, newFolder(basedir,
folder).toAbsolutePath().toString());
}
private void createAndSetFolderProperty(
- String key, String folder, HiveConf conf, TemporaryFolder basedir)
{
- conf.set(key, newFolder(basedir, folder).getAbsolutePath());
+ String key, String folder, HiveConf conf, Path basedir) {
+ conf.set(key, newFolder(basedir, folder).toAbsolutePath().toString());
}
}
diff --git a/paimon-hive/pom.xml b/paimon-hive/pom.xml
index 92f32d1336..37e54dc4ac 100644
--- a/paimon-hive/pom.xml
+++ b/paimon-hive/pom.xml
@@ -46,7 +46,8 @@ under the License.
<properties>
<hive.version>2.3.10</hive.version>
- <hiverunner.version>4.0.0</hiverunner.version>
+ <tez.version>0.10.0</tez.version>
+ <hiverunner.version>5.5.0</hiverunner.version>
<reflections.version>0.9.8</reflections.version>
<aws.version>1.12.319</aws.version>
<iceberg.flink.version>1.19</iceberg.flink.version>
