This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d61e77974a [hive][clone] support create database if not exists (#5536)
d61e77974a is described below
commit d61e77974ada9b2fd2f4edd354a37402f6ae88e9
Author: shyjsarah <[email protected]>
AuthorDate: Fri Apr 25 15:29:35 2025 +0800
[hive][clone] support create database if not exists (#5536)
---
.../paimon/flink/clone/ListCloneFilesFunction.java | 5 +++
.../apache/paimon/hive/migrate/HiveCloneUtils.java | 12 +++++
.../paimon/hive/procedure/CloneActionITCase.java | 52 ++++++++++++++++++++++
3 files changed, 69 insertions(+)
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/ListCloneFilesFunction.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/ListCloneFilesFunction.java
index 6e933ac434..f41a743536 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/ListCloneFilesFunction.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/ListCloneFilesFunction.java
@@ -72,6 +72,11 @@ public class ListCloneFilesFunction
String sourceType =
sourceCatalogConfig.get(CatalogOptions.METASTORE.key());
checkNotNull(sourceType);
+ // create database if not exists
+ Map<String, String> databaseOptions =
+ HiveCloneUtils.getDatabaseOptions(hiveCatalog,
tuple.f0.getDatabaseName());
+ targetCatalog.createDatabase(tuple.f1.getDatabaseName(), true,
databaseOptions);
+
Schema schema = HiveCloneUtils.hiveTableToPaimonSchema(hiveCatalog,
tuple.f0);
Map<String, String> options = schema.options();
// only support Hive to unaware-bucket table now
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveCloneUtils.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveCloneUtils.java
index 5c982968f0..84d878e9ee 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveCloneUtils.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveCloneUtils.java
@@ -31,6 +31,7 @@ import org.apache.paimon.schema.Schema;
import org.apache.paimon.types.RowType;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
@@ -60,6 +61,17 @@ public class HiveCloneUtils {
public static final Predicate<FileStatus> HIDDEN_PATH_FILTER =
p -> !p.getPath().getName().startsWith("_") &&
!p.getPath().getName().startsWith(".");
+ public static Map<String, String> getDatabaseOptions(
+ HiveCatalog hiveCatalog, String databaseName) throws Exception {
+ IMetaStoreClient client = hiveCatalog.getHmsClient();
+ Database database = client.getDatabase(databaseName);
+ Map<String, String> paimonOptions = new HashMap<>();
+ if (database.getDescription() != null) {
+ paimonOptions.put("comment", database.getDescription());
+ }
+ return paimonOptions;
+ }
+
public static List<Identifier> listTables(HiveCatalog hiveCatalog) throws
Exception {
IMetaStoreClient client = hiveCatalog.getHmsClient();
List<Identifier> results = new ArrayList<>();
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/procedure/CloneActionITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/procedure/CloneActionITCase.java
index 0a4deee915..83b948767f 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/procedure/CloneActionITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/procedure/CloneActionITCase.java
@@ -447,6 +447,58 @@ public class CloneActionITCase extends ActionITCaseBase {
}
}
+ @Test
+ public void testCloneWithNotExistedDatabase() throws Exception {
+ String format = "avro";
+
+ TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().build();
+ tEnv.executeSql("CREATE CATALOG HIVE WITH ('type'='hive')");
+ tEnv.useCatalog("HIVE");
+ tEnv.getConfig().setSqlDialect(SqlDialect.HIVE);
+ tEnv.executeSql(
+ "CREATE TABLE hivetable (id string) PARTITIONED BY (id2 int,
id3 int) STORED AS "
+ + format);
+ tEnv.executeSql("INSERT INTO hivetable VALUES" + data(100)).await();
+
+ tEnv.getConfig().setSqlDialect(SqlDialect.DEFAULT);
+ String query = "SELECT * FROM hivetable";
+ List<Row> r1 = ImmutableList.copyOf(tEnv.executeSql(query).collect());
+
+ tEnv.executeSql(
+ "CREATE CATALOG PAIMON WITH ('type'='paimon', 'warehouse' = '"
+ warehouse + "')");
+ tEnv.useCatalog("PAIMON");
+
+ List<String> args =
+ new ArrayList<>(
+ Arrays.asList(
+ "clone",
+ "--database",
+ "default",
+ "--table",
+ "hivetable",
+ "--catalog_conf",
+ "metastore=hive",
+ "--catalog_conf",
+ "uri=thrift://localhost:" + PORT,
+ "--target_database",
+ "test",
+ "--target_table",
+ "test_table",
+ "--target_catalog_conf",
+ "warehouse=" + warehouse));
+
+ createAction(CloneAction.class, args).run();
+ FileStoreTable paimonTable =
+ paimonTable(tEnv, "PAIMON", Identifier.create("test",
"test_table"));
+
+
Assertions.assertThat(paimonTable.partitionKeys()).containsExactly("id2",
"id3");
+
+ List<Row> r2 =
+ ImmutableList.copyOf(tEnv.executeSql("SELECT * FROM
test.test_table").collect());
+
+ Assertions.assertThatList(r1).containsExactlyInAnyOrderElementsOf(r2);
+ }
+
private String[] ddls() {
// has primary key
String ddl0 =
