This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 78f73eb8ff [test] Add random to table names in Spark Migrate Tests
78f73eb8ff is described below
commit 78f73eb8ff003a47985d375223cd60430ad48b72
Author: JingsongLi <[email protected]>
AuthorDate: Mon Mar 17 22:53:40 2025 +0800
[test] Add random to table names in Spark Migrate Tests
---
.../procedure/MigrateDatabaseProcedureTest.scala | 40 ++++-----
.../spark/procedure/MigrateFileProcedureTest.scala | 79 +++++++++---------
.../procedure/MigrateTableProcedureTest.scala | 94 ++++++++++++----------
3 files changed, 114 insertions(+), 99 deletions(-)
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala
index 20d7290bb6..a5a164d1bc 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala
@@ -22,30 +22,34 @@ import org.apache.paimon.spark.PaimonHiveTestBase
import org.apache.spark.sql.Row
+import java.util.concurrent.ThreadLocalRandom
+
class MigrateDatabaseProcedureTest extends PaimonHiveTestBase {
+ private val random = ThreadLocalRandom.current().nextInt(10000)
+
Seq("parquet", "orc", "avro").foreach(
format => {
test(s"Paimon migrate database procedure: migrate $format
non-partitioned database") {
- withTable("hive_tbl", "hive_tbl1") {
+ withTable(s"hive_tbl$random", s"hive_tbl1$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
spark.sql(s"""
- |CREATE TABLE hive_tbl1 (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl1$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
- var rows0 = spark.sql("SHOW CREATE TABLE hive_tbl").collect()
+ var rows0 = spark.sql(s"SHOW CREATE TABLE hive_tbl$random").collect()
assert(!rows0.apply(0).toString().contains("USING paimon"))
- rows0 = spark.sql("SHOW CREATE TABLE hive_tbl1").collect()
+ rows0 = spark.sql(s"SHOW CREATE TABLE hive_tbl1$random").collect()
assert(!rows0.apply(0).toString().contains("USING paimon"))
- spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b',
'p2')")
+ spark.sql(s"INSERT INTO hive_tbl$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
checkAnswer(
spark.sql(
@@ -54,13 +58,13 @@ class MigrateDatabaseProcedureTest extends
PaimonHiveTestBase {
)
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
- var rows1 = spark.sql("SHOW CREATE TABLE hive_tbl").collect()
+ var rows1 = spark.sql(s"SHOW CREATE TABLE hive_tbl$random").collect()
assert(rows1.apply(0).toString().contains("USING paimon"))
- rows1 = spark.sql("SHOW CREATE TABLE hive_tbl1").collect()
+ rows1 = spark.sql(s"SHOW CREATE TABLE hive_tbl1$random").collect()
assert(rows1.apply(0).toString().contains("USING paimon"))
}
}
@@ -70,37 +74,37 @@ class MigrateDatabaseProcedureTest extends
PaimonHiveTestBase {
format => {
test(
s"Paimon migrate database procedure: migrate $format database with
setting parallelism") {
- withTable("hive_tbl_01", "hive_tbl_02") {
+ withTable(s"hive_tbl_01$random", s"hive_tbl_02$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl_01 (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl_01$random (id STRING, name
STRING, pt STRING)
|USING $format
|""".stripMargin)
spark.sql(s"""
- |CREATE TABLE hive_tbl_02 (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl_02$random (id STRING, name
STRING, pt STRING)
|USING $format
|""".stripMargin)
- var rows0 = spark.sql("SHOW CREATE TABLE hive_tbl_01").collect()
+ var rows0 = spark.sql(s"SHOW CREATE TABLE
hive_tbl_01$random").collect()
assert(!rows0.apply(0).toString().contains("USING paimon"))
- rows0 = spark.sql("SHOW CREATE TABLE hive_tbl_02").collect()
+ rows0 = spark.sql(s"SHOW CREATE TABLE hive_tbl_02$random").collect()
assert(!rows0.apply(0).toString().contains("USING paimon"))
- spark.sql(s"INSERT INTO hive_tbl_01 VALUES ('1', 'a', 'p1'), ('2',
'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl_01$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
spark.sql(
s"CALL sys.migrate_database(source_type => 'hive', database =>
'$hiveDbName', options => 'file.format=$format', parallelism => 6)")
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl_01 ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl_01$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
- var rows1 = spark.sql("SHOW CREATE TABLE hive_tbl_01").collect()
+ var rows1 = spark.sql(s"SHOW CREATE TABLE
hive_tbl_01$random").collect()
assert(rows1.apply(0).toString().contains("USING paimon"))
- rows1 = spark.sql("SHOW CREATE TABLE hive_tbl_02").collect()
+ rows1 = spark.sql(s"SHOW CREATE TABLE hive_tbl_02$random").collect()
assert(rows1.apply(0).toString().contains("USING paimon"))
}
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala
index 8217c74f99..492350a0f7 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala
@@ -22,33 +22,38 @@ import org.apache.paimon.spark.PaimonHiveTestBase
import org.apache.spark.sql.Row
+import java.util.concurrent.ThreadLocalRandom
+
class MigrateFileProcedureTest extends PaimonHiveTestBase {
+
+ private val random = ThreadLocalRandom.current().nextInt(10000)
+
Seq("parquet", "orc", "avro").foreach(
format => {
test(s"Paimon migrate file procedure: migrate $format non-partitioned
table") {
- withTable(s"hive_tbl1$format", s"paimon_tbl1$format") {
+ withTable(s"hive_tbl1$random", s"paimon_tbl1$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl1$format (id STRING, name STRING,
pt STRING)
+ |CREATE TABLE hive_tbl1$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl1$format VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl1$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
// create paimon table
spark.sql(s"""
- |CREATE TABLE paimon_tbl1$format (id STRING, name
STRING, pt STRING)
+ |CREATE TABLE paimon_tbl1$random (id STRING, name
STRING, pt STRING)
|USING PAIMON
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
|""".stripMargin)
- spark.sql(s"INSERT INTO paimon_tbl1$format VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
+ spark.sql(s"INSERT INTO paimon_tbl1$random VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl1$format', target_table =>
'$hiveDbName.paimon_tbl1$format')")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl1$random', target_table =>
'$hiveDbName.paimon_tbl1$random')")
checkAnswer(
- spark.sql(s"SELECT * FROM paimon_tbl1$format ORDER BY id"),
+ spark.sql(s"SELECT * FROM paimon_tbl1$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1")
:: Row(
"4",
"d",
@@ -61,29 +66,29 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
format => {
test(
s"Paimon migrate file procedure: migrate $format non-partitioned table
with parallelism") {
- withTable(s"hive_tbl_02$format", s"paimon_tbl_02$format") {
+ withTable(s"hive_tbl_02$random", s"paimon_tbl_02$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl_02$format (id STRING, name
STRING, pt STRING)
+ |CREATE TABLE hive_tbl_02$random (id STRING, name
STRING, pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl_02$format VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl_02$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
// create paimon table
spark.sql(s"""
- |CREATE TABLE paimon_tbl_02$format (id STRING, name
STRING, pt STRING)
+ |CREATE TABLE paimon_tbl_02$random (id STRING, name
STRING, pt STRING)
|USING PAIMON
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
|""".stripMargin)
- spark.sql(s"INSERT INTO paimon_tbl_02$format VALUES ('3', 'c',
'p1'), ('4', 'd', 'p2')")
+ spark.sql(s"INSERT INTO paimon_tbl_02$random VALUES ('3', 'c',
'p1'), ('4', 'd', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl_02$format', target_table =>
'$hiveDbName.paimon_tbl_02$format', parallelism => 6)")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl_02$random', target_table =>
'$hiveDbName.paimon_tbl_02$random', parallelism => 6)")
checkAnswer(
- spark.sql(s"SELECT * FROM paimon_tbl_02$format ORDER BY id"),
+ spark.sql(s"SELECT * FROM paimon_tbl_02$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1")
:: Row(
"4",
"d",
@@ -96,31 +101,31 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
format => {
test(
s"Paimon migrate file procedure: migrate $format non-partitioned table
with delete source table") {
- withTable(s"hive_tbl3$format", s"paimon_tbl3$format") {
+ withTable(s"hive_tbl3$random", s"paimon_tbl3$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl3$format (id STRING, name STRING,
pt STRING)
+ |CREATE TABLE hive_tbl3$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl3$format VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl3$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
// create paimon table
spark.sql(s"""
- |CREATE TABLE paimon_tbl3$format (id STRING, name
STRING, pt STRING)
+ |CREATE TABLE paimon_tbl3$random (id STRING, name
STRING, pt STRING)
|USING PAIMON
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
|""".stripMargin)
- spark.sql(s"INSERT INTO paimon_tbl3$format VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
+ spark.sql(s"INSERT INTO paimon_tbl3$random VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl3$format', target_table =>
'$hiveDbName.paimon_tbl3$format', delete_origin => false)")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl3$random', target_table =>
'$hiveDbName.paimon_tbl3$random', delete_origin => false)")
- checkAnswer(spark.sql(s"SELECT * FROM hive_tbl3$format ORDER BY
id"), Nil)
+ checkAnswer(spark.sql(s"SELECT * FROM hive_tbl3$random ORDER BY
id"), Nil)
checkAnswer(
- spark.sql(s"SELECT * FROM paimon_tbl3$format ORDER BY id"),
+ spark.sql(s"SELECT * FROM paimon_tbl3$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1")
:: Row(
"4",
"d",
@@ -132,31 +137,31 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase
{
Seq("parquet", "orc", "avro").foreach(
format => {
test(s"Paimon migrate file procedure: migrate $format partitioned
table") {
- withTable(s"hive_tbl4$format", s"paimon_tbl4$format") {
+ withTable(s"hive_tbl4$random", s"paimon_tbl4$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl4$format (id STRING, name STRING,
pt STRING)
+ |CREATE TABLE hive_tbl4$random (id STRING, name STRING,
pt STRING)
|USING $format
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl4$format VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl4$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
// create paimon table
spark.sql(s"""
- |CREATE TABLE paimon_tbl4$format (id STRING, name
STRING, pt STRING)
+ |CREATE TABLE paimon_tbl4$random (id STRING, name
STRING, pt STRING)
|USING PAIMON
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO paimon_tbl4$format VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
+ spark.sql(s"INSERT INTO paimon_tbl4$random VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl4$format', target_table =>
'$hiveDbName.paimon_tbl4$format')")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl4$random', target_table =>
'$hiveDbName.paimon_tbl4$random')")
checkAnswer(
- spark.sql(s"SELECT * FROM paimon_tbl4$format ORDER BY id"),
+ spark.sql(s"SELECT * FROM paimon_tbl4$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1")
:: Row(
"4",
"d",
@@ -169,37 +174,37 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase
{
format => {
test(
s"Paimon migrate file procedure: migrate $format partitioned table
with delete source table") {
- withTable(s"hive_tbl5$format", s"paimon_tbl5$format") {
+ withTable(s"hive_tbl5$random", s"paimon_tbl5$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl5$format (id STRING, name STRING,
pt STRING)
+ |CREATE TABLE hive_tbl5$random (id STRING, name STRING,
pt STRING)
|USING $format
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl5$format VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl5$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
// create paimon table
spark.sql(s"""
- |CREATE TABLE paimon_tbl5$format (id STRING, name
STRING, pt STRING)
+ |CREATE TABLE paimon_tbl5$random (id STRING, name
STRING, pt STRING)
|USING PAIMON
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO paimon_tbl5$format VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
+ spark.sql(s"INSERT INTO paimon_tbl5$random VALUES ('3', 'c', 'p1'),
('4', 'd', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl5$format', target_table =>
'$hiveDbName.paimon_tbl5$format', delete_origin => false)")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl5$random', target_table =>
'$hiveDbName.paimon_tbl5$random', delete_origin => false)")
checkAnswer(
- spark.sql(s"SELECT * FROM paimon_tbl5$format ORDER BY id"),
+ spark.sql(s"SELECT * FROM paimon_tbl5$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1")
:: Row(
"4",
"d",
"p2") :: Nil)
- checkAnswer(spark.sql(s"SELECT * FROM hive_tbl5$format ORDER BY
id"), Nil)
+ checkAnswer(spark.sql(s"SELECT * FROM hive_tbl5$random ORDER BY
id"), Nil)
}
}
})
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala
index 9bffbce09f..946681e07a 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala
@@ -23,24 +23,29 @@ import org.apache.paimon.spark.PaimonHiveTestBase
import org.apache.spark.sql.Row
import org.assertj.core.api.Assertions.assertThatThrownBy
+import java.util.concurrent.ThreadLocalRandom
+
class MigrateTableProcedureTest extends PaimonHiveTestBase {
+
+ private val random = ThreadLocalRandom.current().nextInt(10000)
+
Seq("parquet", "orc", "avro").foreach(
format => {
test(s"Paimon migrate table procedure: migrate $format non-partitioned
table") {
- withTable("hive_tbl") {
+ withTable(s"hive_tbl$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b',
'p2')")
+ spark.sql(s"INSERT INTO hive_tbl$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl', options => 'file.format=$format')")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl$random', options => 'file.format=$format')")
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
}
}
@@ -50,20 +55,20 @@ class MigrateTableProcedureTest extends PaimonHiveTestBase {
format => {
test(
s"Paimon migrate table procedure: migrate $format non-partitioned
table with setting parallelism") {
- withTable("hive_tbl_01") {
+ withTable(s"hive_tbl_01$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl_01 (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl_01$random (id STRING, name
STRING, pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl_01 VALUES ('1', 'a', 'p1'), ('2',
'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl_01$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl_01', options => 'file.format=$format', parallelism => 6)")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl_01$random', options => 'file.format=$format', parallelism
=> 6)")
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl_01 ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl_01$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
}
}
@@ -72,25 +77,25 @@ class MigrateTableProcedureTest extends PaimonHiveTestBase {
Seq("parquet", "orc", "avro").foreach(
format => {
test(s"Paimon migrate table procedure: migrate $format table with
options_map") {
- withTable("hive_tbl") {
+ withTable(s"hive_tbl$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b',
'p2')")
+ spark.sql(s"INSERT INTO hive_tbl$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl', options => 'file.format=$format', options_map =>
map('orc.encrypt', 'pii:id,name'))")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl$random', options => 'file.format=$format', options_map =>
map('orc.encrypt', 'pii:id,name'))")
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
assert(
spark
- .sql("SHOW CREATE TABLE hive_tbl")
+ .sql(s"SHOW CREATE TABLE hive_tbl$random")
.collect()
.apply(0)
.toString()
@@ -103,23 +108,23 @@ class MigrateTableProcedureTest extends
PaimonHiveTestBase {
format => {
test(
s"Paimon migrate table procedure: migrate $format non-partitioned
table with set target table") {
- withTable("hive_tbl_rn") {
+ withTable(s"hive_tbl_$random", s"target_$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl_$format (id STRING, name STRING,
pt STRING)
+ |CREATE TABLE hive_tbl_$random (id STRING, name STRING,
pt STRING)
|USING $format
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl_$format VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
+ spark.sql(s"INSERT INTO hive_tbl_$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl_$format', options => 'file.format=$format', target_table
=> '$hiveDbName.target_$format', delete_origin => false)")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl_$random', options => 'file.format=$format', target_table
=> '$hiveDbName.target_$random', delete_origin => false)")
checkAnswer(
- spark.sql(s"SELECT * FROM target_$format ORDER BY id"),
+ spark.sql(s"SELECT * FROM target_$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
- checkAnswer(spark.sql(s"SELECT * FROM hive_tbl_$format ORDER BY
id"), Nil)
+ checkAnswer(spark.sql(s"SELECT * FROM hive_tbl_$random ORDER BY
id"), Nil)
}
}
@@ -128,67 +133,67 @@ class MigrateTableProcedureTest extends
PaimonHiveTestBase {
Seq("parquet", "orc", "avro").foreach(
format => {
test(s"Paimon migrate table procedure: migrate $format partitioned
table") {
- withTable("hive_tbl") {
+ withTable(s"hive_tbl$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl (id STRING, name STRING, pt
STRING)
+ |CREATE TABLE hive_tbl$random (id STRING, name STRING,
pt STRING)
|USING $format
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b',
'p2')")
+ spark.sql(s"INSERT INTO hive_tbl$random VALUES ('1', 'a', 'p1'),
('2', 'b', 'p2')")
spark.sql(
- s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl', options => 'file.format=$format')")
+ s"CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl$random', options => 'file.format=$format')")
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl$random ORDER BY id"),
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Nil)
}
}
})
test(s"Paimon migrate table procedure: migrate partitioned table with null
partition") {
- withTable("hive_tbl") {
+ withTable(s"hive_tbl$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl (id STRING, name STRING, pt INT)
+ |CREATE TABLE hive_tbl$random (id STRING, name STRING, pt
INT)
|USING parquet
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 1), ('2', 'b', null)")
+ spark.sql(s"INSERT INTO hive_tbl$random VALUES ('1', 'a', 1), ('2', 'b',
null)")
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl$random ORDER BY id"),
Row("1", "a", 1) :: Row("2", "b", null) :: Nil)
spark.sql(
- s"""CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl',
+ s"""CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl$random',
|options =>
'file.format=parquet,partition.default-name=__HIVE_DEFAULT_PARTITION__')
|""".stripMargin)
checkAnswer(
- spark.sql(s"SELECT * FROM hive_tbl ORDER BY id"),
+ spark.sql(s"SELECT * FROM hive_tbl$random ORDER BY id"),
Row("1", "a", 1) :: Row("2", "b", null) :: Nil)
}
}
test(s"Paimon migrate table procedure: migrate table with wrong options") {
- withTable("hive_tbl") {
+ withTable(s"hive_tbl$random") {
// create hive table
spark.sql(s"""
- |CREATE TABLE hive_tbl (id STRING, name STRING, pt INT)
+ |CREATE TABLE hive_tbl$random (id STRING, name STRING, pt
INT)
|USING parquet
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 1)")
+ spark.sql(s"INSERT INTO hive_tbl$random VALUES ('1', 'a', 1)")
assertThatThrownBy(
() =>
spark.sql(
- s"""CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl',
+ s"""CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.hive_tbl$random',
|options => 'file.format=parquet,bucket=1')
|""".stripMargin))
.hasMessageContaining("Hive migrator only support unaware-bucket
target table")
@@ -198,28 +203,29 @@ class MigrateTableProcedureTest extends
PaimonHiveTestBase {
test(s"Paimon migrate table procedure: select migrate table with partition
filter") {
Seq("parquet", "orc", "avro").foreach(
format => {
- withTable("migrate_tbl") {
+ withTable(s"migrate_tbl$random") {
spark.sql(s"""
- |CREATE TABLE migrate_tbl (id STRING, name STRING, pt
INT)
+ |CREATE TABLE migrate_tbl$random (id STRING, name
STRING, pt INT)
|USING $format
|PARTITIONED BY (pt)
|""".stripMargin)
- spark.sql(s"INSERT INTO migrate_tbl VALUES ('1', 'a', 1), ('2', 'b',
2)")
+ spark.sql(s"INSERT INTO migrate_tbl$random VALUES ('1', 'a', 1),
('2', 'b', 2)")
spark.sql(
- s"""CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.migrate_tbl',
+ s"""CALL sys.migrate_table(source_type => 'hive', table =>
'$hiveDbName.migrate_tbl$random',
|options => 'file.format=$format')
|""".stripMargin)
checkAnswer(
- spark.sql(s"SELECT * FROM migrate_tbl WHERE pt = 1 ORDER BY id"),
+ spark.sql(s"SELECT * FROM migrate_tbl$random WHERE pt = 1 ORDER BY
id"),
Row("1", "a", 1) :: Nil)
checkAnswer(
- spark.sql(s"SELECT * FROM migrate_tbl WHERE pt IS NOT null ORDER
BY id"),
+ spark.sql(s"SELECT * FROM migrate_tbl$random WHERE pt IS NOT null
ORDER BY id"),
Row("1", "a", 1) :: Row("2", "b", 2) :: Nil)
checkAnswer(
- spark.sql(s"SELECT * FROM migrate_tbl WHERE name LIKE 'a%' OR pt
IS null ORDER BY id"),
+ spark.sql(
+ s"SELECT * FROM migrate_tbl$random WHERE name LIKE 'a%' OR pt IS
null ORDER BY id"),
Row("1", "a", 1) :: Nil)
}
})
