This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6f4570d3af0 [cases](regression-test) Add backup restore operation test
(#26169)
6f4570d3af0 is described below
commit 6f4570d3af06fde2b241e3339096c6df820eb85d
Author: walter <[email protected]>
AuthorDate: Tue Oct 31 22:36:45 2023 +0800
[cases](regression-test) Add backup restore operation test (#26169)
1. restore overwrites an exists table
2. backup & restore with exclude table
3. restore to a new table
4. restore mix exists and new tables
5. restore with alias
---
.../backup_restore/test_backup_restore_alias.out | 41 ++++++++
.../backup_restore/test_backup_restore_exclude.out | 29 ++++++
.../test_backup_restore_multi_tables_overwrite.out | 27 ++++++
.../test_backup_restore_alias.groovy | 88 ++++++++++++++++++
.../test_backup_restore_exclude.groovy | 103 +++++++++++++++++++++
...st_backup_restore_multi_tables_overwrite.groovy | 101 ++++++++++++++++++++
.../test_restore_mix_exists_and_new_table.groovy | 99 ++++++++++++++++++++
.../test_restore_to_new_table.groovy | 86 +++++++++++++++++
8 files changed, 574 insertions(+)
diff --git a/regression-test/data/backup_restore/test_backup_restore_alias.out
b/regression-test/data/backup_restore/test_backup_restore_alias.out
new file mode 100644
index 00000000000..a8927690586
--- /dev/null
+++ b/regression-test/data/backup_restore/test_backup_restore_alias.out
@@ -0,0 +1,41 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+20 21
+123 341
+
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+20 21
+123 341
+
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+
diff --git
a/regression-test/data/backup_restore/test_backup_restore_exclude.out
b/regression-test/data/backup_restore/test_backup_restore_exclude.out
new file mode 100644
index 00000000000..1c307a75a8c
--- /dev/null
+++ b/regression-test/data/backup_restore/test_backup_restore_exclude.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+20 20
+21 21
+
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+20 20
+21 21
+
diff --git
a/regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out
b/regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out
new file mode 100644
index 00000000000..6ee3e7f7eae
--- /dev/null
+++
b/regression-test/data/backup_restore/test_backup_restore_multi_tables_overwrite.out
@@ -0,0 +1,27 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+20 20
+21 21
+
+-- !select --
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+10 10
+
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_alias.groovy
b/regression-test/suites/backup_restore/test_backup_restore_alias.groovy
new file mode 100644
index 00000000000..532d04ed13d
--- /dev/null
+++ b/regression-test/suites/backup_restore/test_backup_restore_alias.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_alias", "backup_restore") {
+ String repoName = "test_backup_restore_alias_repo"
+ String dbName = "backup_restore_alias_db"
+ String tableName = "test_backup_restore_alias_table"
+ String aliasName = "test_backup_restore_alias_table_alias"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `id` LARGEINT NOT NULL,
+ `count` LARGEINT SUM DEFAULT "0")
+ AGGREGATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ )
+ """
+
+ List<String> values = []
+ for (int i = 1; i <= 10; ++i) {
+ values.add("(${i}, ${i})")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+ def result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), values.size());
+
+ String snapshotName = "test_backup_restore_snapshot"
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ """
+
+ while (!syncer.checkSnapshotFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ sql "INSERT INTO ${dbName}.${tableName} VALUES (20, 21), (123, 341)"
+ qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY id"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON ( `${tableName}` AS `${aliasName}` )
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "replication_num" = "1"
+ )
+ """
+
+ while (!syncer.checkAllRestoreFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY id"
+ qt_select "SELECT * FROM ${dbName}.${aliasName} ORDER BY id"
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
+
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
new file mode 100644
index 00000000000..6d248bb47ff
--- /dev/null
+++ b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_exclude", "backup_restore") {
+ String dbName = "backup_restore_exclude"
+ String suiteName = "test_backup_restore_exclude"
+ String repoName = "${suiteName}_repo"
+ String snapshotName = "${suiteName}_snapshot"
+ String tableNamePrefix = "${suiteName}_tables"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+
+ int numTables = 10;
+ int numRows = 10;
+ List<String> tables = []
+ for (int i = 0; i < numTables; ++i) {
+ String tableName = "${tableNamePrefix}_${i}"
+ tables.add(tableName)
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `id` LARGEINT NOT NULL,
+ `count` LARGEINT SUM DEFAULT "0"
+ )
+ AGGREGATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ )
+ """
+ List<String> values = []
+ for (int j = 1; j <= numRows; ++j) {
+ values.add("(${j}, ${j})")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+ def result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), numRows);
+ }
+
+ def backupExcludeTable = tables.removeLast();
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ EXCLUDE (${backupExcludeTable})
+ """
+
+ while (!syncer.checkSnapshotFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ // Overwrite exists table.
+ sql "INSERT INTO ${dbName}.${backupExcludeTable} VALUES (20, 20), (21, 21)"
+ qt_select "SELECT * FROM ${dbName}.${backupExcludeTable} ORDER BY id"
+
+ def restoreExcludeTable = tables.removeLast()
+ sql "DROP TABLE ${dbName}.${restoreExcludeTable} FORCE"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ EXCLUDE (${restoreExcludeTable})
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "replication_num" = "1"
+ )
+ """
+
+ while (!syncer.checkAllRestoreFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ qt_select "SELECT * FROM ${dbName}.${backupExcludeTable} ORDER BY id"
+ for (def tableName in tables) {
+ result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), numRows);
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ }
+
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
+
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
new file mode 100644
index 00000000000..e7e2a7f3fe4
--- /dev/null
+++
b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_multi_tables_overwrite", "backup_restore") {
+ String dbName = "backup_restore_multi_tables_overwrite_db"
+ String suiteName = "test_backup_restore_multi_tables_overwrite"
+ String repoName = "${suiteName}_repo"
+ String snapshotName = "${suiteName}_snapshot"
+ String tableNamePrefix = "${suiteName}_tables"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+
+ int numTables = 10;
+ int numRows = 10;
+ List<String> tables = []
+ for (int i = 0; i < numTables; ++i) {
+ String tableName = "${tableNamePrefix}_${i}"
+ tables.add(tableName)
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `id` LARGEINT NOT NULL,
+ `count` LARGEINT SUM DEFAULT "0"
+ )
+ AGGREGATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ )
+ """
+ List<String> values = []
+ for (int j = 1; j <= numRows; ++j) {
+ values.add("(${j}, ${j})")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+ def result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), numRows);
+ }
+
+ def backupTables = tables[0..5]
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${backupTables.join(",")})
+ """
+
+ while (!syncer.checkSnapshotFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ // Overwrite exists table.
+ def firstTableName = backupTables[0]
+ sql "INSERT INTO ${dbName}.${firstTableName} VALUES (20, 20), (21, 21)"
+ qt_select "SELECT * FROM ${dbName}.${firstTableName} ORDER BY id"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON (${backupTables.join(",")})
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "replication_num" = "1"
+ )
+ """
+
+ while (!syncer.checkAllRestoreFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ qt_select "SELECT * FROM ${dbName}.${firstTableName} ORDER BY id"
+ for (def tableName in tables) {
+ result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), numRows);
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ }
+
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
+
diff --git
a/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
new file mode 100644
index 00000000000..2c9b0b31aed
--- /dev/null
+++
b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_restore_mix_exists_and_new_table", "backup_restore") {
+ String dbName = "restore_mix_exists_and_new_table"
+ String suiteName = "test_restore_mix_exists_and_new_table"
+ String repoName = "${suiteName}_repo"
+ String snapshotName = "${suiteName}_snapshot"
+ String tableNamePrefix = "${suiteName}_tables"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+
+ int numTables = 10;
+ int numRows = 10;
+ List<String> tables = []
+ for (int i = 0; i < numTables; ++i) {
+ String tableName = "${tableNamePrefix}_${i}"
+ tables.add(tableName)
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `id` LARGEINT NOT NULL,
+ `count` LARGEINT SUM DEFAULT "0"
+ )
+ AGGREGATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ )
+ """
+ List<String> values = []
+ for (int j = 1; j <= numRows; ++j) {
+ values.add("(${j}, ${j})")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+ def result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), numRows);
+ }
+
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tables.join(",")})
+ """
+
+ while (!syncer.checkSnapshotFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ def dropTables = tables[0..5]
+ for (def tableName in dropTables) {
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ }
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON (${tables.join(",")})
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "replication_num" = "1"
+ )
+ """
+
+ while (!syncer.checkAllRestoreFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ for (def tableName in tables) {
+ result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), numRows);
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ }
+
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
+
diff --git
a/regression-test/suites/backup_restore/test_restore_to_new_table.groovy
b/regression-test/suites/backup_restore/test_restore_to_new_table.groovy
new file mode 100644
index 00000000000..74d2ec08bb1
--- /dev/null
+++ b/regression-test/suites/backup_restore/test_restore_to_new_table.groovy
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_restore_to_new_table", "backup_restore") {
+ String repoName = "test_restore_to_new_table_repo"
+ String dbName = "restore_to_new_table_db"
+ String tableName = "test_restore_to_new_table_table"
+ String snapshotName = "test_backup_restore_snapshot"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `id` LARGEINT NOT NULL,
+ `count` LARGEINT SUM DEFAULT "0")
+ AGGREGATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ )
+ """
+
+ List<String> values = []
+ for (int i = 1; i <= 10; ++i) {
+ values.add("(${i}, ${i})")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+ def result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), values.size());
+
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ """
+
+ while (!syncer.checkSnapshotFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON ( `${tableName}`)
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "replication_num" = "1"
+ )
+ """
+
+ while (!syncer.checkAllRestoreFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), values.size());
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
