This is an automated email from the ASF dual-hosted git repository.
xy720 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7f5b6f510d1 [bug](cloud restore) rewrite table properties and
partition info in cloud restore (#63696)
7f5b6f510d1 is described below
commit 7f5b6f510d19d2945e088ea670ecbfaa829ecd6f
Author: xy720 <[email protected]>
AuthorDate: Wed Jun 10 17:04:58 2026 +0800
[bug](cloud restore) rewrite table properties and partition info in cloud
restore (#63696)
### What problem does this PR solve?
In cloud mode, rewrite all unsupported table properties and some
partition info from the source cluster.
These table properties (e.g., dynamic_partition.replication_num,
dynamic_partition.replication_allocation,
dynamic_partition.storage_policy, in_memory,
storage_medium,min_load_replica_num...) are not applicable in cloud
mode.
And rewrite some non-applicable partition info as well.
If kept, they would cause some critical problems.
For example, dynamic partition scheduler creates new partitions with
source cluster replication settings, leading to write failures like:
"alive replica num < 1 load required replica num 2".
---
.../java/org/apache/doris/backup/RestoreJob.java | 26 ++++-
.../org/apache/doris/catalog/PartitionInfo.java | 31 ++++--
.../org/apache/doris/catalog/TableProperty.java | 12 +++
.../cloud/common/util/CloudPropertyAnalyzer.java | 1 +
.../org/apache/doris/catalog/OlapTableTest.java | 120 +++++++++++++++++++++
...st_show_create_table_with_storage_policy.groovy | 6 +-
6 files changed, 187 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 965f9ea485e..f6454441553 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1162,9 +1162,20 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
if (reserveReplica) {
restoreReplicaAlloc =
remotePartitionInfo.getReplicaAllocation(remotePartId);
}
+ boolean isInMemory =
remotePartitionInfo.getIsInMemory(remotePartId);
+ if (Config.isCloudMode()) {
+ // In cloud mode, storage_medium, cooldown_time,
storage_policy and in_memory
+ // from the source cluster are not applicable. Reset
them to defaults.
+ remoteDataProperty = new DataProperty(
+ DataProperty.DEFAULT_STORAGE_MEDIUM,
+ DataProperty.MAX_COOLDOWN_TIME_MS,
+ "",
+ remoteDataProperty.isMutable());
+ isInMemory = false;
+ }
localPartitionInfo.addPartition(restoredPart.getId(),
false, remoteItem,
remoteDataProperty, restoreReplicaAlloc,
- remotePartitionInfo.getIsInMemory(remotePartId),
+ isInMemory,
remotePartitionInfo.getIsMutable(remotePartId));
}
localTbl.addPartition(restoredPart);
@@ -1706,9 +1717,20 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
if (reserveReplica) {
restoreReplicaAlloc =
remotePartitionInfo.getReplicaAllocation(remotePartId);
}
+ boolean isInMemory =
remotePartitionInfo.getIsInMemory(remotePartId);
+ if (Config.isCloudMode()) {
+ // In cloud mode, storage_medium, cooldown_time,
storage_policy and in_memory
+ // from the source cluster are not applicable. Reset them to
defaults.
+ remoteDataProperty = new DataProperty(
+ DataProperty.DEFAULT_STORAGE_MEDIUM,
+ DataProperty.MAX_COOLDOWN_TIME_MS,
+ "",
+ remoteDataProperty.isMutable());
+ isInMemory = false;
+ }
localPartitionInfo.addPartition(restorePart.getId(), false,
remotePartitionInfo.getItem(remotePartId),
remoteDataProperty, restoreReplicaAlloc,
- remotePartitionInfo.getIsInMemory(remotePartId),
+ isInMemory,
remotePartitionInfo.getIsMutable(remotePartId));
localTbl.addPartition(restorePart);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 89e6c637a3a..5cec453baac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTabletType;
@@ -427,15 +428,33 @@ public class PartitionInfo {
idToStoragePolicy = Maps.newHashMap();
for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) {
- idToDataProperty.put(entry.getKey(),
origIdToDataProperty.get(entry.getValue()));
- idToReplicaAllocation.put(entry.getKey(),
- restoreReplicaAlloc == null ?
origIdToReplicaAllocation.get(entry.getValue())
+ long newPartId = entry.getKey();
+ long origPartId = entry.getValue();
+
+ if (Config.isCloudMode()) {
+ // In cloud mode, storage_medium, cooldown_time, and
storage_policy are not applicable.
+ // Reset DataProperty to default and clear storage policy to
avoid carrying over
+ // source cluster's storage settings that have no meaning in
cloud mode.
+ DataProperty origDataProperty =
origIdToDataProperty.get(origPartId);
+ idToDataProperty.put(newPartId, new DataProperty(
+ DataProperty.DEFAULT_STORAGE_MEDIUM,
+ DataProperty.MAX_COOLDOWN_TIME_MS,
+ "",
+ origDataProperty != null ?
origDataProperty.isMutable() : true));
+ idToStoragePolicy.put(newPartId, "");
+ idToInMemory.put(newPartId, false);
+ } else {
+ idToDataProperty.put(newPartId,
origIdToDataProperty.get(origPartId));
+ idToStoragePolicy.put(newPartId,
origIdToStoragePolicy.getOrDefault(origPartId, ""));
+ idToInMemory.put(newPartId, origIdToInMemory.get(origPartId));
+ }
+
+ idToReplicaAllocation.put(newPartId,
+ restoreReplicaAlloc == null ?
origIdToReplicaAllocation.get(origPartId)
: restoreReplicaAlloc);
if (!isSinglePartitioned) {
- idToItem.put(entry.getKey(),
origIdToItem.get(entry.getValue()));
+ idToItem.put(newPartId, origIdToItem.get(origPartId));
}
- idToInMemory.put(entry.getKey(),
origIdToInMemory.get(entry.getValue()));
- idToStoragePolicy.put(entry.getKey(),
origIdToStoragePolicy.getOrDefault(entry.getValue(), ""));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index cb5a7242288..388ea3a8c4e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
@@ -191,6 +192,17 @@ public class TableProperty implements GsonPostProcessable {
*/
public TableProperty resetPropertiesForRestore(boolean
reserveDynamicPartitionEnable, boolean reserveReplica,
ReplicaAllocation
replicaAlloc) {
+ if (Config.isCloudMode()) {
+ // In cloud mode, rewrite all unsupported or forced properties
from the source cluster.
+ // These properties (e.g., replication_num,
replication_allocation, storage_policy,
+ // storage_medium, in_memory, etc.) are not applicable in cloud
mode. If kept, they would
+ // cause some critical problems.
+ PropertyAnalyzer.getInstance().rewriteForceProperties(properties);
+ buildInMemory();
+ buildStorageMedium();
+ buildStoragePolicy();
+ buildMinLoadReplicaNum();
+ }
// disable dynamic partition
if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
if (!reserveDynamicPartitionEnable) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
index 7ac160f4b18..6acb3577f82 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
@@ -41,6 +41,7 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer {
RewriteProperty.delete("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_NUM),
RewriteProperty.delete("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION),
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_MEDIUM),
+
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_POLICY),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_ALLOCATION,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 0cefab3d91d..87b4c7c78b7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.info.IndexType;
+import org.apache.doris.cloud.common.util.CloudPropertyAnalyzer;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.rpc.VersionHelper;
import org.apache.doris.common.Config;
@@ -28,6 +29,7 @@ import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.UnitTestUtil;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import com.google.common.collect.Lists;
@@ -131,6 +133,124 @@ public class OlapTableTest {
Assert.assertEquals((short) 3,
olapTable.getDefaultReplicaAllocation().getTotalReplicaNum());
}
+ @Test
+ public void testResetPropertiesForRestoreInCloudMode() {
+ // simulate a restoring table with properties that are unsupported in
cloud mode
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put(DynamicPartitionProperty.ENABLE, "true");
+ properties.put(DynamicPartitionProperty.TIME_UNIT, "DAY");
+ properties.put(DynamicPartitionProperty.TIME_ZONE, "Asia/Shanghai");
+ properties.put(DynamicPartitionProperty.START, "-3");
+ properties.put(DynamicPartitionProperty.END, "3");
+ properties.put(DynamicPartitionProperty.PREFIX, "p");
+ properties.put(DynamicPartitionProperty.BUCKETS, "10");
+ properties.put(DynamicPartitionProperty.REPLICATION_NUM, "3");
+ properties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
"tag.location.default:3");
+ properties.put(DynamicPartitionProperty.STORAGE_MEDIUM, "SSD");
+ properties.put(PropertyAnalyzer.PROPERTIES_INMEMORY, "true");
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD");
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY,
"s3_policy");
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
"2025-01-01 00:00:00");
+ properties.put(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM, "2");
+ properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "3");
+ properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
"tag.location.default:3");
+ properties.put("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "3");
+ properties.put("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION, "tag.location.default:3");
+
+ TableProperty tableProperty = new TableProperty(properties);
+ OlapTable olapTable = new OlapTable();
+ olapTable.setTableProperty(tableProperty);
+
+ try (MockedStatic<Config> mockedConfig =
Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS);
+ MockedStatic<PropertyAnalyzer> mockedPA =
+ Mockito.mockStatic(PropertyAnalyzer.class,
Mockito.CALLS_REAL_METHODS)) {
+ mockedConfig.when(Config::isCloudMode).thenReturn(true);
+ mockedConfig.when(Config::isNotCloudMode).thenReturn(false);
+ mockedPA.when(PropertyAnalyzer::getInstance).thenReturn(new
CloudPropertyAnalyzer());
+
+ ReplicaAllocation cloudReplicaAlloc = new
ReplicaAllocation((short) 1);
+ // reserveDynamicPartitionEnable=true, reserveReplica=false
(forced in cloud mode)
+ olapTable.resetPropertiesForRestore(true, false,
cloudReplicaAlloc, false);
+
+ Map<String, String> resultProps =
olapTable.getTableProperty().getProperties();
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM));
+ Assert.assertEquals((short) 1,
olapTable.getDefaultReplicaAllocation().getTotalReplicaNum());
+ Assert.assertFalse(olapTable.getTableProperty().isInMemory());
+ Assert.assertNull(olapTable.getTableProperty().getStorageMedium());
+ Assert.assertEquals("",
olapTable.getTableProperty().getStoragePolicy());
+
Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().getEnable());
+
Assert.assertTrue(resultProps.containsKey(DynamicPartitionProperty.REPLICATION_NUM));
+
Assert.assertTrue(resultProps.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION));
+
Assert.assertFalse(resultProps.containsKey(DynamicPartitionProperty.STORAGE_MEDIUM));
+ }
+ }
+
+ @Test
+ public void testResetPartitionIdForRestore() {
+ PartitionInfo partitionInfo = new PartitionInfo(PartitionType.RANGE);
+ long origPartId = 1000L;
+ DataProperty origDataProperty = new DataProperty(TStorageMedium.SSD,
1735689600000L, "s3_policy");
+ ReplicaAllocation origReplicaAlloc = new ReplicaAllocation((short) 3);
+ partitionInfo.addPartition(origPartId, origDataProperty,
origReplicaAlloc, true, true);
+
+ Map<Long, Long> partitionIdMap = Maps.newHashMap();
+ long newPartId = 2000L;
+ partitionIdMap.put(newPartId, origPartId);
+
+ ReplicaAllocation restoreReplicaAlloc = new ReplicaAllocation((short)
2);
+
+ try (MockedStatic<Config> mockedConfig =
Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS)) {
+ mockedConfig.when(Config::isCloudMode).thenReturn(false);
+ mockedConfig.when(Config::isNotCloudMode).thenReturn(true);
+
+ partitionInfo.resetPartitionIdForRestore(partitionIdMap,
restoreReplicaAlloc, false);
+ Assert.assertEquals((short) 2,
+
partitionInfo.getReplicaAllocation(newPartId).getTotalReplicaNum());
+ DataProperty newDataProperty =
partitionInfo.getDataProperty(newPartId);
+ Assert.assertEquals(TStorageMedium.SSD,
newDataProperty.getStorageMedium());
+ Assert.assertEquals(1735689600000L,
newDataProperty.getCooldownTimeMs());
+ Assert.assertEquals("s3_policy",
newDataProperty.getStoragePolicy());
+ Assert.assertTrue(partitionInfo.getIsInMemory(newPartId));
+ }
+ }
+
+ @Test
+ public void testResetPartitionIdForRestoreInCloudMode() {
+ PartitionInfo partitionInfo = new PartitionInfo(PartitionType.RANGE);
+ long origPartId = 1000L;
+ DataProperty origDataProperty = new DataProperty(TStorageMedium.SSD,
1735689600000L, "s3_policy");
+ ReplicaAllocation origReplicaAlloc = new ReplicaAllocation((short) 3);
+ partitionInfo.addPartition(origPartId, origDataProperty,
origReplicaAlloc, true, true);
+
+ Map<Long, Long> partitionIdMap = Maps.newHashMap();
+ long newPartId = 2000L;
+ partitionIdMap.put(newPartId, origPartId);
+
+ ReplicaAllocation cloudReplicaAlloc = new ReplicaAllocation((short) 1);
+
+ try (MockedStatic<Config> mockedConfig =
Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS);
+ MockedStatic<PropertyAnalyzer> mockedPA =
+ Mockito.mockStatic(PropertyAnalyzer.class,
Mockito.CALLS_REAL_METHODS)) {
+ mockedConfig.when(Config::isCloudMode).thenReturn(true);
+ mockedConfig.when(Config::isNotCloudMode).thenReturn(false);
+ mockedPA.when(PropertyAnalyzer::getInstance).thenReturn(new
CloudPropertyAnalyzer());
+
+ partitionInfo.resetPartitionIdForRestore(partitionIdMap,
cloudReplicaAlloc, false);
+ Assert.assertEquals((short) 1,
+
partitionInfo.getReplicaAllocation(newPartId).getTotalReplicaNum());
+ DataProperty newDataProperty =
partitionInfo.getDataProperty(newPartId);
+ Assert.assertEquals(DataProperty.DEFAULT_STORAGE_MEDIUM,
newDataProperty.getStorageMedium());
+ Assert.assertEquals(DataProperty.MAX_COOLDOWN_TIME_MS,
newDataProperty.getCooldownTimeMs());
+ Assert.assertEquals("", newDataProperty.getStoragePolicy());
+ Assert.assertTrue(newDataProperty.isMutable());
+ Assert.assertFalse(partitionInfo.getIsInMemory(newPartId));
+ }
+ }
+
@Test
public void testBuildVariantEnableFlattenNestedWithLegacyPropertyKey()
throws IOException {
Map<String, String> properties = Maps.newHashMap();
diff --git
a/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
b/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
index 9194984c854..1b973bf419e 100644
---
a/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
+++
b/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
@@ -89,7 +89,11 @@ suite("test_show_create_table_with_storage_policy") {
String createSql = ret[0][1]
ret = sql """ SHOW PARTITIONS FROM ${tableName} """
- assertEquals(ret[9][12], storagePolicyName)
+ if (!isCloudMode()) {
+ assertEquals(ret[9][12], storagePolicyName)
+ } else {
+ assertEquals(ret[9][12], "")
+ }
sql """ DROP TABLE IF EXISTS ${tableName} """
// create table successfully with stmt from show create table
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
