This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 6fc573b4f4b branch-4.1: [bug](cloud restore) rewrite table properties
and partition info in cloud restore (#64466)
6fc573b4f4b is described below
commit 6fc573b4f4b99cd027c2895e41b105a216f32229
Author: xy720 <[email protected]>
AuthorDate: Tue Jun 16 16:12:09 2026 +0800
branch-4.1: [bug](cloud restore) rewrite table properties and partition
info in cloud restore (#64466)
### What problem does this PR solve?
pick #63696
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../java/org/apache/doris/backup/RestoreJob.java | 26 ++++-
.../org/apache/doris/catalog/PartitionInfo.java | 31 +++++-
.../org/apache/doris/catalog/TableProperty.java | 12 ++
.../cloud/common/util/CloudPropertyAnalyzer.java | 1 +
.../org/apache/doris/catalog/OlapTableTest.java | 122 +++++++++++++++++++++
...st_show_create_table_with_storage_policy.groovy | 6 +-
6 files changed, 189 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index fbd0148f7ca..5297d6f13f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1140,9 +1140,20 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
if (reserveReplica) {
restoreReplicaAlloc =
remotePartitionInfo.getReplicaAllocation(remotePartId);
}
+ boolean isInMemory =
remotePartitionInfo.getIsInMemory(remotePartId);
+ if (Config.isCloudMode()) {
+ // In cloud mode, storage_medium, cooldown_time,
storage_policy and in_memory
+ // from the source cluster are not applicable. Reset
them to defaults.
+ remoteDataProperty = new DataProperty(
+ DataProperty.DEFAULT_STORAGE_MEDIUM,
+ DataProperty.MAX_COOLDOWN_TIME_MS,
+ "",
+ remoteDataProperty.isMutable());
+ isInMemory = false;
+ }
localPartitionInfo.addPartition(restoredPart.getId(),
false, remoteItem,
remoteDataProperty, restoreReplicaAlloc,
- remotePartitionInfo.getIsInMemory(remotePartId),
+ isInMemory,
remotePartitionInfo.getIsMutable(remotePartId));
}
localTbl.addPartition(restoredPart);
@@ -1678,9 +1689,20 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
if (reserveReplica) {
restoreReplicaAlloc =
remotePartitionInfo.getReplicaAllocation(remotePartId);
}
+ boolean isInMemory =
remotePartitionInfo.getIsInMemory(remotePartId);
+ if (Config.isCloudMode()) {
+ // In cloud mode, storage_medium, cooldown_time,
storage_policy and in_memory
+ // from the source cluster are not applicable. Reset them to
defaults.
+ remoteDataProperty = new DataProperty(
+ DataProperty.DEFAULT_STORAGE_MEDIUM,
+ DataProperty.MAX_COOLDOWN_TIME_MS,
+ "",
+ remoteDataProperty.isMutable());
+ isInMemory = false;
+ }
localPartitionInfo.addPartition(restorePart.getId(), false,
remotePartitionInfo.getItem(remotePartId),
remoteDataProperty, restoreReplicaAlloc,
- remotePartitionInfo.getIsInMemory(remotePartId),
+ isInMemory,
remotePartitionInfo.getIsMutable(remotePartId));
localTbl.addPartition(restorePart);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 89e6c637a3a..5cec453baac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTabletType;
@@ -427,15 +428,33 @@ public class PartitionInfo {
idToStoragePolicy = Maps.newHashMap();
for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) {
- idToDataProperty.put(entry.getKey(),
origIdToDataProperty.get(entry.getValue()));
- idToReplicaAllocation.put(entry.getKey(),
- restoreReplicaAlloc == null ?
origIdToReplicaAllocation.get(entry.getValue())
+ long newPartId = entry.getKey();
+ long origPartId = entry.getValue();
+
+ if (Config.isCloudMode()) {
+ // In cloud mode, storage_medium, cooldown_time, and
storage_policy are not applicable.
+ // Reset DataProperty to default and clear storage policy to
avoid carrying over
+ // source cluster's storage settings that have no meaning in
cloud mode.
+ DataProperty origDataProperty =
origIdToDataProperty.get(origPartId);
+ idToDataProperty.put(newPartId, new DataProperty(
+ DataProperty.DEFAULT_STORAGE_MEDIUM,
+ DataProperty.MAX_COOLDOWN_TIME_MS,
+ "",
+ origDataProperty != null ?
origDataProperty.isMutable() : true));
+ idToStoragePolicy.put(newPartId, "");
+ idToInMemory.put(newPartId, false);
+ } else {
+ idToDataProperty.put(newPartId,
origIdToDataProperty.get(origPartId));
+ idToStoragePolicy.put(newPartId,
origIdToStoragePolicy.getOrDefault(origPartId, ""));
+ idToInMemory.put(newPartId, origIdToInMemory.get(origPartId));
+ }
+
+ idToReplicaAllocation.put(newPartId,
+ restoreReplicaAlloc == null ?
origIdToReplicaAllocation.get(origPartId)
: restoreReplicaAlloc);
if (!isSinglePartitioned) {
- idToItem.put(entry.getKey(),
origIdToItem.get(entry.getValue()));
+ idToItem.put(newPartId, origIdToItem.get(origPartId));
}
- idToInMemory.put(entry.getKey(),
origIdToInMemory.get(entry.getValue()));
- idToStoragePolicy.put(entry.getKey(),
origIdToStoragePolicy.getOrDefault(entry.getValue(), ""));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 584eb6dde24..830a668a3a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
@@ -184,6 +185,17 @@ public class TableProperty implements GsonPostProcessable {
*/
public TableProperty resetPropertiesForRestore(boolean
reserveDynamicPartitionEnable, boolean reserveReplica,
ReplicaAllocation
replicaAlloc) {
+ if (Config.isCloudMode()) {
+ // In cloud mode, rewrite all unsupported or forced properties
from the source cluster.
+ // These properties (e.g., replication_num,
replication_allocation, storage_policy,
+ // storage_medium, in_memory, etc.) are not applicable in cloud
mode. If kept, they would
+ // cause some critical problems.
+ PropertyAnalyzer.getInstance().rewriteForceProperties(properties);
+ buildInMemory();
+ buildStorageMedium();
+ buildStoragePolicy();
+ buildMinLoadReplicaNum();
+ }
// disable dynamic partition
if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
if (!reserveDynamicPartitionEnable) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
index 7ac160f4b18..6acb3577f82 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
@@ -41,6 +41,7 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer {
RewriteProperty.delete("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_NUM),
RewriteProperty.delete("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION),
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_MEDIUM),
+
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_POLICY),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_ALLOCATION,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index ec9670cb8e7..045c4dee3ea 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.cloud.common.util.CloudPropertyAnalyzer;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.rpc.VersionHelper;
import org.apache.doris.common.Config;
@@ -33,6 +34,7 @@ import org.apache.doris.resource.Tag;
import org.apache.doris.resource.computegroup.ComputeGroup;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TFetchOption;
+import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.utframe.UtFrameUtils;
@@ -42,6 +44,8 @@ import mockit.Mock;
import mockit.MockUp;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import java.io.DataInputStream;
import java.io.DataOutputStream;
@@ -140,6 +144,124 @@ public class OlapTableTest {
Assert.assertEquals((short) 3,
olapTable.getDefaultReplicaAllocation().getTotalReplicaNum());
}
+ @Test
+ public void testResetPropertiesForRestoreInCloudMode() {
+ // simulate a restoring table with properties that are unsupported in
cloud mode
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put(DynamicPartitionProperty.ENABLE, "true");
+ properties.put(DynamicPartitionProperty.TIME_UNIT, "DAY");
+ properties.put(DynamicPartitionProperty.TIME_ZONE, "Asia/Shanghai");
+ properties.put(DynamicPartitionProperty.START, "-3");
+ properties.put(DynamicPartitionProperty.END, "3");
+ properties.put(DynamicPartitionProperty.PREFIX, "p");
+ properties.put(DynamicPartitionProperty.BUCKETS, "10");
+ properties.put(DynamicPartitionProperty.REPLICATION_NUM, "3");
+ properties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
"tag.location.default:3");
+ properties.put(DynamicPartitionProperty.STORAGE_MEDIUM, "SSD");
+ properties.put(PropertyAnalyzer.PROPERTIES_INMEMORY, "true");
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD");
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY,
"s3_policy");
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
"2025-01-01 00:00:00");
+ properties.put(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM, "2");
+ properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "3");
+ properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
"tag.location.default:3");
+ properties.put("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "3");
+ properties.put("default." +
PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION, "tag.location.default:3");
+
+ TableProperty tableProperty = new TableProperty(properties);
+ OlapTable olapTable = new OlapTable();
+ olapTable.setTableProperty(tableProperty);
+
+ try (MockedStatic<Config> mockedConfig =
Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS);
+ MockedStatic<PropertyAnalyzer> mockedPA =
+ Mockito.mockStatic(PropertyAnalyzer.class,
Mockito.CALLS_REAL_METHODS)) {
+ mockedConfig.when(Config::isCloudMode).thenReturn(true);
+ mockedConfig.when(Config::isNotCloudMode).thenReturn(false);
+ mockedPA.when(PropertyAnalyzer::getInstance).thenReturn(new
CloudPropertyAnalyzer());
+
+ ReplicaAllocation cloudReplicaAlloc = new
ReplicaAllocation((short) 1);
+ // reserveDynamicPartitionEnable=true, reserveReplica=false
(forced in cloud mode)
+ olapTable.resetPropertiesForRestore(true, false,
cloudReplicaAlloc, false);
+
+ Map<String, String> resultProps =
olapTable.getTableProperty().getProperties();
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME));
+
Assert.assertFalse(resultProps.containsKey(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM));
+ Assert.assertEquals((short) 1,
olapTable.getDefaultReplicaAllocation().getTotalReplicaNum());
+ Assert.assertFalse(olapTable.getTableProperty().isInMemory());
+ Assert.assertNull(olapTable.getTableProperty().getStorageMedium());
+ Assert.assertEquals("",
olapTable.getTableProperty().getStoragePolicy());
+
Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().getEnable());
+
Assert.assertTrue(resultProps.containsKey(DynamicPartitionProperty.REPLICATION_NUM));
+
Assert.assertTrue(resultProps.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION));
+
Assert.assertFalse(resultProps.containsKey(DynamicPartitionProperty.STORAGE_MEDIUM));
+ }
+ }
+
+ @Test
+ public void testResetPartitionIdForRestore() {
+ PartitionInfo partitionInfo = new PartitionInfo(PartitionType.RANGE);
+ long origPartId = 1000L;
+ DataProperty origDataProperty = new DataProperty(TStorageMedium.SSD,
1735689600000L, "s3_policy");
+ ReplicaAllocation origReplicaAlloc = new ReplicaAllocation((short) 3);
+ partitionInfo.addPartition(origPartId, origDataProperty,
origReplicaAlloc, true, true);
+
+ Map<Long, Long> partitionIdMap = Maps.newHashMap();
+ long newPartId = 2000L;
+ partitionIdMap.put(newPartId, origPartId);
+
+ ReplicaAllocation restoreReplicaAlloc = new ReplicaAllocation((short)
2);
+
+ try (MockedStatic<Config> mockedConfig =
Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS)) {
+ mockedConfig.when(Config::isCloudMode).thenReturn(false);
+ mockedConfig.when(Config::isNotCloudMode).thenReturn(true);
+
+ partitionInfo.resetPartitionIdForRestore(partitionIdMap,
restoreReplicaAlloc, false);
+ Assert.assertEquals((short) 2,
+
partitionInfo.getReplicaAllocation(newPartId).getTotalReplicaNum());
+ DataProperty newDataProperty =
partitionInfo.getDataProperty(newPartId);
+ Assert.assertEquals(TStorageMedium.SSD,
newDataProperty.getStorageMedium());
+ Assert.assertEquals(1735689600000L,
newDataProperty.getCooldownTimeMs());
+ Assert.assertEquals("s3_policy",
newDataProperty.getStoragePolicy());
+ Assert.assertTrue(partitionInfo.getIsInMemory(newPartId));
+ }
+ }
+
+ @Test
+ public void testResetPartitionIdForRestoreInCloudMode() {
+ PartitionInfo partitionInfo = new PartitionInfo(PartitionType.RANGE);
+ long origPartId = 1000L;
+ DataProperty origDataProperty = new DataProperty(TStorageMedium.SSD,
1735689600000L, "s3_policy");
+ ReplicaAllocation origReplicaAlloc = new ReplicaAllocation((short) 3);
+ partitionInfo.addPartition(origPartId, origDataProperty,
origReplicaAlloc, true, true);
+
+ Map<Long, Long> partitionIdMap = Maps.newHashMap();
+ long newPartId = 2000L;
+ partitionIdMap.put(newPartId, origPartId);
+
+ ReplicaAllocation cloudReplicaAlloc = new ReplicaAllocation((short) 1);
+
+ try (MockedStatic<Config> mockedConfig =
Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS);
+ MockedStatic<PropertyAnalyzer> mockedPA =
+ Mockito.mockStatic(PropertyAnalyzer.class,
Mockito.CALLS_REAL_METHODS)) {
+ mockedConfig.when(Config::isCloudMode).thenReturn(true);
+ mockedConfig.when(Config::isNotCloudMode).thenReturn(false);
+ mockedPA.when(PropertyAnalyzer::getInstance).thenReturn(new
CloudPropertyAnalyzer());
+
+ partitionInfo.resetPartitionIdForRestore(partitionIdMap,
cloudReplicaAlloc, false);
+ Assert.assertEquals((short) 1,
+
partitionInfo.getReplicaAllocation(newPartId).getTotalReplicaNum());
+ DataProperty newDataProperty =
partitionInfo.getDataProperty(newPartId);
+ Assert.assertEquals(DataProperty.DEFAULT_STORAGE_MEDIUM,
newDataProperty.getStorageMedium());
+ Assert.assertEquals(DataProperty.MAX_COOLDOWN_TIME_MS,
newDataProperty.getCooldownTimeMs());
+ Assert.assertEquals("", newDataProperty.getStoragePolicy());
+ Assert.assertTrue(newDataProperty.isMutable());
+ Assert.assertFalse(partitionInfo.getIsInMemory(newPartId));
+ }
+ }
+
@Test
public void testBuildVariantEnableFlattenNestedWithLegacyPropertyKey()
throws IOException {
Map<String, String> properties = Maps.newHashMap();
diff --git
a/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
b/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
index 9194984c854..1b973bf419e 100644
---
a/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
+++
b/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy
@@ -89,7 +89,11 @@ suite("test_show_create_table_with_storage_policy") {
String createSql = ret[0][1]
ret = sql """ SHOW PARTITIONS FROM ${tableName} """
- assertEquals(ret[9][12], storagePolicyName)
+ if (!isCloudMode()) {
+ assertEquals(ret[9][12], storagePolicyName)
+ } else {
+ assertEquals(ret[9][12], "")
+ }
sql """ DROP TABLE IF EXISTS ${tableName} """
// create table successfully with stmt from show create table
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
