This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fd0c161081 [enhance](ColdHeatSeparation) forbid change storage policy
to another one with different storage resource (#22519)
fd0c161081 is described below
commit fd0c1610812cd033cab3a73626b56e29830ee951
Author: AlexYue <[email protected]>
AuthorDate: Thu Aug 10 16:32:09 2023 +0800
[enhance](ColdHeatSeparation) forbid change storage policy to another one
with different storage resource (#22519)
---
.../apache/doris/common/util/PropertyAnalyzer.java | 26 +++++++++++++++++++++-
.../java/org/apache/doris/alter/AlterTest.java | 11 ++++++++-
2 files changed, 35 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 1b8d45f569..f916d13336 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -190,7 +190,11 @@ public class PropertyAnalyzer {
TStorageMedium storageMedium = oldDataProperty.getStorageMedium();
long cooldownTimestamp = oldDataProperty.getCooldownTimeMs();
- String newStoragePolicy = oldDataProperty.getStoragePolicy();
+ final String oldStoragePolicy = oldDataProperty.getStoragePolicy();
+ // When we create one table with table's property set storage policy,
+ // the properties wouldn't contain storage policy so the
hasStoragePolicy would be false,
+ // then we would just set the partition's storage policy the same as
the table's
+ String newStoragePolicy = oldStoragePolicy;
boolean hasStoragePolicy = false;
boolean storageMediumSpecified = false;
@@ -252,6 +256,26 @@ public class PropertyAnalyzer {
}
StoragePolicy storagePolicy = (StoragePolicy) policy;
+ // Consider a scenario where if cold data has already been
uploaded to resource A,
+ // and the user attempts to modify the policy to upload it to
resource B,
+ // the data needs to be transferred from A to B.
+ // However, Doris currently does not support cross-bucket data
transfer, therefore,
+ // changing the policy to a different policy with different
resource is disabled.
+ // As for the case where the resource is the same, modifying the
cooldown time is allowed,
+ // as this will not affect the already cooled data, but only the
new data after modifying the policy.
+ if (null != oldStoragePolicy &&
!oldStoragePolicy.equals(newStoragePolicy)) {
+ // check remote storage policy
+ StoragePolicy oldPolicy =
StoragePolicy.ofCheck(oldStoragePolicy);
+ Policy p =
Env.getCurrentEnv().getPolicyMgr().getPolicy(oldPolicy);
+ if ((p instanceof StoragePolicy)) {
+ String newResource = storagePolicy.getStorageResource();
+ String oldResource = ((StoragePolicy)
p).getStorageResource();
+ if (!newResource.equals(oldResource)) {
+ throw new AnalysisException("currently do not support
change origin "
+ + "storage policy to another one with
different resource: ");
+ }
+ }
+ }
// check remote storage cool down timestamp
if (storagePolicy.getCooldownTimestampMs() != -1) {
if (storagePolicy.getCooldownTimestampMs() <= currentTimeMs) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
index d18f5a57f8..af70ca249a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
@@ -160,6 +160,10 @@ public class AlterTest {
"CREATE STORAGE POLICY testPolicy2\n" + "PROPERTIES(\n" + "
\"storage_resource\" = \"remote_s3\",\n"
+ " \"cooldown_ttl\" = \"1\"\n" + ");");
+ createRemoteStoragePolicy(
+ "CREATE STORAGE POLICY testPolicyAnotherResource\n" +
"PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3_1\",\n"
+ + " \"cooldown_ttl\" = \"1\"\n" + ");");
+
createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1
date,\n" + " k2 int,\n" + " v1 int sum\n"
+ ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION
p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01'),\n"
@@ -597,11 +601,16 @@ public class AlterTest {
}
Assert.assertEquals(oldDataProperty,
tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
- // alter remote_storage
+ // alter remote_storage to one not exist policy
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set
('storage_policy' = 'testPolicy3')";
alterTable(stmt, true);
Assert.assertEquals(oldDataProperty,
tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
+ // alter remote_storage to one another one which points to another
resource
+ stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set
('storage_policy' = 'testPolicyAnotherResource')";
+ alterTable(stmt, true);
+ Assert.assertEquals(oldDataProperty,
tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
+
// alter recover to old state
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set
("
+ "'storage_medium' = 'SSD', "
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
