This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 0133b003bb1 branch-4.0:[fix](regression) Adjust large TTL cache
regression case(pick#64789) (#66127)
0133b003bb1 is described below
commit 0133b003bb1a32004426854f0ad0123a214b92a6
Author: zhengyu <[email protected]>
AuthorDate: Tue Jul 28 23:08:17 2026 +0800
branch-4.0:[fix](regression) Adjust large TTL cache regression
case(pick#64789) (#66127)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #64789
---
.../cloud/alter/CloudSchemaChangeHandler.java | 9 ++++-
.../apache/doris/common/util/PropertyAnalyzer.java | 32 +++++++++++------
.../commands/info/ModifyTablePropertiesOp.java | 2 ++
.../apache/doris/common/PropertyAnalyzerTest.java | 38 ++++++++++++++++++++
.../cloud_p0/cache/ttl/alter_ttl_max_int64.groovy | 41 ++++++++++++++++++++--
5 files changed, 108 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
index 1b4c1899845..30bceb03db4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
@@ -30,6 +30,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.rpc.MetaServiceProxy;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
@@ -65,7 +66,13 @@ public class CloudSchemaChangeHandler extends
SchemaChangeHandler {
UpdatePartitionMetaParam param = new UpdatePartitionMetaParam();
if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
- long ttlSeconds =
Long.parseLong(properties.get(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS));
+ long ttlSeconds;
+ try {
+ ttlSeconds = PropertyAnalyzer.analyzeFileCacheTtlSeconds(
+
properties.get(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS));
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
olapTable.readLock();
try {
if (ttlSeconds == olapTable.getTTLSeconds()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 962dd6751c3..d4db907582d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -126,6 +126,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_INMEMORY = "in_memory";
public static final String PROPERTIES_FILE_CACHE_TTL_SECONDS =
"file_cache_ttl_seconds";
+ public static final long MAX_FILE_CACHE_TTL_SECONDS = Long.MAX_VALUE / 2L;
// _auto_bucket can only set in create table stmt rewrite bucket and can
not be changed
public static final String PROPERTIES_AUTO_BUCKET = "_auto_bucket";
@@ -606,20 +607,31 @@ public class PropertyAnalyzer {
public static long analyzeTTL(Map<String, String> properties) throws
AnalysisException {
long ttlSeconds = 0;
if (properties != null &&
properties.containsKey(PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
- String ttlSecondsStr =
properties.get(PROPERTIES_FILE_CACHE_TTL_SECONDS);
- try {
- ttlSeconds = Long.parseLong(ttlSecondsStr);
- if (ttlSeconds < 0) {
- throw new NumberFormatException();
- }
- } catch (NumberFormatException e) {
- throw new AnalysisException("The value " + ttlSecondsStr + "
formats error or is out of range "
- + "(0 < integer < Long.MAX_VALUE)");
- }
+ ttlSeconds =
analyzeFileCacheTtlSeconds(properties.get(PROPERTIES_FILE_CACHE_TTL_SECONDS));
}
return ttlSeconds;
}
+ public static long analyzeFileCacheTtlSeconds(String ttlSecondsStr) throws
AnalysisException {
+ long ttlSeconds;
+ try {
+ ttlSeconds = Long.parseLong(ttlSecondsStr);
+ } catch (NumberFormatException e) {
+ throw invalidFileCacheTtlSecondsException(ttlSecondsStr);
+ }
+ if (ttlSeconds < 0 || ttlSeconds > MAX_FILE_CACHE_TTL_SECONDS) {
+ throw invalidFileCacheTtlSecondsException(ttlSecondsStr);
+ }
+ return ttlSeconds;
+ }
+
+ private static AnalysisException
invalidFileCacheTtlSecondsException(String ttlSecondsStr) {
+ return new AnalysisException("The value " + ttlSecondsStr + " formats
error or is out of range "
+ + "(0 <= integer <= " + MAX_FILE_CACHE_TTL_SECONDS + ").
Larger values may overflow in BE "
+ + "and change TTL cache to normal cache; please use " +
MAX_FILE_CACHE_TTL_SECONDS
+ + " or a smaller value.");
+ }
+
public static int analyzePartitionRetentionCount(Map<String, String>
properties) throws AnalysisException {
int retentionCount = -1;
if (properties != null &&
properties.containsKey(PROPERTIES_PARTITION_RETENTION_COUNT)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
index 6376d24a46a..eea71384582 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
@@ -343,6 +343,8 @@ public class ModifyTablePropertiesOp extends AlterTableOp {
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
this.needTableStable = false;
+ PropertyAnalyzer.analyzeFileCacheTtlSeconds(
+
properties.get(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS));
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT))
{
// do a check here for valid value
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 926d803f931..586139dca78 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -143,6 +143,44 @@ public class PropertyAnalyzerTest {
Assert.assertEquals(0.05,
PropertyAnalyzer.analyzeBloomFilterFpp(properties), 0.0001);
}
+ @Test
+ public void testAnalyzeFileCacheTtlSeconds() throws AnalysisException {
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
"0");
+ Assert.assertEquals(0L, PropertyAnalyzer.analyzeTTL(properties));
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
+ String.valueOf(PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS));
+ Assert.assertEquals(PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS,
PropertyAnalyzer.analyzeTTL(properties));
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
+ String.valueOf(PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS +
1L));
+ try {
+ PropertyAnalyzer.analyzeTTL(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("please use "
+ + PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS));
+ }
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
String.valueOf(Long.MAX_VALUE));
+ try {
+ PropertyAnalyzer.analyzeTTL(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Larger values may
overflow in BE"));
+ Assert.assertTrue(e.getMessage().contains("please use"));
+ }
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
"invalid");
+ try {
+ PropertyAnalyzer.analyzeTTL(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("formats error or is out
of range"));
+ }
+ }
+
@Test
public void testStorageMedium() throws AnalysisException {
long tomorrowTs = System.currentTimeMillis() / 1000 + 86400;
diff --git
a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
index 9ffde534860..afb67b123f5 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
@@ -19,7 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_ttl_max_int64") {
sql """ use @regression_cluster_name1 """
- def ttlProperties = """
PROPERTIES("file_cache_ttl_seconds"="9223372036854775807") """
+ long safeLargeTtlSeconds = Long.MAX_VALUE / 2L
+ def ttlProperties = """
PROPERTIES("file_cache_ttl_seconds"="${safeLargeTtlSeconds}") """
String[][] backends = sql """ show backends """
String backendId;
def backendIdToBackendIP = [:]
@@ -47,7 +48,6 @@ suite("test_ttl_max_int64") {
}
}
- def tables = [customer_ttl: 15000000]
def s3BucketName = getS3BucketName()
def s3WithProperties = """WITH S3 (
|"AWS_ACCESS_KEY" = "${getS3AK()}",
@@ -93,12 +93,47 @@ suite("test_ttl_max_int64") {
}
}
+ def getTabletIds = { String table ->
+ def tablets = sql "show tablets from ${table}"
+ assertTrue(tablets.size() > 0, "No tablets found for table ${table}")
+ tablets.collect { it[0] as Long }
+ }
+
+ def waitForFileCacheType = { List<Long> tabletIds, String expectedType,
long timeoutMs = 60000L, long intervalMs = 2000L ->
+ long start = System.currentTimeMillis()
+ while (System.currentTimeMillis() - start < timeoutMs) {
+ boolean allMatch = true
+ for (Long tabletId in tabletIds) {
+ def rows = sql "select type from
information_schema.file_cache_info where tablet_id = ${tabletId}"
+ if (rows.isEmpty()) {
+ logger.warn("file_cache_info is empty for tablet
${tabletId} while waiting for ${expectedType}")
+ allMatch = false
+ break
+ }
+ def mismatch = rows.find { row ->
!row[0]?.toString()?.equalsIgnoreCase(expectedType) }
+ if (mismatch) {
+ logger.info("tablet ${tabletId} has cache types
${rows.collect { it[0] }} while waiting for ${expectedType}")
+ allMatch = false
+ break
+ }
+ }
+ if (allMatch) {
+ logger.info("All file cache entries for tablets ${tabletIds}
are ${expectedType}")
+ return
+ }
+ sleep(intervalMs)
+ }
+ assertTrue(false, "Timeout waiting for file_cache_info type
${expectedType} for tablets ${tabletIds}")
+ }
+
clearFileCache.call() {
respCode, body -> {}
}
sleep(30000)
load_customer_once("customer_ttl")
+ def tabletIds = getTabletIds.call("customer_ttl")
+ waitForFileCacheType.call(tabletIds, "ttl")
sleep(30000) // 30s
getMetricsMethod.call() {
respCode, body ->
@@ -113,7 +148,7 @@ suite("test_ttl_max_int64") {
continue
}
def i = line.indexOf(' ')
- assertTrue(line.substring(i).toLong() > 838860800)
+ assertTrue(line.substring(i).trim().toLong() > 838860800)
flag1 = true
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
