This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1866967e048 [fix](regression) Adjust large TTL cache regression case
(#64789)
1866967e048 is described below
commit 1866967e0487f1b8d01d59720e32a302fa7e1110
Author: zhengyu <[email protected]>
AuthorDate: Mon Jul 6 11:18:52 2026 +0800
[fix](regression) Adjust large TTL cache regression case (#64789)
### What problem does this PR solve?
Problem Summary: The cloud TTL regression case used Long.MAX_VALUE for
file_cache_ttl_seconds and expected the loaded data to enter TTL cache.
The current product behavior allows this value in FE, while BE overflow
protection may convert the effective expiration to 0, causing the cache
entry to become NORMAL and making the test assert behavior that is
already classified as a product bug. This PR adjusts the case to use a
very large TTL value that stays below the runtime overflow boundary, and
adds an information_schema.file_cache_info type check so the case
verifies that the configured TTL cache semantics are actually effective.
---
.../cloud/alter/CloudSchemaChangeHandler.java | 9 ++++-
.../apache/doris/common/util/PropertyAnalyzer.java | 32 +++++++++++------
.../commands/info/ModifyTablePropertiesOp.java | 2 ++
.../apache/doris/common/PropertyAnalyzerTest.java | 38 ++++++++++++++++++++
.../cloud_p0/cache/ttl/alter_ttl_max_int64.groovy | 41 ++++++++++++++++++++--
5 files changed, 108 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
index c2155963273..47e6fbf0cbe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
@@ -30,6 +30,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.rpc.MetaServiceProxy;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
@@ -65,7 +66,13 @@ public class CloudSchemaChangeHandler extends
SchemaChangeHandler {
UpdatePartitionMetaParam param = new UpdatePartitionMetaParam();
if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
- long ttlSeconds =
Long.parseLong(properties.get(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS));
+ long ttlSeconds;
+ try {
+ ttlSeconds = PropertyAnalyzer.analyzeFileCacheTtlSeconds(
+
properties.get(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS));
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
olapTable.readLock();
try {
if (ttlSeconds == olapTable.getTTLSeconds()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index b27db96bbe1..305fcf19064 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -130,6 +130,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_INMEMORY = "in_memory";
public static final String PROPERTIES_FILE_CACHE_TTL_SECONDS =
"file_cache_ttl_seconds";
+ public static final long MAX_FILE_CACHE_TTL_SECONDS = Long.MAX_VALUE / 2L;
// _auto_bucket can only set in create table stmt rewrite bucket and can
not be changed
public static final String PROPERTIES_AUTO_BUCKET = "_auto_bucket";
@@ -623,20 +624,31 @@ public class PropertyAnalyzer {
public static long analyzeTTL(Map<String, String> properties) throws
AnalysisException {
long ttlSeconds = 0;
if (properties != null &&
properties.containsKey(PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
- String ttlSecondsStr =
properties.get(PROPERTIES_FILE_CACHE_TTL_SECONDS);
- try {
- ttlSeconds = Long.parseLong(ttlSecondsStr);
- if (ttlSeconds < 0) {
- throw new NumberFormatException();
- }
- } catch (NumberFormatException e) {
- throw new AnalysisException("The value " + ttlSecondsStr + "
formats error or is out of range "
- + "(0 < integer < Long.MAX_VALUE)");
- }
+ ttlSeconds =
analyzeFileCacheTtlSeconds(properties.get(PROPERTIES_FILE_CACHE_TTL_SECONDS));
}
return ttlSeconds;
}
+ public static long analyzeFileCacheTtlSeconds(String ttlSecondsStr) throws
AnalysisException {
+ long ttlSeconds;
+ try {
+ ttlSeconds = Long.parseLong(ttlSecondsStr);
+ } catch (NumberFormatException e) {
+ throw invalidFileCacheTtlSecondsException(ttlSecondsStr);
+ }
+ if (ttlSeconds < 0 || ttlSeconds > MAX_FILE_CACHE_TTL_SECONDS) {
+ throw invalidFileCacheTtlSecondsException(ttlSecondsStr);
+ }
+ return ttlSeconds;
+ }
+
+ private static AnalysisException
invalidFileCacheTtlSecondsException(String ttlSecondsStr) {
+ return new AnalysisException("The value " + ttlSecondsStr + " formats
error or is out of range "
+ + "(0 <= integer <= " + MAX_FILE_CACHE_TTL_SECONDS + ").
Larger values may overflow in BE "
+ + "and change TTL cache to normal cache; please use " +
MAX_FILE_CACHE_TTL_SECONDS
+ + " or a smaller value.");
+ }
+
public static int analyzePartitionRetentionCount(Map<String, String>
properties) throws AnalysisException {
int retentionCount = -1;
if (properties != null &&
properties.containsKey(PROPERTIES_PARTITION_RETENTION_COUNT)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
index 00f711c6584..6e24f648490 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
@@ -324,6 +324,8 @@ public class ModifyTablePropertiesOp extends AlterTableOp {
PropertyAnalyzer.analyzeGroupCommitMode(properties, false);
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
+ PropertyAnalyzer.analyzeFileCacheTtlSeconds(
+
properties.get(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS));
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT))
{
// do a check here for valid value
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 81878c9b1a3..2f196d8554f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -145,6 +145,44 @@ public class PropertyAnalyzerTest {
Assert.assertEquals(0.05,
PropertyAnalyzer.analyzeBloomFilterFpp(properties), 0.0001);
}
+ @Test
+ public void testAnalyzeFileCacheTtlSeconds() throws AnalysisException {
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
"0");
+ Assert.assertEquals(0L, PropertyAnalyzer.analyzeTTL(properties));
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
+ String.valueOf(PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS));
+ Assert.assertEquals(PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS,
PropertyAnalyzer.analyzeTTL(properties));
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
+ String.valueOf(PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS +
1L));
+ try {
+ PropertyAnalyzer.analyzeTTL(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("please use "
+ + PropertyAnalyzer.MAX_FILE_CACHE_TTL_SECONDS));
+ }
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
String.valueOf(Long.MAX_VALUE));
+ try {
+ PropertyAnalyzer.analyzeTTL(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Larger values may
overflow in BE"));
+ Assert.assertTrue(e.getMessage().contains("please use"));
+ }
+
+ properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
"invalid");
+ try {
+ PropertyAnalyzer.analyzeTTL(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("formats error or is out
of range"));
+ }
+ }
+
@Test
public void testStorageMedium() throws AnalysisException {
long tomorrowTs = System.currentTimeMillis() / 1000 + 86400;
diff --git
a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
index 9ffde534860..afb67b123f5 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_max_int64.groovy
@@ -19,7 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_ttl_max_int64") {
sql """ use @regression_cluster_name1 """
- def ttlProperties = """
PROPERTIES("file_cache_ttl_seconds"="9223372036854775807") """
+ long safeLargeTtlSeconds = Long.MAX_VALUE / 2L
+ def ttlProperties = """
PROPERTIES("file_cache_ttl_seconds"="${safeLargeTtlSeconds}") """
String[][] backends = sql """ show backends """
String backendId;
def backendIdToBackendIP = [:]
@@ -47,7 +48,6 @@ suite("test_ttl_max_int64") {
}
}
- def tables = [customer_ttl: 15000000]
def s3BucketName = getS3BucketName()
def s3WithProperties = """WITH S3 (
|"AWS_ACCESS_KEY" = "${getS3AK()}",
@@ -93,12 +93,47 @@ suite("test_ttl_max_int64") {
}
}
+ def getTabletIds = { String table ->
+ def tablets = sql "show tablets from ${table}"
+ assertTrue(tablets.size() > 0, "No tablets found for table ${table}")
+ tablets.collect { it[0] as Long }
+ }
+
+ def waitForFileCacheType = { List<Long> tabletIds, String expectedType,
long timeoutMs = 60000L, long intervalMs = 2000L ->
+ long start = System.currentTimeMillis()
+ while (System.currentTimeMillis() - start < timeoutMs) {
+ boolean allMatch = true
+ for (Long tabletId in tabletIds) {
+ def rows = sql "select type from
information_schema.file_cache_info where tablet_id = ${tabletId}"
+ if (rows.isEmpty()) {
+ logger.warn("file_cache_info is empty for tablet
${tabletId} while waiting for ${expectedType}")
+ allMatch = false
+ break
+ }
+ def mismatch = rows.find { row ->
!row[0]?.toString()?.equalsIgnoreCase(expectedType) }
+ if (mismatch) {
+ logger.info("tablet ${tabletId} has cache types
${rows.collect { it[0] }} while waiting for ${expectedType}")
+ allMatch = false
+ break
+ }
+ }
+ if (allMatch) {
+ logger.info("All file cache entries for tablets ${tabletIds}
are ${expectedType}")
+ return
+ }
+ sleep(intervalMs)
+ }
+ assertTrue(false, "Timeout waiting for file_cache_info type
${expectedType} for tablets ${tabletIds}")
+ }
+
clearFileCache.call() {
respCode, body -> {}
}
sleep(30000)
load_customer_once("customer_ttl")
+ def tabletIds = getTabletIds.call("customer_ttl")
+ waitForFileCacheType.call(tabletIds, "ttl")
sleep(30000) // 30s
getMetricsMethod.call() {
respCode, body ->
@@ -113,7 +148,7 @@ suite("test_ttl_max_int64") {
continue
}
def i = line.indexOf(' ')
- assertTrue(line.substring(i).toLong() > 838860800)
+ assertTrue(line.substring(i).trim().toLong() > 838860800)
flag1 = true
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
