This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new e6eadf2fc53 branch-4.2: [fix](regression) Fix test_ttl_lru_evict and
test_stale_rowset in cloud file cache #69847 (#70116)
e6eadf2fc53 is described below
commit e6eadf2fc53bfd75d17679bc4b0c12591d02f5e0
Author: Xin Liao <[email protected]>
AuthorDate: Wed Nov 4 08:33:36 2026 +0800
branch-4.2: [fix](regression) Fix test_ttl_lru_evict and test_stale_rowset
in cloud file cache #69847 (#70116)
### What problem does this PR solve?
Issue Number: None
Related PR: #69847
Problem Summary:
Backport #69847 to `branch-4.2`.
`test_ttl_lru_evict` could take longer than its TTL to fill a large
cache, so the test now temporarily reduces cache capacity and restores
it afterward. `test_stale_rowset` now polls for stale-rowset cache
reclamation instead of relying on a single fixed wait.
The backport includes prerequisite changes in the same TTL test from
#69705: the metric helper, bounded loading, TTL duration, and expiry
polling. It does not include the BE changes from #69705. Both final
regression files match upstream #69847 exactly.
### Release note
None
### Check List (For Author)
- Test: Groovy 4.0.19 syntax parsing/conversion and `git diff --check`
passed. The regression entry point could not start because the local
Mac's Xcode license has not been accepted; no runtime regression pass is
claimed. CI `run buildall` is requested separately after PR creation.
- Behavior changed: No. Regression tests only.
- Does this need documentation: No.
---
.../cache/compaction/test_stale_rowset.groovy | 19 ++-
.../cloud_p0/cache/ttl/test_ttl_lru_evict.groovy | 164 +++++++++++----------
2 files changed, 103 insertions(+), 80 deletions(-)
diff --git
a/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy
b/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy
index bc65e78584a..b1ac9ff5892 100644
--- a/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy
+++ b/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy
@@ -171,13 +171,20 @@ suite("test_stale_rowset") {
select count(*) from ${tableName};
"""
- sleep(60000);
- def backendIdToAfterCompactionCacheSize = getCurCacheSize()
- for (String[] backend in backends) {
- if (backend[9].equals("true") &&
backend[19].contains("regression_cluster_name1")) {
- assertTrue(backendIdToAfterLoadCacheSize.get(backend[0]) >
- backendIdToAfterCompactionCacheSize.get(backend[0]))
+ // The cache of the input rowsets is removed only when the BE sweeps the
stale rowsets, and the
+ // file_cache_cache_size metric is refreshed every few seconds after that.
So poll until it
+ // shrinks instead of reading it once after a fixed wait.
+ def backendIdToAfterCompactionCacheSize = [:]
+ def shrunk = false
+ long deadline = System.currentTimeMillis() + 180000
+ while (!shrunk && System.currentTimeMillis() < deadline) {
+ sleep(5000)
+ backendIdToAfterCompactionCacheSize = getCurCacheSize()
+ shrunk = backendIdToAfterLoadCacheSize.every { id, size ->
+ size > backendIdToAfterCompactionCacheSize.get(id)
}
}
+ assertTrue(shrunk, "file cache did not shrink after compaction, after
load: " +
+ "${backendIdToAfterLoadCacheSize}, after compaction:
${backendIdToAfterCompactionCacheSize}")
sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
}
diff --git
a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy
b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy
index f452d071e04..32703fb907b 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy
@@ -58,7 +58,10 @@ suite("test_ttl_lru_evict") {
logger.info("getS3SK:${getS3SK()}");
logger.info("getS3Endpoint:${getS3Endpoint()}");
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="150") """
+ // Long enough that the first loads are still TTL when the TTL queue
overflows. How many loads
+ // that takes depends on the cache capacity of the environment.
+ def ttlSeconds = 300
+ def ttlProperties = """
PROPERTIES("file_cache_ttl_seconds"="${ttlSeconds}") """
String[][] backends = sql """ show backends """
String backendId;
def backendIdToBackendIP = [:]
@@ -206,65 +209,83 @@ suite("test_ttl_lru_evict") {
}
}
- clearFileCache.call() {
- respCode, body -> {}
- }
-
- long ttl_cache_evict_size_begin = 0;
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_evict_size")) {
- if (line.startsWith("#")) {
+ // Values of every metric line whose name contains the given name, one per
cache path.
+ def getMetricValues = { String name ->
+ def values = []
+ getMetricsMethod.call() {
+ respCode, body ->
+ assertEquals("${respCode}".toString(), "200")
+ for (String line in "${body}".toString().split('\n')) {
+ if (line.startsWith("#") || !line.contains(name)) {
continue
}
- def i = line.indexOf(' ')
- ttl_cache_evict_size_begin = line.substring(i).toLong()
- flag1 = true
+ values.add(line.substring(line.indexOf('
')).trim().toLong())
}
- }
- assertTrue(flag1)
+ }
+ assertTrue(!values.isEmpty(), "metric ${name} not found")
+ return values
}
- // load for MANY times to this dup table to make cache full enough to evict
- // It will takes huge amount of time, so it should be p1/p2 level case.
- // But someone told me it is the right place. Never mind just do it!
+ clearFileCache.call() {
+ respCode, body -> {}
+ }
- for (int i = 0; i < 10; i++) {
- load_customer_once("customer_ttl")
+ long ttl_cache_evict_size_begin =
getMetricValues("ttl_cache_evict_size")[0]
+ long ttl_cache_size_begin = getMetricValues("ttl_cache_size")[0]
+ def path_capacities = getMetricValues("file_cache_capacity")
+ // The capacity is reset for all cache paths at once, so it can only be
restored when they
+ // have the same capacity.
+ assertEquals(1, path_capacities.unique(false).size(),
+ "cache paths have different capacities: ${path_capacities}")
+ long path_capacity = path_capacities[0]
+ long cache_capacity = path_capacity
+
+ // Sets the capacity of every cache path of the BE.
+ def resetCapacity = { long capacity ->
+ def (code, out, err) = curl("GET", ("http://"; +
backendIdToBackendIP.get(backendId) + ":" +
+ backendIdToBackendHttpPort.get(backendId) +
+ "/api/file_cache?op=reset&capacity=${capacity}").toString())
+ assertEquals(0, code)
+ assertEquals([capacity] * path_capacities.size(),
getMetricValues("file_cache_capacity"))
+ cache_capacity = capacity
+ logger.info("file cache capacity of each path reset to ${capacity}")
}
- sleep(10000) // 10s
- // first, we test whether ttl evict actively
- long ttl_cache_evict_size_end = 0;
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_evict_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- ttl_cache_evict_size_end = line.substring(i).toLong()
- flag1 = true
- }
+ onFinish {
+ if (cache_capacity != path_capacity) {
+ resetCapacity(path_capacity)
+ }
+ }
+
+ // first, we test whether ttl evict actively.
+ // Load to this dup table until the TTL queue overflows. TTL blocks are
evicted only when the
+ // whole cache is full, and filling a large cache can take longer than the
TTL. Once the TTL
+ // expires, the blocks become NORMAL and are no longer evicted as TTL. So
the cache is shrunk
+ // to a few loads after the first one.
+ long evicted = 0
+ long loads = 0
+ long max_loads = 10
+ while (evicted <= 238860800 && loads < max_loads) {
+ load_customer_once("customer_ttl")
+ loads++
+ if (loads == 1) {
+ // ttl_cache_size is refreshed by the cache monitor, so wait for a
refresh that
+ // includes the whole load.
+
sleep(get_be_param("file_cache_background_monitor_interval_ms").toLong() + 1000)
+ long bytes_per_load = getMetricValues("ttl_cache_size")[0] -
ttl_cache_size_begin
+ assertTrue(bytes_per_load > 0, "the first load did not add any TTL
cache")
+ if (bytes_per_load * 4 < path_capacity) {
+ resetCapacity(bytes_per_load * 4)
}
- assertTrue(flag1)
+ // This many loads overflow the cache for sure.
+ max_loads = Math.max(max_loads,
cache_capacity.intdiv(bytes_per_load) + 3)
+ logger.info("cache capacity: ${cache_capacity}, TTL bytes per
load: ${bytes_per_load}, " +
+ "max loads: ${max_loads}")
+ }
+ evicted = getMetricValues("ttl_cache_evict_size")[0] -
ttl_cache_evict_size_begin
+ logger.info("ttl evict diff after ${loads} loads: ${evicted}")
}
- // Note: this only applies when the case is run
- // sequentially, coz we don't know what other cases are
- // doing with TTL cache
- logger.info("ttl evict diff:" + (ttl_cache_evict_size_end -
ttl_cache_evict_size_begin).toString())
- assertTrue((ttl_cache_evict_size_end - ttl_cache_evict_size_begin) >
238860800)
+ assertTrue(evicted > 238860800,
+ "TTL cache evicted only ${evicted} bytes after ${loads} loads,
capacity ${cache_capacity}")
// then we test skip_cache count when doing query when ttl cache is full
// we expect it to be rare
@@ -322,31 +343,26 @@ suite("test_ttl_lru_evict") {
logger.info("skip cache diff:" + (skip_cache_count_end -
skip_cache_count_begin).toString())
assertTrue((skip_cache_count_end - skip_cache_count_begin) < 1000000)
+ // The rest of the case does not need a full cache.
+ if (cache_capacity != path_capacity) {
+ resetCapacity(path_capacity)
+ }
+
// finally, we test whether LRU queue clean itself up when all ttl
// cache evict after expiration
- sleep(200000)
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_lru_queue_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- // all ttl will expire, so the ttl LRU queue set to 0
- // Note: this only applies when the case is run
- // sequentially, coz we don't know what other cases are
- // doing with TTL cache
- assertEquals(line.substring(i).toLong(), 0)
- flag1 = true
- }
- }
- assertTrue(flag1)
+ // Note: this only applies when the case is run
+ // sequentially, coz we don't know what other cases are
+ // doing with TTL cache
+ long ttl_queue_size = -1
+ long deadline = System.currentTimeMillis() + (ttlSeconds + 120) * 1000L
+ while (System.currentTimeMillis() < deadline) {
+ // all ttl will expire, so the ttl LRU queue set to 0
+ ttl_queue_size = getMetricValues("ttl_cache_lru_queue_size")[0]
+ if (ttl_queue_size == 0) {
+ break
+ }
+ sleep(10000)
}
+ assertEquals(0, ttl_queue_size)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
