This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 4025a6d23 feat(config): add RocksDB periodic compaction controls
(#3170)
4025a6d23 is described below
commit 4025a6d23698b6bd5383511f259d10988fe7dc0e
Author: sryan yuan <[email protected]>
AuthorDate: Tue Sep 16 17:31:34 2025 +0800
feat(config): add RocksDB periodic compaction controls (#3170)
Introduces three new configuration fields for RocksDB periodic
compaction management:
1. `rocksdb.periodic_compaction_seconds` - Sets compaction frequency
2. `rocksdb.ttl` - Enables TTL-based compaction
3. `rocksdb.daily_offpeak_time_utc` - Configures off-peak compaction
window
---------
Co-authored-by: yxj25245 <[email protected]>
Co-authored-by: Twice <[email protected]>
Co-authored-by: mwish <[email protected]>
Co-authored-by: Twice <[email protected]>
---
kvrocks.conf | 51 ++++++++++++++++++++++++++++
src/config/config.cc | 7 ++++
src/config/config.h | 6 ++++
src/storage/storage.cc | 3 ++
tests/gocase/unit/config/config_test.go | 59 +++++++++++++++++++++++++++++++++
5 files changed, 126 insertions(+)
diff --git a/kvrocks.conf b/kvrocks.conf
index a4754793b..fb34204ba 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -1124,5 +1124,56 @@ rocksdb.max_compaction_bytes 0
# Default: 0
rocksdb.sst_file_delete_rate_bytes_per_sec 0
+# Enable RocksDB periodic compaction to force full compaction of SST files
older than the specified time (in seconds).
+# If a compaction filter is registered, it will be applied during these
compactions.
+# Set to 0 to disable this feature.
+#
+# Default: 18446744073709551614 (0xFFFFFFFFFFFFFFFE, UINT64_MAX - 1), a
special value indicating RocksDB-controlled behavior.
+# Currently, RocksDB interprets this default as 30 days (2592000 seconds).
+#
+# Typical use cases:
+# - Enforcing data cleanup via compaction filters (e.g., TTL expiration)
+# - Automatically refreshing data encoding/compression formats without manual
intervention
+#
+# Reference:
https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#periodic-compaction
+#
+# rocksdb.periodic_compaction_seconds 2592000
+
+# Enable RocksDB Time-to-Live (TTL) to automatically schedule compaction for
SST files containing expired data.
+# - Files containing data older than the TTL (in seconds) will be prioritized
for background compaction.
+# - Requires a registered compaction filter (e.g., TTL filter) to identify and
remove expired entries.
+# - Set to 0 to disable TTL-based compaction.
+#
+# Default: 18446744073709551614 (0xFFFFFFFFFFFFFFFE, UINT64_MAX - 1),
delegating control to RocksDB.
+# Current RocksDB behavior interprets this default as 30 days (2592000
seconds).
+#
+# Use cases:
+# - Automatic expiration of ephemeral data (e.g., session tokens, temporary
logs)
+# - Lifecycle management for time-series datasets
+#
+# Reference: https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#ttl
+#
+# rocksdb.ttl 2592000
+
+# Schedule RocksDB periodic compactions during daily off-peak windows to
reduce operational impact.
+#
+# Requirements:
+# - Periodic compaction must be enabled (`periodic-compaction-seconds > 0`)
+# - Time format: "HH:MM-HH:MM" in UTC (e.g., "02:00-04:30" for a 2.5-hour
window)
+# - Empty string disables off-peak scheduling
+#
+# Behavior:
+# - RocksDB proactively triggers periodic compactions during the specified
off-peak window
+# - Compactions are optimized to complete before the next peak period begins
+#
+# Default: "" (disabled)
+#
+# Typical use cases:
+# - Minimize compaction I/O during business hours for latency-sensitive
workloads
+# - Align resource-heavy operations with maintenance windows
+#
+# Reference:
https://github.com/facebook/rocksdb/wiki/Daily-Off%E2%80%90peak-Time-Option
+rocksdb.daily_offpeak_time_utc ""
+
################################ NAMESPACE
#####################################
# namespace.test change.me
diff --git a/src/config/config.cc b/src/config/config.cc
index 746932dbe..c3b3400f7 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -307,6 +307,10 @@ Config::Config() {
{"rocksdb.max_compaction_bytes", false, new
Int64Field(&rocks_db.max_compaction_bytes, 0, 0, INT64_MAX)},
{"rocksdb.sst_file_delete_rate_bytes_per_sec", false,
new Int64Field(&rocks_db.sst_file_delete_rate_bytes_per_sec, 0, 0,
INT64_MAX)},
+ {"rocksdb.periodic_compaction_seconds", false,
+ new UInt64Field(&rocks_db.periodic_compaction_seconds,
kDefaultRocksdbPeriodicCompactionSeconds, 0, UINT64_MAX)},
+ {"rocksdb.ttl", false, new UInt64Field(&rocks_db.ttl,
kDefaultRocksdbTTL, 0, UINT64_MAX)},
+ {"rocksdb.daily_offpeak_time_utc", false, new
StringField(&rocks_db.daily_offpeak_time_utc, "")},
/* rocksdb write options */
{"rocksdb.write_options.sync", true, new
YesNoField(&rocks_db.write_options.sync, false)},
@@ -732,6 +736,9 @@ void Config::initFieldCallback() {
srv->storage->SetSstFileDeleteRateBytesPerSecond(rocks_db.sst_file_delete_rate_bytes_per_sec);
return Status::OK();
}},
+ {"rocksdb.periodic_compaction_seconds", set_cf_option_cb},
+ {"rocksdb.ttl", set_cf_option_cb},
+ {"rocksdb.daily_offpeak_time_utc", set_db_option_cb},
{"rocksdb.level0_slowdown_writes_trigger",
[this, &set_cf_option_cb](Server *srv, const std::string &k,
[[maybe_unused]] const std::string &v) ->
Status {
diff --git a/src/config/config.h b/src/config/config.h
index 30527eca6..c56a99f0b 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -57,6 +57,9 @@ constexpr const uint32_t kDefaultPort = 6666;
constexpr const char *kDefaultNamespace = "__namespace";
constexpr int KVROCKS_MAX_LSM_LEVEL = 7;
+constexpr const uint64_t kDefaultRocksdbTTL = UINT64_MAX - 1;
+constexpr const uint64_t kDefaultRocksdbPeriodicCompactionSeconds = UINT64_MAX
- 1;
+
const std::vector<ConfigEnum<spdlog::level::level_enum>> log_levels{
{"debug", spdlog::level::debug}, {"info", spdlog::level::info},
{"warning", spdlog::level::warn},
{"error", spdlog::level::err}, {"fatal", spdlog::level::critical},
@@ -243,6 +246,9 @@ struct Config {
bool partition_filters;
int64_t max_compaction_bytes;
int64_t sst_file_delete_rate_bytes_per_sec = 0;
+ uint64_t periodic_compaction_seconds =
kDefaultRocksdbPeriodicCompactionSeconds;
+ uint64_t ttl = kDefaultRocksdbTTL;
+ std::string daily_offpeak_time_utc;
struct WriteOptions {
bool sync;
diff --git a/src/storage/storage.cc b/src/storage/storage.cc
index 1138f2ace..3fb83de52 100644
--- a/src/storage/storage.cc
+++ b/src/storage/storage.cc
@@ -219,6 +219,9 @@ rocksdb::Options Storage::InitRocksDBOptions() {
options.level_compaction_dynamic_level_bytes =
config_->rocks_db.level_compaction_dynamic_level_bytes;
options.max_background_jobs = config_->rocks_db.max_background_jobs;
options.max_compaction_bytes =
static_cast<uint64_t>(config_->rocks_db.max_compaction_bytes);
+ options.periodic_compaction_seconds =
config_->rocks_db.periodic_compaction_seconds;
+ options.ttl = config_->rocks_db.ttl;
+ options.daily_offpeak_time_utc = config_->rocks_db.daily_offpeak_time_utc;
// avoid blocking io on iteration
// see https://github.com/facebook/rocksdb/wiki/IO#avoid-blocking-io
diff --git a/tests/gocase/unit/config/config_test.go
b/tests/gocase/unit/config/config_test.go
index 3287318ea..be3276c3c 100644
--- a/tests/gocase/unit/config/config_test.go
+++ b/tests/gocase/unit/config/config_test.go
@@ -22,6 +22,7 @@ package config
import (
"context"
"log"
+ "math"
"os"
"path/filepath"
"sort"
@@ -395,3 +396,61 @@ func TestConfigSstFileDeleteRateBytesPerSec(t *testing.T) {
require.EqualValues(t, "1073741824", result[parameter])
})
}
+
+func TestConfigPeriodicCompactionSecondsAndTTL(t *testing.T) {
+ t.Parallel()
+ srv := util.StartServer(t, map[string]string{})
+ defer srv.Close()
+
+ cli := srv.NewClient()
+ defer func() { require.NoError(t, cli.Close()) }()
+
+ testTemplate := func(t *testing.T, field string) func(*testing.T) {
+ return func(*testing.T) {
+ ctx := context.Background()
+ parameter := field
+ result, err := cli.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t,
strconv.FormatUint(math.MaxUint64-1, 10), result[parameter])
+
+ util.ErrorRegexp(t, cli.ConfigSet(ctx, parameter,
"-1").Err(), ".*not started as an integer")
+
+ require.NoError(t, cli.ConfigSet(ctx, parameter,
strconv.FormatUint(math.MaxUint64, 10)).Err())
+ result, err = cli.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t,
strconv.FormatUint(math.MaxUint64, 10), result[parameter])
+ }
+ }
+
+ t.Run("Get and Set rocksdb.periodic_compaction_seconds",
testTemplate(t, "rocksdb.periodic_compaction_seconds"))
+ t.Run("Get and Set rocksdb.ttl", testTemplate(t, "rocksdb.ttl"))
+}
+
+func TestConfigDailyOffpeakTimeUTC(t *testing.T) {
+ t.Parallel()
+ srv := util.StartServer(t, map[string]string{})
+ defer srv.Close()
+
+ cli := srv.NewClient()
+ defer func() { require.NoError(t, cli.Close()) }()
+
+ t.Run("Get and Set rocksdb.daily_offpeak_time_utc", func(t *testing.T) {
+ ctx := context.Background()
+ parameter := "rocksdb.daily_offpeak_time_utc"
+ result, err := cli.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t, "", result[parameter])
+
+ util.ErrorRegexp(t, cli.ConfigSet(ctx, parameter,
"VALUE").Err(), ".*Invalid argument.*")
+
+ require.NoError(t, cli.ConfigSet(ctx, parameter,
"00:00-00:30").Err())
+ result, err = cli.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t, "00:00-00:30", result[parameter])
+
+ require.NoError(t, cli.ConfigSet(ctx, parameter, "").Err())
+ result, err = cli.ConfigGet(ctx, parameter).Result()
+ require.NoError(t, err)
+ require.EqualValues(t, "", result[parameter])
+ })
+}
