This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 0186b8446 perf(scan): optimise scan performance when scanning keys
with a tag (#3156)
0186b8446 is described below
commit 0186b8446f6bbe91afe01cbaab6e10474ba4c20b
Author: sryan yuan <[email protected]>
AuthorDate: Thu Sep 4 12:27:45 2025 +0800
perf(scan): optimise scan performance when scanning keys with a tag (#3156)
When scanning keys with a tag, we can directly scan the slot to which
the keys belong, avoiding unnecessary scanning on other slots.
Co-authored-by: yxj25245 <[email protected]>
Co-authored-by: hulk <[email protected]>
---
src/commands/cmd_server.cc | 2 +-
src/commands/scan_base.h | 5 +++++
src/storage/redis_db.cc | 11 ++++++++---
src/storage/redis_db.h | 2 +-
tests/gocase/unit/scan/scan_test.go | 37 +++++++++++++++++++++++++++++++++++++
5 files changed, 52 insertions(+), 5 deletions(-)
diff --git a/src/commands/cmd_server.cc b/src/commands/cmd_server.cc
index 9bb3461aa..fc108035e 100644
--- a/src/commands/cmd_server.cc
+++ b/src/commands/cmd_server.cc
@@ -926,7 +926,7 @@ class CommandScan : public CommandScanBase {
std::vector<std::string> keys;
std::string end_key;
- const auto s = redis_db.Scan(ctx, key_name, limit_, prefix_, suffix_glob_,
&keys, &end_key, type_);
+ const auto s = redis_db.Scan(ctx, key_name, limit_, prefix_, suffix_glob_,
&keys, &end_key, type_, scan_slot_);
if (!s.ok()) {
return {Status::RedisExecErr, s.ToString()};
}
diff --git a/src/commands/scan_base.h b/src/commands/scan_base.h
index d80c1e4e1..730f4457b 100644
--- a/src/commands/scan_base.h
+++ b/src/commands/scan_base.h
@@ -20,6 +20,7 @@
#pragma once
+#include "cluster/redis_slot.h"
#include "commander.h"
#include "commands/command_parser.h"
#include "error_constants.h"
@@ -51,6 +52,9 @@ class CommandScanBase : public Commander {
return {Status::RedisParseErr, "Invalid glob pattern: " + s.Msg()};
}
std::tie(prefix_, suffix_glob_) = util::SplitGlob(glob_pattern);
+ if (std::string_view tag = GetTagFromKey(glob_pattern); !tag.empty()) {
+ scan_slot_ = GetSlotIdFromKey(glob_pattern);
+ }
} else if (parser.EatEqICase("count")) {
limit_ = GET_OR_RET(parser.TakeInt());
if (limit_ <= 0) {
@@ -105,6 +109,7 @@ class CommandScanBase : public Commander {
int limit_ = 20;
RedisType type_ = kRedisNone;
bool no_values_ = false;
+ std::optional<int> scan_slot_;
};
class CommandSubkeyScanBase : public CommandScanBase {
diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc
index 6f12f5890..2898c3b85 100644
--- a/src/storage/redis_db.cc
+++ b/src/storage/redis_db.cc
@@ -317,7 +317,8 @@ rocksdb::Status Database::Keys(engine::Context &ctx, const
std::string &prefix,
rocksdb::Status Database::Scan(engine::Context &ctx, const std::string
&cursor, uint64_t limit,
const std::string &prefix, const std::string
&suffix_glob,
- std::vector<std::string> *keys, std::string
*end_cursor, RedisType type) {
+ std::vector<std::string> *keys, std::string
*end_cursor, RedisType type,
+ std::optional<int> scan_slot) {
end_cursor->clear();
uint64_t cnt = 0;
uint16_t slot_start = 0;
@@ -328,7 +329,11 @@ rocksdb::Status Database::Scan(engine::Context &ctx, const
std::string &cursor,
std::string ns_cursor = AppendNamespacePrefix(cursor);
if (storage_->IsSlotIdEncoded()) {
- slot_start = cursor.empty() ? 0 : GetSlotIdFromKey(cursor);
+ if (scan_slot.has_value()) {
+ slot_start = scan_slot.value();
+ } else {
+ slot_start = cursor.empty() ? 0 : GetSlotIdFromKey(cursor);
+ }
ns_prefix = ComposeNamespaceKey(namespace_, "", false);
if (!prefix.empty()) {
PutFixed16(&ns_prefix, slot_start);
@@ -392,7 +397,7 @@ rocksdb::Status Database::Scan(engine::Context &ctx, const
std::string &cursor,
break;
}
- if (++slot_id >= HASH_SLOTS_SIZE) {
+ if (++slot_id >= HASH_SLOTS_SIZE || scan_slot.has_value()) {
break;
}
diff --git a/src/storage/redis_db.h b/src/storage/redis_db.h
index 9563b1e1d..16d98dcd3 100644
--- a/src/storage/redis_db.h
+++ b/src/storage/redis_db.h
@@ -124,7 +124,7 @@ class Database {
[[nodiscard]] rocksdb::Status Scan(engine::Context &ctx, const std::string
&cursor, uint64_t limit,
const std::string &prefix, const
std::string &suffix_glob,
std::vector<std::string> *keys,
std::string *end_cursor = nullptr,
- RedisType type = kRedisNone);
+ RedisType type = kRedisNone,
std::optional<int> scan_slot = std::nullopt);
[[nodiscard]] rocksdb::Status RandomKey(engine::Context &ctx, const
std::string &cursor, std::string *key);
std::string AppendNamespacePrefix(const Slice &user_key);
[[nodiscard]] rocksdb::Status ClearKeysOfSlotRange(engine::Context &ctx,
const rocksdb::Slice &ns,
diff --git a/tests/gocase/unit/scan/scan_test.go
b/tests/gocase/unit/scan/scan_test.go
index 5f7f62bea..e0eb8e4d7 100644
--- a/tests/gocase/unit/scan/scan_test.go
+++ b/tests/gocase/unit/scan/scan_test.go
@@ -32,6 +32,43 @@ import (
"golang.org/x/exp/slices"
)
+func TestScanTagKey(t *testing.T) {
+ srv := util.StartServer(t, map[string]string{"cluster-enabled": "yes"})
+ defer srv.Close()
+
+ ctx := context.Background()
+ rdb := srv.NewClient()
+ defer func() { require.NoError(t, rdb.Close()) }()
+
+ nodeID := "YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY"
+ require.NoError(t, rdb.Do(ctx, "clusterx", "SETNODEID", nodeID).Err())
+ clusterNodes := fmt.Sprintf("%s %s %d master - 0-16383", nodeID,
srv.Host(), srv.Port())
+ require.NoError(t, rdb.Do(ctx, "clusterx", "SETNODES", clusterNodes,
"1").Err())
+
+ // Slot keys from 0-100
+ initKeys := []string{
+ "{3560}key", "{22179}key", "{48756}key", "{2977}key",
"{4569}key", "{460}key", "{4384}key", "{41432}key", "{46920}key", "{9073}key",
+ "{2281}key", "{15129}key", "{5465}key", "{18928}key",
"{5288}key", "{4872}key", "{4883}key", "{5279}key", "{40322}key", "{5494}key",
+ "{23669}key", "{2270}key", "{6915}key", "{49046}key",
"{50482}key", "{1407}key", "{491}key", "{4598}key", "{2986}key", "{42942}key",
+ "{7819}key", "{3591}key", "{13280}key", "{4128}key",
"{41073}key", "{18072}key", "{14289}key", "{3121}key", "{11316}key",
"{9929}key",
+ "{40792}key", "{5024}key", "{12561}key", "{104856}key",
"{9432}key", "{83909}key", "{15568}key", "{8825}key", "{2631}key", "{15599}key",
+ "{49407}key", "{10406}key", "{5638}key", "{12590}key",
"{13987}key", "{40763}key", "{108747}key", "{8322}key", "{15882}key",
"{14278}key",
+ "{1046}key", "{41082}key", "{21848}key", "{13271}key",
"{11423}key", "{26968}key", "{49835}key", "{3614}key", "{13358}key",
"{1782}key",
+ "{714}key", "{68536}key", "{42131}key", "{7687}key",
"{9307}key", "{2718}key", "{12254}key", "{78296}key", "{24308}key", "{5711}key",
+ "{19057}key", "{12448}key", "{69226}key", "{21266}key",
"{49332}key", "{10333}key", "{2104}key", "{53181}key", "{79586}key",
"{13544}key",
+ "{1773}key", "{108}key", "{6197}key", "{43621}key",
"{3008}key", "{8417}key", "{18306}key", "{13719}key", "{45862}key", "{355}key"}
+ for _, key := range initKeys {
+ require.NoError(t, rdb.Set(ctx, key, "1", 0).Err())
+ }
+
+ cursor, keys := scan(t, rdb, "0", "count", 100)
+ require.Equal(t, initKeys, keys)
+ require.NotEqual(t, "0", cursor)
+ cursor, keys = scan(t, rdb, "0", "match", "{355}*")
+ require.Equal(t, []string{"{355}key"}, keys)
+ require.Equal(t, cursor, "0")
+}
+
func TestScanSlotRemainingKeys(t *testing.T) {
srv := util.StartServer(t, map[string]string{"cluster-enabled": "yes"})
defer srv.Close()
