This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 9598a26dd feat(search): support to skip expired keys in query (#3086)
9598a26dd is described below
commit 9598a26ddc439911927b482431840487eaead853
Author: Twice <[email protected]>
AuthorDate: Fri Aug 1 20:07:52 2025 +0800
feat(search): support to skip expired keys in query (#3086)
---
src/search/executors/filter_executor.h | 3 +++
src/search/executors/topn_executor.h | 3 +++
src/search/indexer.cc | 4 ++--
src/search/plan_executor.cc | 15 +++++++++++++--
tests/gocase/unit/search/search_test.go | 27 +++++++++++++++++++++++++++
5 files changed, 48 insertions(+), 4 deletions(-)
diff --git a/src/search/executors/filter_executor.h
b/src/search/executors/filter_executor.h
index 39b6c56cd..dfce82f2e 100644
--- a/src/search/executors/filter_executor.h
+++ b/src/search/executors/filter_executor.h
@@ -78,6 +78,7 @@ struct QueryExprEvaluator {
StatusOr<bool> Visit(TagContainExpr *v) const {
auto val = GET_OR_RET(ctx->Retrieve(ctx->db_ctx, row, v->field->info));
+ if (val.IsNull()) return false;
CHECK(val.Is<kqir::StringArray>());
auto tags = val.Get<kqir::StringArray>();
@@ -93,6 +94,7 @@ struct QueryExprEvaluator {
StatusOr<bool> Visit(NumericCompareExpr *v) const {
auto l_val = GET_OR_RET(ctx->Retrieve(ctx->db_ctx, row, v->field->info));
+ if (l_val.IsNull()) return false;
CHECK(l_val.Is<kqir::Numeric>());
auto l = l_val.Get<kqir::Numeric>();
@@ -118,6 +120,7 @@ struct QueryExprEvaluator {
StatusOr<bool> Visit(VectorRangeExpr *v) const {
auto val = GET_OR_RET(ctx->Retrieve(ctx->db_ctx, row, v->field->info));
+ if (val.IsNull()) return false;
CHECK(val.Is<kqir::NumericArray>());
auto l_values = val.Get<kqir::NumericArray>();
diff --git a/src/search/executors/topn_executor.h
b/src/search/executors/topn_executor.h
index 7eade082e..226045eca 100644
--- a/src/search/executors/topn_executor.h
+++ b/src/search/executors/topn_executor.h
@@ -58,6 +58,9 @@ struct TopNExecutor : ExecutorNode {
auto get_order = [this](RowType &row) -> StatusOr<double> {
auto order_val = GET_OR_RET(ctx->Retrieve(ctx->db_ctx, row,
topn->order->field->info));
+ // TODO(twice): here we return NaN if this field is not found,
+ // but we should consider to just skip this row instead.
+ if (order_val.IsNull()) return std::nan("");
CHECK(order_val.Is<kqir::Numeric>());
return order_val.Get<kqir::Numeric>();
};
diff --git a/src/search/indexer.cc b/src/search/indexer.cc
index 84b5a6547..e7089e54c 100644
--- a/src/search/indexer.cc
+++ b/src/search/indexer.cc
@@ -44,7 +44,7 @@ StatusOr<FieldValueRetriever>
FieldValueRetriever::Create(IndexOnDataType type,
HashMetadata metadata(false);
auto s = db.GetMetadata(ctx, ns_key, &metadata);
- if (!s.ok()) return {Status::NotOK, s.ToString()};
+ if (!s.ok()) return {s.IsNotFound() ? Status::NotFound : Status::NotOK,
s.ToString()};
return FieldValueRetriever(db, metadata, key);
} else if (type == IndexOnDataType::JSON) {
Json db(storage, ns);
@@ -52,7 +52,7 @@ StatusOr<FieldValueRetriever>
FieldValueRetriever::Create(IndexOnDataType type,
JsonMetadata metadata(false);
JsonValue value;
auto s = db.read(ctx, ns_key, &metadata, &value);
- if (!s.ok()) return {Status::NotOK, s.ToString()};
+ if (!s.ok()) return {s.IsNotFound() ? Status::NotFound : Status::NotOK,
s.ToString()};
return FieldValueRetriever(value);
} else {
unreachable();
diff --git a/src/search/plan_executor.cc b/src/search/plan_executor.cc
index 6fdd375eb..afb2de03f 100644
--- a/src/search/plan_executor.cc
+++ b/src/search/plan_executor.cc
@@ -37,6 +37,7 @@
#include "search/executors/topn_executor.h"
#include "search/indexer.h"
#include "search/ir_plan.h"
+#include "status.h"
namespace kqir {
@@ -167,10 +168,20 @@ auto ExecutorContext::Retrieve(engine::Context &ctx,
RowType &row, const FieldIn
return iter->second;
}
- auto retriever = GET_OR_RET(
- redis::FieldValueRetriever::Create(field->index->metadata.on_data_type,
row.key, storage, field->index->ns));
+ auto s_retriever =
+ redis::FieldValueRetriever::Create(field->index->metadata.on_data_type,
row.key, storage, field->index->ns);
+
+ if (s_retriever.Is<Status::NotFound>()) {
+ row.fields.emplace(field, kqir::Null{});
+ return kqir::Null{};
+ }
+ auto retriever = GET_OR_RET(std::move(s_retriever));
auto s = retriever.Retrieve(ctx, field->name, field->metadata.get());
+ if (s.Is<Status::NotFound>()) {
+ row.fields.emplace(field, kqir::Null{});
+ return kqir::Null{};
+ }
if (!s) return s;
row.fields.emplace(field, *s);
diff --git a/tests/gocase/unit/search/search_test.go
b/tests/gocase/unit/search/search_test.go
index 615aba513..241cd966f 100644
--- a/tests/gocase/unit/search/search_test.go
+++ b/tests/gocase/unit/search/search_test.go
@@ -24,6 +24,7 @@ import (
"context"
"encoding/binary"
"testing"
+ "time"
"github.com/apache/kvrocks/tests/gocase/util"
"github.com/redis/go-redis/v9"
@@ -205,4 +206,30 @@ func TestSearch(t *testing.T) {
srv.Restart()
verify(t)
})
+
+ t.Run("FT.SEARCH with expired keys", func(t *testing.T) {
+ require.NoError(t, rdb.Do(ctx, "FT.CREATE", "testidx_expired",
"ON", "HASH", "PREFIX", "1", "test_expired:", "SCHEMA", "a", "TAG", "b",
"NUMERIC").Err())
+ require.NoError(t, rdb.Do(ctx, "HSET", "test_expired:k1", "a",
"x,y", "b", "11").Err())
+ require.NoError(t, rdb.Do(ctx, "HSET", "test_expired:k2", "a",
"x,z", "b", "22").Err())
+ require.NoError(t, rdb.Do(ctx, "HSET", "test_expired:k3", "a",
"y,z", "b", "33").Err())
+ require.NoError(t, rdb.Do(ctx, "HSET", "test_expired:k4", "a",
"x,y,z", "b", "44").Err())
+
+ res := rdb.Do(ctx, "FT.SEARCHSQL", "select * from
testidx_expired where a hastag \"z\" and b < 40")
+ require.NoError(t, res.Err())
+ // result should be [2 test_expired:k2 [a x,z b 22]
test_expired:k3 [a y,z b 33]]
+ require.Equal(t, 5, len(res.Val().([]interface{})))
+ require.Equal(t, int64(2), res.Val().([]interface{})[0])
+ require.Equal(t, "test_expired:k2",
res.Val().([]interface{})[1])
+ require.Equal(t, "test_expired:k3",
res.Val().([]interface{})[3])
+
+ require.NoError(t, rdb.Do(ctx, "EXPIRE", "test_expired:k2",
1).Err())
+ time.Sleep(time.Millisecond * 1500) // wait for the key to
expire
+
+ res = rdb.Do(ctx, "FT.SEARCHSQL", "select * from
testidx_expired where a hastag \"z\" and b < 40")
+ require.NoError(t, res.Err())
+ // result should be [1 test_expired:k3 [a y,z b 33]]
+ require.Equal(t, 3, len(res.Val().([]interface{})))
+ require.Equal(t, int64(1), res.Val().([]interface{})[0])
+ require.Equal(t, "test_expired:k3",
res.Val().([]interface{})[1])
+ })
}
