SteNicholas commented on code in PR #424:
URL: https://github.com/apache/paimon-cpp/pull/424#discussion_r4249559455
##########
src/paimon/global_index/lucene/lucene_global_index_reader.cpp:
##########
@@ -17,23 +17,444 @@
*/
#include "paimon/global_index/lucene/lucene_global_index_reader.h"
+#include <algorithm>
+#include <cmath>
+#include <functional>
+#include <initializer_list>
+#include <limits>
+#include <utility>
+
#include "arrow/c/bridge.h"
-#include "lucene++/DisjunctionMaxQuery.h"
#include "lucene++/FileUtils.h"
#include "paimon/common/utils/options_utils.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/rapidjson_util.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
+#include "paimon/common/utils/string_utils.h"
#include "paimon/global_index/bitmap_scored_global_index_result.h"
#include "paimon/global_index/lucene/jieba_analyzer.h"
-#include "paimon/global_index/lucene/lucene_collector.h"
#include "paimon/global_index/lucene/lucene_defs.h"
#include "paimon/global_index/lucene/lucene_directory.h"
#include "paimon/global_index/lucene/lucene_filter.h"
#include "paimon/global_index/lucene/lucene_utils.h"
#include "paimon/io/data_input_stream.h"
+#include "rapidjson/document.h"
+#include "rapidjson/error/en.h"
namespace paimon::lucene {
+namespace {
+
+std::string JsonString(const rapidjson::Value& value) {
+ return std::string(value.GetString(), value.GetStringLength());
+}
+
+/// Trims Unicode White_Space as Rust `str::trim` does in the native DSL.
+std::wstring TrimDslWhitespace(const std::wstring& value) {
+ constexpr wchar_t kWhitespace[] =
+ L"\t\n\v\f\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005"
+ L"\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000";
+ size_t first = value.find_first_not_of(kWhitespace);
+ if (first == std::wstring::npos) {
+ return L"";
+ }
+ size_t last = value.find_last_not_of(kWhitespace);
+ return value.substr(first, last - first + 1);
+}
+
+bool TrimmedIsOneOf(const std::string& value, std::initializer_list<const
wchar_t*> candidates) {
+ std::wstring trimmed =
TrimDslWhitespace(LuceneUtils::StringToWstring(value));
+ for (const wchar_t* candidate : candidates) {
+ if (trimmed == candidate) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/// Translates the JSON DSL query of `FullTextSearch` into a Lucene query on
the indexed field.
+///
+/// The DSL is defined by apache/paimon-full-text `core/src/query.rs`. The
supported queries use
+/// its field names, aliases and defaults. Duplicate known fields are
rejected, and unknown fields
+/// are ignored as in the native engine.
+/// `match`, `multi_match`, `match_phrase` and `boolean` queries are
translated into term, boolean
+/// and phrase queries; `boost` queries and fuzzy matching are rejected.
+class DslQueryTranslator {
+ public:
+ /// Splits text into index terms with the analyzer used at indexing time.
+ using Analyzer = std::function<std::vector<std::wstring>(const
std::string&)>;
+
+ DslQueryTranslator(const std::wstring& wfield_name, Analyzer analyzer)
+ : wfield_name_(wfield_name), analyzer_(std::move(analyzer)) {}
+
+ Result<Lucene::QueryPtr> Translate(const std::string& query) const
noexcept(false) {
+ // RapidJSON treats a NUL byte as the end of input even when a length
is provided.
+ if (query.find('\0') != std::string::npos) {
+ return Status::Invalid("lucene full-text query must not contain
NUL characters");
+ }
+ rapidjson::Document document;
+ document.Parse(query.data(), query.size());
Review Comment:
Thanks, addressed in 90d936df:
- `Translate` now parses with `rapidjson::kParseIterativeFlag`, so parsing
no longer recurses, and the document is freed by its memory pool without
recursion.
- `DslQueryTranslator` tracks the nesting depth of the JSON arrays and
objects it translates and returns an `Invalid` status ("full-text query exceeds
the maximum nesting depth of 127") before recursing past 127 levels. This
matches serde_json's default recursion limit in paimon-full-text, so both
backends accept the same queries. As in serde_json, unknown fields are skipped
without counting toward the limit.
- `TestNestedQuery` in `lucene_global_index_test.cpp` covers 41 levels of
`{"boolean":{"must":[...]}}` (accepted), 42 levels and mixed `must`/`queries`
nesting just past the limit (rejected), 100000 levels (rejected with a status),
and an unknown field nested 100000 levels (ignored).
##########
src/paimon/global_index/full_text/full_text_global_index_test.cpp:
##########
@@ -395,6 +391,10 @@ TEST_F(FullTextGlobalIndexTest, TestWriteAndSearch) {
ASSERT_OK_AND_ASSIGN(auto result,
reader->VisitFullTextSearch(MakeSearch(MatchQuery("THE LAKE"))));
CheckRowIds(result, {0, 2});
+ ASSERT_OK_AND_ASSIGN(
+ result, reader->VisitFullTextSearch(
+ MakeSearch(R"({"multi_match":{"query":"paimon
lake","columns":["text"]}})")));
Review Comment:
Thanks, added in 90d936df:
- `full-text`: `TestNestedBooleanAndBoost` in
`full_text_global_index_test.cpp` checks `paimon AND (streaming OR batch)` →
rows {0, 1}, and `paimon AND NOT (streaming OR batch)` in the `queries` form →
row {2}. It also runs `{"match":{"query":"lake","boost":1.0}}` and the same
query with `"boost":2.0`, asserting that both return rows {0, 1, 2, 3} and that
each boosted score is 2× the unboosted one.
- `lucene-fts`: `TestNestedQuery` in `lucene_global_index_test.cpp` runs the
same two nested `boolean` queries on the same rows with the same expected row
ids.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
