zjw1111 commented on code in PR #424:
URL: https://github.com/apache/paimon-cpp/pull/424#discussion_r4246056094
##########
src/paimon/global_index/lucene/lucene_global_index_reader.cpp:
##########
@@ -17,23 +17,444 @@
*/
#include "paimon/global_index/lucene/lucene_global_index_reader.h"
+#include <algorithm>
+#include <cmath>
+#include <functional>
+#include <initializer_list>
+#include <limits>
+#include <utility>
+
#include "arrow/c/bridge.h"
-#include "lucene++/DisjunctionMaxQuery.h"
#include "lucene++/FileUtils.h"
#include "paimon/common/utils/options_utils.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/rapidjson_util.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
+#include "paimon/common/utils/string_utils.h"
#include "paimon/global_index/bitmap_scored_global_index_result.h"
#include "paimon/global_index/lucene/jieba_analyzer.h"
-#include "paimon/global_index/lucene/lucene_collector.h"
#include "paimon/global_index/lucene/lucene_defs.h"
#include "paimon/global_index/lucene/lucene_directory.h"
#include "paimon/global_index/lucene/lucene_filter.h"
#include "paimon/global_index/lucene/lucene_utils.h"
#include "paimon/io/data_input_stream.h"
+#include "rapidjson/document.h"
+#include "rapidjson/error/en.h"
namespace paimon::lucene {
+namespace {
+
+std::string JsonString(const rapidjson::Value& value) {
+ return std::string(value.GetString(), value.GetStringLength());
+}
+
+/// Trims Unicode White_Space as Rust `str::trim` does in the native DSL.
+std::wstring TrimDslWhitespace(const std::wstring& value) {
+ constexpr wchar_t kWhitespace[] =
+ L"\t\n\v\f\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005"
+ L"\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000";
+ size_t first = value.find_first_not_of(kWhitespace);
+ if (first == std::wstring::npos) {
+ return L"";
+ }
+ size_t last = value.find_last_not_of(kWhitespace);
+ return value.substr(first, last - first + 1);
+}
+
+bool TrimmedIsOneOf(const std::string& value, std::initializer_list<const
wchar_t*> candidates) {
+ std::wstring trimmed =
TrimDslWhitespace(LuceneUtils::StringToWstring(value));
+ for (const wchar_t* candidate : candidates) {
+ if (trimmed == candidate) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/// Translates the JSON DSL query of `FullTextSearch` into a Lucene query on
the indexed field.
+///
+/// The DSL is defined by apache/paimon-full-text `core/src/query.rs`. The
supported queries use
+/// its field names, aliases and defaults. Duplicate known fields are
rejected, and unknown fields
+/// are ignored as in the native engine.
+/// `match`, `multi_match`, `match_phrase` and `boolean` queries are
translated into term, boolean
+/// and phrase queries; `boost` queries and fuzzy matching are rejected.
+class DslQueryTranslator {
+ public:
+ /// Splits text into index terms with the analyzer used at indexing time.
+ using Analyzer = std::function<std::vector<std::wstring>(const
std::string&)>;
+
+ DslQueryTranslator(const std::wstring& wfield_name, Analyzer analyzer)
+ : wfield_name_(wfield_name), analyzer_(std::move(analyzer)) {}
+
+ Result<Lucene::QueryPtr> Translate(const std::string& query) const
noexcept(false) {
+ // RapidJSON treats a NUL byte as the end of input even when a length
is provided.
+ if (query.find('\0') != std::string::npos) {
+ return Status::Invalid("lucene full-text query must not contain
NUL characters");
+ }
+ rapidjson::Document document;
+ document.Parse(query.data(), query.size());
Review Comment:
[P2] Bound JSON and DSL nesting before recursive processing
Could you add a nesting-depth limit that protects both JSON parsing and DSL
translation? Repeatedly wrapping a valid match query in
`{"boolean":{"must":[previous_query]}}` creates a valid input with arbitrary
depth. The default RapidJSON parse here is recursive, and `TranslateQuery ->
TranslateBoolean -> TranslateQuery` recurses again at line 263, with no depth
bound. Each level has only one clause, so Lucene's clause-count limit does not
help. Sufficiently deep input can exhaust the thread stack and terminate the
process instead of returning a `Status`; the outer `try/catch` cannot handle
that failure.
Please also cover excessive nesting with a regression test. Limiting only
the translator would still leave the parsing stage exposed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
