This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch remove-icu-max-marker-workaround in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a8f618886c490c589117dfcf3113a3fe72777e3b Author: Nick Vatamaniuc <[email protected]> AuthorDate: Wed Jun 24 02:57:16 2026 -0400 Remove old libicu max collation sentinel hack For old libicu versions < 59 on RHEL 6 and 7 we had to add a hack in the NIF driver to handle the max sortable sentinel `<<255,255,255,255>>` [1]. We don't support those versions any longer, and since libicu version 59 the library will automatically sort those as the highest values [2], so we can clean up our collator NIF. [1] https://github.com/apache/couchdb/pull/3491 [2] https://www.unicode.org/reports/tr35/tr35-collation.html#tailored-noncharacter-weights > U+FFFF: This code point is tailored to have a primary weight higher than all other characters. This allows the reliable specification of a range, such as “Sch” ≤ X ≤ “Sch\uFFFF”, to include all strings starting with "sch" or equivalent. --- .../priv/couch_ejson_compare/couch_ejson_compare.c | 49 ++++------------------ src/couch/test/eunit/couch_ejson_compare_tests.erl | 3 +- 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c b/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c index a4e9d1cfa..09608451c 100644 --- a/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c +++ b/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c @@ -18,6 +18,15 @@ #include "erl_nif.h" #include "unicode/ucol.h" #include "unicode/ucasemap.h" +#include "unicode/uvernum.h" + +/* {255,255,255,255} max-key sentinel relies on libicu sorting FFFF byte + * sequence w/ the highest collation weight. However that happens starting with + * libicu 59, so we'll explicitly check the version to avoid silently + * mis-collating data */ +#if U_ICU_VERSION_MAJOR_NUM < 59 +#error "CouchDB requires libicu >= 59 for correct max key collation" +#endif #define MAX_DEPTH 10 @@ -73,14 +82,8 @@ static __inline int atom_sort_order(ErlNifEnv*, ERL_NIF_TERM); static __inline int compare_strings(ctx_t*, ErlNifBinary, ErlNifBinary); static __inline int compare_lists(int, ctx_t*, ERL_NIF_TERM, ERL_NIF_TERM); static __inline int compare_props(int, ctx_t*, ERL_NIF_TERM, ERL_NIF_TERM); -static __inline int is_max_utf8_marker(ErlNifBinary); static __inline UCollator* get_collator(void); -/* Should match the <<255,255,255,255>> in: - * - src/mango/src/mango_idx_view.hrl#L13 - * - src/couch_mrview/src/couch_mrview_util.erl#L40 */ -static const unsigned char max_utf8_marker[] = {255, 255, 255, 255}; - UCollator* get_collator(void) @@ -469,19 +472,6 @@ compare_props(int depth, ctx_t* ctx, ERL_NIF_TERM a, ERL_NIF_TERM b) } -int -is_max_utf8_marker(ErlNifBinary bin) -{ - if (bin.size == sizeof(max_utf8_marker)) { - if(memcmp(bin.data, max_utf8_marker, sizeof(max_utf8_marker)) == 0) { - return 1; - } - return 0; - } - return 0; -} - - int compare_strings(ctx_t* ctx, ErlNifBinary a, ErlNifBinary b) { @@ -489,27 +479,6 @@ compare_strings(ctx_t* ctx, ErlNifBinary a, ErlNifBinary b) UCharIterator iterA, iterB; int result; - /* libicu versions earlier than 59 (at least) don't consider the - * {255,255,255,255} to be the highest sortable string as CouchDB expects. - * While we are still shipping CentOS 7 packages with libicu 50, we should - * explicitly check for the marker, later on we can remove the max - * logic */ - - int a_is_max = is_max_utf8_marker(a); - int b_is_max = is_max_utf8_marker(b); - - if(a_is_max && b_is_max) { - return 0; - } - - if(a_is_max) { - return 1; - } - - if(b_is_max) { - return -1; - } - uiter_setUTF8(&iterA, (const char *) a.data, (uint32_t) a.size); uiter_setUTF8(&iterB, (const char *) b.data, (uint32_t) b.size); diff --git a/src/couch/test/eunit/couch_ejson_compare_tests.erl b/src/couch/test/eunit/couch_ejson_compare_tests.erl index df982c12c..85b01aaaa 100644 --- a/src/couch/test/eunit/couch_ejson_compare_tests.erl +++ b/src/couch/test/eunit/couch_ejson_compare_tests.erl @@ -36,7 +36,8 @@ <<"B">>, <<"ba">>, <<"bb">>, - % Highest sorting unicode value. Special case in the nif + % Highest sorting unicode value + % libicu >= 59 sorts it highest natively ?MAX_UNICODE_STRING, [<<"a">>], [<<"b">>],
