This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new c8365b053 Remove old libicu max collation sentinel hack
c8365b053 is described below
commit c8365b05370cf3b9e77a619291a5d41b5ad3b0de
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Jun 24 02:57:16 2026 -0400
Remove old libicu max collation sentinel hack
For old libicu versions < 59 on RHEL 6 and 7 we had to add a hack in the NIF
driver to handle the max sortable sentinel `<<255,255,255,255>>` [1]. We
don't
support those versions any longer, and since libicu version 59 the library
will
automatically sort those as the highest values [2], so we can clean up our
collator NIF.
[1] https://github.com/apache/couchdb/pull/3491
[2]
https://www.unicode.org/reports/tr35/tr35-collation.html#tailored-noncharacter-weights
> U+FFFF: This code point is tailored to have a primary weight higher than
all
other characters. This allows the reliable specification of a range, such as
“Sch” ≤ X ≤ “Sch\uFFFF”, to include all strings starting with "sch" or
equivalent.
---
.../priv/couch_ejson_compare/couch_ejson_compare.c | 49 ++++------------------
src/couch/test/eunit/couch_ejson_compare_tests.erl | 3 +-
2 files changed, 11 insertions(+), 41 deletions(-)
diff --git a/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c
b/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c
index a4e9d1cfa..09608451c 100644
--- a/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c
+++ b/src/couch/priv/couch_ejson_compare/couch_ejson_compare.c
@@ -18,6 +18,15 @@
#include "erl_nif.h"
#include "unicode/ucol.h"
#include "unicode/ucasemap.h"
+#include "unicode/uvernum.h"
+
+/* {255,255,255,255} max-key sentinel relies on libicu sorting FFFF byte
+ * sequence w/ the highest collation weight. However that happens starting with
+ * libicu 59, so we'll explicitly check the version to avoid silently
+ * mis-collating data */
+#if U_ICU_VERSION_MAJOR_NUM < 59
+#error "CouchDB requires libicu >= 59 for correct max key collation"
+#endif
#define MAX_DEPTH 10
@@ -73,14 +82,8 @@ static __inline int atom_sort_order(ErlNifEnv*,
ERL_NIF_TERM);
static __inline int compare_strings(ctx_t*, ErlNifBinary, ErlNifBinary);
static __inline int compare_lists(int, ctx_t*, ERL_NIF_TERM, ERL_NIF_TERM);
static __inline int compare_props(int, ctx_t*, ERL_NIF_TERM, ERL_NIF_TERM);
-static __inline int is_max_utf8_marker(ErlNifBinary);
static __inline UCollator* get_collator(void);
-/* Should match the <<255,255,255,255>> in:
- * - src/mango/src/mango_idx_view.hrl#L13
- * - src/couch_mrview/src/couch_mrview_util.erl#L40 */
-static const unsigned char max_utf8_marker[] = {255, 255, 255, 255};
-
UCollator*
get_collator(void)
@@ -469,19 +472,6 @@ compare_props(int depth, ctx_t* ctx, ERL_NIF_TERM a,
ERL_NIF_TERM b)
}
-int
-is_max_utf8_marker(ErlNifBinary bin)
-{
- if (bin.size == sizeof(max_utf8_marker)) {
- if(memcmp(bin.data, max_utf8_marker, sizeof(max_utf8_marker)) == 0) {
- return 1;
- }
- return 0;
- }
- return 0;
-}
-
-
int
compare_strings(ctx_t* ctx, ErlNifBinary a, ErlNifBinary b)
{
@@ -489,27 +479,6 @@ compare_strings(ctx_t* ctx, ErlNifBinary a, ErlNifBinary b)
UCharIterator iterA, iterB;
int result;
- /* libicu versions earlier than 59 (at least) don't consider the
- * {255,255,255,255} to be the highest sortable string as CouchDB expects.
- * While we are still shipping CentOS 7 packages with libicu 50, we should
- * explicitly check for the marker, later on we can remove the max
- * logic */
-
- int a_is_max = is_max_utf8_marker(a);
- int b_is_max = is_max_utf8_marker(b);
-
- if(a_is_max && b_is_max) {
- return 0;
- }
-
- if(a_is_max) {
- return 1;
- }
-
- if(b_is_max) {
- return -1;
- }
-
uiter_setUTF8(&iterA, (const char *) a.data, (uint32_t) a.size);
uiter_setUTF8(&iterB, (const char *) b.data, (uint32_t) b.size);
diff --git a/src/couch/test/eunit/couch_ejson_compare_tests.erl
b/src/couch/test/eunit/couch_ejson_compare_tests.erl
index df982c12c..85b01aaaa 100644
--- a/src/couch/test/eunit/couch_ejson_compare_tests.erl
+++ b/src/couch/test/eunit/couch_ejson_compare_tests.erl
@@ -36,7 +36,8 @@
<<"B">>,
<<"ba">>,
<<"bb">>,
- % Highest sorting unicode value. Special case in the nif
+ % Highest sorting unicode value
+ % libicu >= 59 sorts it highest natively
?MAX_UNICODE_STRING,
[<<"a">>],
[<<"b">>],
