join -i and uniq -i used memcasecmp, which only folds ASCII A-Z, so in UTF-8 locales É and é did not match despite -i. That matches the long-standing FIXME in join's keycmp.
In multibyte locales, compare c32tolower of each mcel character (same approach as gnulib exclude). Unibyte locales keep the historical memcasecmp + length behavior. As before, -i does not use LC_COLLATE. This is simple codepoint folding, not full Unicode case mapping: É matches é, but ß and SS remain distinct. * src/memcasecmp-mcel.h (memcasecmp_fields): New shared helper. * src/join.c (keycmp): Use it for -i. * src/uniq.c (different): Likewise. * src/local.mk: List the new header. * tests/join/join.pl: Cover É/é join and ß/SS non-join. * tests/uniq/uniq.pl: Cover É/é collapse and ß/SS distinct. * NEWS: Mention the improvement. Signed-off-by: Iván Ezequiel Rodriguez <[email protected]> --- NEWS | 5 ++++ src/join.c | 17 +++++--------- src/local.mk | 1 + src/memcasecmp-mcel.h | 54 +++++++++++++++++++++++++++++++++++++++++++ src/uniq.c | 6 ++--- tests/join/join.pl | 18 +++++++++++++++ tests/uniq/uniq.pl | 13 ++++++++++- 7 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 src/memcasecmp-mcel.h diff --git a/NEWS b/NEWS index f1040bde5..15873e90a 100644 --- a/NEWS +++ b/NEWS @@ -97,6 +97,11 @@ GNU coreutils NEWS -*- outline -*- stat -f -c%T now reports the file system type, and tail -f uses inotify for this file system. + 'join -i' and 'uniq -i' fold multi-byte letters with c32tolower in + multibyte locales (e.g. É matches é), instead of only ASCII via + memcasecmp. This is simple codepoint folding, not full Unicode case + mapping (ß and SS remain distinct). + uname adds the -A,--all-labeled option to label all output, one item per line. ** Changes in behavior diff --git a/src/join.c b/src/join.c index 4346758a6..0cb8e1640 100644 --- a/src/join.c +++ b/src/join.c @@ -28,6 +28,7 @@ #include "linebuffer.h" #include "mcel.h" #include "memcasecmp.h" +#include "memcasecmp-mcel.h" #include "quote.h" #include "skipchars.h" #include "stdio--.h" @@ -397,17 +398,11 @@ keycmp (struct line const *line1, struct line const *line2, return 1; if (ignore_case) - { - /* FIXME: ignore_case does not work with NLS (in particular, - with multibyte chars). */ - diff = memcasecmp (beg1, beg2, MIN (len1, len2)); - } - else - { - if (hard_LC_COLLATE) - return xmemcoll (beg1, len1, beg2, len2); - diff = memcmp (beg1, beg2, MIN (len1, len2)); - } + return memcasecmp_fields (beg1, len1, beg2, len2); + + if (hard_LC_COLLATE) + return xmemcoll (beg1, len1, beg2, len2); + diff = memcmp (beg1, beg2, MIN (len1, len2)); if (diff) return diff; diff --git a/src/local.mk b/src/local.mk index 6ed4ed78f..2694b92a9 100644 --- a/src/local.mk +++ b/src/local.mk @@ -56,6 +56,7 @@ noinst_HEADERS = \ src/iopoll.h \ src/longlong.h \ src/ls.h \ + src/memcasecmp-mcel.h \ src/octhexdigits.h \ src/operand2sig.h \ src/printenv.h \ diff --git a/src/memcasecmp-mcel.h b/src/memcasecmp-mcel.h new file mode 100644 index 000000000..c69e24811 --- /dev/null +++ b/src/memcasecmp-mcel.h @@ -0,0 +1,54 @@ +/* Case-insensitive compare of memory regions using mcel / c32tolower. + + Copyright (C) 2026 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* Include after system.h, mcel.h, and memcasecmp.h. */ + +/* Compare S1[0..N1) and S2[0..N2) ignoring case. + Unibyte locales: historical memcasecmp + length. + Multibyte: compare c32tolower of each mcel character so equal-folded + fields match even when UTF-8 byte lengths differ. This is simple + codepoint case folding, not full Unicode case mapping (e.g. ß vs SS + remain distinct). Does not consult LC_COLLATE. */ +ATTRIBUTE_PURE +static inline int +memcasecmp_fields (char const *s1, idx_t n1, char const *s2, idx_t n2) +{ + if (MB_CUR_MAX <= 1) + { + int diff = memcasecmp (s1, s2, MIN (n1, n2)); + return diff ? diff : _GL_CMP (n1, n2); + } + + char const *p1 = s1; + char const *e1 = s1 + n1; + char const *p2 = s2; + char const *e2 = s2 + n2; + + while (p1 < e1 && p2 < e2) + { + mcel_t g1 = mcel_scan (p1, e1); + mcel_t g2 = mcel_scan (p2, e2); + char32_t c1 = g1.err ? to_uchar (*p1) : c32tolower (g1.ch); + char32_t c2 = g2.err ? to_uchar (*p2) : c32tolower (g2.ch); + if (c1 != c2) + return _GL_CMP (c1, c2); + p1 += g1.err ? 1 : g1.len; + p2 += g2.err ? 1 : g2.len; + } + + return (p1 < e1) - (p2 < e2); +} diff --git a/src/uniq.c b/src/uniq.c index f91c4212c..9f4d5697b 100644 --- a/src/uniq.c +++ b/src/uniq.c @@ -32,6 +32,7 @@ #include "stdio--.h" #include "xstrtol.h" #include "memcasecmp.h" +#include "memcasecmp-mcel.h" #include "quote.h" /* The official name of this program (e.g., no 'g' prefix). */ @@ -304,9 +305,8 @@ static bool different (char *old, char *new, idx_t oldlen, idx_t newlen) { if (ignore_case) - return oldlen != newlen || memcasecmp (old, new, oldlen); - else - return oldlen != newlen || memcmp (old, new, oldlen); + return memcasecmp_fields (old, oldlen, new, newlen) != 0; + return oldlen != newlen || memcmp (old, new, oldlen); } /* Output the line in linebuffer LINE to standard output diff --git a/tests/join/join.pl b/tests/join/join.pl index 1bc6d1305..6345f88ce 100755 --- a/tests/join/join.pl +++ b/tests/join/join.pl @@ -401,6 +401,24 @@ if ($mb_locale ne 'C') push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; } push @Tests, @new; + + # -i with UTF-8 letters: memcasecmp is byte-wise and misses these. + # Added after the -mb duplication to avoid a second ENV spec. + # É (U+00C9) / é (U+00E9) fold together; ß (U+00DF) / SS do not + # (c32tolower is not full Unicode case mapping). + push @Tests, + ['i-utf8-join', '-i', + {IN=>"\xC3\x89 a\n"}, {IN=>"\xC3\xA9 b\n"}, + {OUT=>"\xC3\x89 a b\n"}, + {ENV => "LC_ALL=$mb_locale"}], + ['i-utf8-nojoin', '', + {IN=>"\xC3\x89 a\n"}, {IN=>"\xC3\xA9 b\n"}, + {OUT=>""}, + {ENV => "LC_ALL=$mb_locale"}], + ['i-utf8-sharp-s', '-i', + {IN=>"\xC3\x9F a\n"}, {IN=>"SS b\n"}, + {OUT=>""}, + {ENV => "LC_ALL=$mb_locale"}]; } @Tests = triple_test \@Tests; diff --git a/tests/uniq/uniq.pl b/tests/uniq/uniq.pl index 0df7ec62d..0f9be97ff 100755 --- a/tests/uniq/uniq.pl +++ b/tests/uniq/uniq.pl @@ -302,7 +302,18 @@ if ($mb_locale ne 'C') my @Locale_Tests = ( ['w1-mb', '-w1', {IN => $trouble_with_w1}, {OUT => $trouble_with_w1}, - {ENV => "LC_ALL=$mb_locale"}] + {ENV => "LC_ALL=$mb_locale"}], + # -i with UTF-8 letters: memcasecmp is byte-wise and misses these. + # É/é fold together; ß/SS do not (not full Unicode case mapping). + ['i-utf8', '-i', + {IN=>"\xC3\x89\n\xC3\xA9\n"}, {OUT=>"\xC3\x89\n"}, + {ENV => "LC_ALL=$mb_locale"}], + ['i-utf8-distinct', '-i', + {IN=>"\xC3\x89\nB\n"}, {OUT=>"\xC3\x89\nB\n"}, + {ENV => "LC_ALL=$mb_locale"}], + ['i-utf8-sharp-s', '-i', + {IN=>"\xC3\x9F\nSS\n"}, {OUT=>"\xC3\x9F\nSS\n"}, + {ENV => "LC_ALL=$mb_locale"}] ); push @Tests, @Locale_Tests; } -- 2.43.0
