On Wed, 2026-07-15 at 18:25 -0400, Tom Lane wrote:
> Master seems sufficient.
In that case I did a bit of refactoring to clean it up and put it in a
new unicode_limits.h header.
Regards,
Jeff Davis
From b09c5bc0a437fe196940d7633fac4985e5538a06 Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Mon, 13 Jul 2026 19:31:56 -0700
Subject: [PATCH v2] Generate unicode_limits.h.
To ensure we do not overflow a size_t while case mapping on a 32-bit
platform, we need to know the maximum amount a UTF8 string can expand.
Calculate that maximum while generating unicode tables as a part of
the update-unicode target, and output into a new header
unicode_limits.h. Minor refactoring along the way.
Add a StaticAssertDecl to check that a MaxAllocSize text value
expanded by that amount would still have a length that fits in
size_t. (We couldn't actually create a new text value out of that, but
we still need to avoid overflow.)
Reported-by: Tom Lane <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/backend/utils/adt/pg_locale_builtin.c | 11 +
src/common/unicode/Makefile | 6 +-
.../unicode/generate-unicode_case_table.pl | 194 ++++++++++++------
src/common/unicode/meson.build | 2 +-
src/common/unicode_case.c | 12 +-
src/include/common/unicode_case_table.h | 9 +-
src/include/common/unicode_limits.h | 39 ++++
7 files changed, 202 insertions(+), 71 deletions(-)
create mode 100644 src/include/common/unicode_limits.h
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..5c67c6ffc3a 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -15,11 +15,22 @@
#include "catalog/pg_collation.h"
#include "common/unicode_case.h"
#include "common/unicode_category.h"
+#include "common/unicode_limits.h"
#include "miscadmin.h"
#include "utils/builtins.h"
+#include "utils/memutils.h"
#include "utils/pg_locale.h"
#include "utils/syscache.h"
+/*
+ * The largest text value must fit in MaxAllocSize, but then may grow during
+ * case mapping. While the resulting string will not be representable as a new
+ * text value, we must at least be sure not to overflow a size_t while
+ * processing it.
+ */
+StaticAssertDecl(SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION > MaxAllocSize,
+ "case mapping may overflow size_t");
+
extern pg_locale_t create_pg_locale_builtin(Oid collid,
MemoryContext context);
extern char *get_collation_actual_version_builtin(const char *collcollate);
diff --git a/src/common/unicode/Makefile b/src/common/unicode/Makefile
index c150e403cda..714ccbe00bd 100644
--- a/src/common/unicode/Makefile
+++ b/src/common/unicode/Makefile
@@ -21,7 +21,7 @@ CPPFLAGS += $(ICU_CFLAGS)
# By default, do nothing.
all:
-update-unicode: unicode_case_table.h unicode_category_table.h unicode_east_asian_fw_table.h unicode_nonspacing_table.h unicode_norm_hashfunc.h unicode_norm_table.h unicode_normprops_table.h unicode_version.h
+update-unicode: unicode_case_table.h unicode_category_table.h unicode_east_asian_fw_table.h unicode_nonspacing_table.h unicode_norm_hashfunc.h unicode_limits.h unicode_norm_table.h unicode_normprops_table.h unicode_version.h
mv $^ $(top_srcdir)/src/include/common/
$(MAKE) case-check
$(MAKE) category-check
@@ -36,6 +36,8 @@ CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNorma
unicode_version.h: generate-unicode_version.pl
$(PERL) $< --version $(UNICODE_VERSION)
+unicode_limits.h: unicode_case_table.h
+
unicode_case_table.h: generate-unicode_case_table.pl CaseFolding.txt SpecialCasing.txt UnicodeData.txt
$(PERL) $<
@@ -91,4 +93,4 @@ clean:
rm -f $(OBJS) case_test case_test.o category_test category_test.o norm_test norm_test.o
distclean: clean
- rm -f CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt SpecialCasing.txt UnicodeData.txt norm_test_table.h unicode_case_table.h unicode_category_table.h unicode_norm_table.h
+ rm -f CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt SpecialCasing.txt UnicodeData.txt norm_test_table.h unicode_case_table.h unicode_category_table.h unicode_limits.h unicode_norm_table.h
diff --git a/src/common/unicode/generate-unicode_case_table.pl b/src/common/unicode/generate-unicode_case_table.pl
index c1b94f47f56..d1dd3a5e8f2 100644
--- a/src/common/unicode/generate-unicode_case_table.pl
+++ b/src/common/unicode/generate-unicode_case_table.pl
@@ -4,13 +4,14 @@
# or locale-specific mappings.
#
# Input: CaseFolding.txt SpecialCasing.txt UnicodeData.txt
-# Output: unicode_case_table.h
+# Output: unicode_case_table.h unicode_limits.h
#
# Copyright (c) 2000-2026, PostgreSQL Global Development Group
use strict;
use warnings FATAL => 'all';
use Getopt::Long;
+use Encode qw(encode_utf8);
use FindBin;
use lib "$FindBin::RealBin/../../tools/";
@@ -19,11 +20,22 @@ my $output_path = '.';
GetOptions('outdir:s' => \$output_path);
+my $output_limits_file = "$output_path/unicode_limits.h";
my $output_table_file = "$output_path/unicode_case_table.h";
+# The following are computed during table generation:
+
# The maximum number of codepoints that can result from case mapping
-# of a single character. See Unicode section 5.18 "Case Mappings".
-my $MAX_CASE_EXPANSION = 3;
+# of a single character.
+my $UNICODE_MAX_CASEMAP_CODEPOINTS = 0;
+
+# The maximum expansion factor (in bytes) when case mapping a UTF8
+# string.
+my $UTF8_MAX_CASEMAP_EXPANSION = 0;
+
+# The maximum number of UTF8 bytes that can result from case mapping
+# a single code point.
+my $UTF8_CASEMAP_BUFSZ = 0;
my $FH;
@@ -44,15 +56,22 @@ while (my $line = <$FH>)
die "Simple_Titlecase $code out of range" if $simple_titlecase > 0x10FFFF;
die "Simple_Uppercase $code out of range" if $simple_uppercase > 0x10FFFF;
- if ($simple_lowercase || $simple_titlecase || $simple_uppercase)
- {
- $simple{$code} = {
- Simple_Lowercase => ($simple_lowercase || $code),
- Simple_Titlecase => ($simple_titlecase || $code),
- Simple_Uppercase => ($simple_uppercase || $code),
- Simple_Foldcase => $code,
- };
- }
+ next if !$simple_lowercase && !$simple_titlecase && !$simple_uppercase;
+
+ $simple_lowercase ||= $code;
+ $simple_titlecase ||= $code;
+ $simple_uppercase ||= $code;
+
+ update_limits('uppercase', $code, ($simple_uppercase));
+ update_limits('lowercase', $code, ($simple_lowercase));
+ update_limits('titlecase', $code, ($simple_titlecase));
+
+ $simple{$code} = {
+ Simple_Lowercase => $simple_lowercase,
+ Simple_Titlecase => $simple_titlecase,
+ Simple_Uppercase => $simple_uppercase,
+ Simple_Foldcase => $code,
+ };
}
close $FH;
@@ -105,19 +124,9 @@ while (my $line = <$FH>)
push @upper, $code if (scalar @upper == 0);
push @fold, $code;
- # none should map to more than 3 codepoints
- die "lowercase expansion for 0x$elts[0] exceeds maximum: '$elts[1]'"
- if (scalar @lower) > $MAX_CASE_EXPANSION;
- die "titlecase expansion for 0x$elts[0] exceeds maximum: '$elts[2]'"
- if (scalar @title) > $MAX_CASE_EXPANSION;
- die "uppercase expansion for 0x$elts[0] exceeds maximum: '$elts[3]'"
- if (scalar @upper) > $MAX_CASE_EXPANSION;
-
- # pad arrays to a fixed length of 3
- while (scalar @upper < $MAX_CASE_EXPANSION) { push @upper, 0x000000 }
- while (scalar @lower < $MAX_CASE_EXPANSION) { push @lower, 0x000000 }
- while (scalar @title < $MAX_CASE_EXPANSION) { push @title, 0x000000 }
- while (scalar @fold < $MAX_CASE_EXPANSION) { push @fold, 0x000000 }
+ update_limits('lowercase', $code, @lower);
+ update_limits('titlecase', $code, @title);
+ update_limits('uppercase', $code, @upper);
# Characters with special mappings may not have simple mappings;
# ensure that an entry exists.
@@ -174,6 +183,8 @@ while (my $line = <$FH>)
die "unsupported status type '$status'"
if $status ne 'S' && $status ne 'C' && $status ne 'F';
+ update_limits('casefold', $code, @fold);
+
# initialize simple case mappings if they don't exist
$simple{$code} ||= {
Simple_Lowercase => $code,
@@ -197,26 +208,12 @@ while (my $line = <$FH>)
if ($status eq 'F' || ($status eq 'C' && defined $special{$code}))
{
- while (scalar @fold < $MAX_CASE_EXPANSION) { push @fold, 0x000000 }
-
#initialize special case mappings if they don't exist
if (!defined $special{$code})
{
my @lower = ($simple{$code}{Simple_Lowercase});
my @title = ($simple{$code}{Simple_Titlecase});
my @upper = ($simple{$code}{Simple_Uppercase});
- while (scalar @lower < $MAX_CASE_EXPANSION)
- {
- push @lower, 0x000000;
- }
- while (scalar @title < $MAX_CASE_EXPANSION)
- {
- push @title, 0x000000;
- }
- while (scalar @upper < $MAX_CASE_EXPANSION)
- {
- push @upper, 0x000000;
- }
$special{$code} = {
Lowercase => \@lower,
Titlecase => \@title,
@@ -245,6 +242,53 @@ die
"special case map contains $num_special entries which cannot be represented in uint8"
if ($num_special > 256);
+open my $LIMITS, '>', $output_limits_file
+ or die "Could not open output file $output_limits_file: $!\n";
+
+print $LIMITS <<"EOS";
+/*-------------------------------------------------------------------------
+ *
+ * unicode_limits.h
+ * Useful limits calculated from Unicode data.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/unicode_limits.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef UNICODE_LIMITS_H
+#define UNICODE_LIMITS_H
+
+/*
+ * File auto-generated by src/common/unicode/generate-unicode_case_table.pl,
+ * do not edit.
+ */
+
+/*
+ * The maximum number of codepoints that can result from case mapping
+ * of a single code point. See Unicode section 5.18 "Case Mappings".
+ */
+#define UNICODE_MAX_CASEMAP_CODEPOINTS $UNICODE_MAX_CASEMAP_CODEPOINTS
+
+/*
+ * The maximum expansion factor of a UTF8 string due to case mapping.
+ */
+#define UTF8_MAX_CASEMAP_EXPANSION $UTF8_MAX_CASEMAP_EXPANSION
+
+/*
+ * The maximum number of UTF8 bytes needed to store the result of case
+ * mapping a single code point.
+ */
+#define UTF8_CASEMAP_BUFSZ $UTF8_CASEMAP_BUFSZ
+
+#endif
+EOS
+
+close $LIMITS;
+
# Start writing out the output files
open my $OT, '>', $output_table_file
or die "Could not open output file $output_table_file: $!\n";
@@ -270,12 +314,7 @@ print $OT <<"EOS";
*/
#include "common/unicode_case.h"
-
-/*
- * The maximum number of codepoints that can result from case mapping
- * of a single character. See Unicode section 5.18 "Case Mappings".
- */
-#define MAX_CASE_EXPANSION 3
+#include "common/unicode_limits.h"
/*
* Case mapping condition flags. For now, only Final_Sigma is supported.
@@ -296,7 +335,7 @@ typedef enum
typedef struct
{
int16 conditions;
- char32_t map[NCaseKind][MAX_CASE_EXPANSION];
+ char32_t map[NCaseKind][UNICODE_MAX_CASEMAP_CODEPOINTS];
} pg_special_case;
/*
@@ -310,18 +349,26 @@ EOS
foreach my $code (sort { $a <=> $b } (keys %special))
{
- die if scalar @{ $special{$code}{Lowercase} } != $MAX_CASE_EXPANSION;
- die if scalar @{ $special{$code}{Titlecase} } != $MAX_CASE_EXPANSION;
- die if scalar @{ $special{$code}{Uppercase} } != $MAX_CASE_EXPANSION;
- die if scalar @{ $special{$code}{Foldcase} } != $MAX_CASE_EXPANSION;
- my $lower = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Lowercase} });
- my $title = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Titlecase} });
- my $upper = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Uppercase} });
- my $fold = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Foldcase} });
+ my @lower = @{ $special{$code}{Lowercase} };
+ my @title = @{ $special{$code}{Titlecase} };
+ my @upper = @{ $special{$code}{Uppercase} };
+ my @fold = @{ $special{$code}{Foldcase} };
+
+ # pad to fixed length
+ push @lower, 0x000000
+ while (scalar @lower < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+ push @title, 0x000000
+ while (scalar @title < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+ push @upper, 0x000000
+ while (scalar @upper < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+ push @fold, 0x000000
+ while (scalar @fold < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+
+ my $lower = join ", ", (map { sprintf "0x%06x", $_ } @lower);
+ my $title = join ", ", (map { sprintf "0x%06x", $_ } @title);
+ my $upper = join ", ", (map { sprintf "0x%06x", $_ } @upper);
+ my $fold = join ", ", (map { sprintf "0x%06x", $_ } @fold);
+
printf $OT "\t{%s, ", $special{$code}{Conditions};
printf $OT
"{[CaseLower] = {%s},[CaseTitle] = {%s},[CaseUpper] = {%s},[CaseFold] = {%s}}},\n",
@@ -522,6 +569,37 @@ EOS
close $OT;
+# calculate, check, and update limits
+sub update_limits
+{
+ my ($casekind, $codepoint, @mapping) = @_;
+ my $n = scalar @mapping;
+
+ # See Unicode section 5.18 "Case Mappings"
+ my $mapping_str = join ' ', map { sprintf "0x%06x", $_ } @mapping;
+ die sprintf("%s expansion for 0x%06x exceeds maximum: %s",
+ $casekind, $codepoint, $mapping_str)
+ if $n > 3;
+
+ $UNICODE_MAX_CASEMAP_CODEPOINTS = $n
+ if $n > $UNICODE_MAX_CASEMAP_CODEPOINTS;
+
+ my $utf8len1 = length(encode_utf8(chr $codepoint));
+ my $utf8len2 = length(encode_utf8(join '', map { chr } @mapping));
+ my $factor = int(($utf8len1 + $utf8len2 - 1) / $utf8len1);
+ $UTF8_MAX_CASEMAP_EXPANSION = $factor
+ if $factor > $UTF8_MAX_CASEMAP_EXPANSION;
+
+ # Not guaranteed by Unicode standard. If larger than expected,
+ # then evaluate C type overflow risks before using the generated
+ # table.
+ die "UTF8 casemap expansion is greater than 3"
+ if $UTF8_MAX_CASEMAP_EXPANSION > 3;
+
+ $UTF8_CASEMAP_BUFSZ = $utf8len2
+ if $utf8len2 > $UTF8_CASEMAP_BUFSZ;
+}
+
# The function generates C code with a series of nested if-else conditions
# to search for the matching interval.
sub branch
diff --git a/src/common/unicode/meson.build b/src/common/unicode/meson.build
index 8a31fe40f6c..a32d44d1816 100644
--- a/src/common/unicode/meson.build
+++ b/src/common/unicode/meson.build
@@ -28,7 +28,7 @@ update_unicode_common_targets = []
update_unicode_common_targets += \
custom_target('unicode_case_table.h',
input: [unicode_data['CaseFolding.txt'], unicode_data['SpecialCasing.txt'], unicode_data['UnicodeData.txt']],
- output: ['unicode_case_table.h'],
+ output: ['unicode_case_table.h', 'unicode_limits.h'],
command: [
perl, files('generate-unicode_case_table.pl'),
'--outdir', '@OUTDIR@', '@INPUT@'],
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 399f08235a9..00668a7d4f3 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -237,6 +237,12 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
size_t result_len = 0;
size_t boundary = 0;
+ /*
+ * Must be guaranteed by caller to avoid overflow (text values limited to
+ * MaxAllocSize anyway).
+ */
+ Assert(srclen < SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION);
+
Assert((str_casekind == CaseTitle && wbnext && wbstate) ||
(str_casekind != CaseTitle && !wbnext && !wbstate));
@@ -299,9 +305,9 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
}
break;
case CASEMAP_SPECIAL:
- /* replace with up to MAX_CASE_EXPANSION characters */
+ /* replace with up to UNICODE_MAX_CASEMAP_CODEPOINTS */
Assert(simple == 0);
- for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
+ for (int i = 0; i < UNICODE_MAX_CASEMAP_CODEPOINTS && special[i]; i++)
{
char32_t u2 = special[i];
size_t u2len = unicode_utf8len(u2);
@@ -415,7 +421,7 @@ check_special_conditions(int conditions, const char *str, size_t len,
*
* If full is true, and a special case mapping is found and the conditions are
* met, 'special' is set to the mapping result (which is an array of up to
- * MAX_CASE_EXPANSION characters) and CASEMAP_SPECIAL is returned.
+ * UNICODE_MAX_CASEMAP_CODEPOINTS) and CASEMAP_SPECIAL is returned.
*
* Otherwise, search for a simple mapping, and if found, set 'simple' to the
* result and return CASEMAP_SIMPLE.
diff --git a/src/include/common/unicode_case_table.h b/src/include/common/unicode_case_table.h
index b2274627c4f..9930943453f 100644
--- a/src/include/common/unicode_case_table.h
+++ b/src/include/common/unicode_case_table.h
@@ -18,12 +18,7 @@
*/
#include "common/unicode_case.h"
-
-/*
- * The maximum number of codepoints that can result from case mapping
- * of a single character. See Unicode section 5.18 "Case Mappings".
- */
-#define MAX_CASE_EXPANSION 3
+#include "common/unicode_limits.h"
/*
* Case mapping condition flags. For now, only Final_Sigma is supported.
@@ -44,7 +39,7 @@ typedef enum
typedef struct
{
int16 conditions;
- char32_t map[NCaseKind][MAX_CASE_EXPANSION];
+ char32_t map[NCaseKind][UNICODE_MAX_CASEMAP_CODEPOINTS];
} pg_special_case;
/*
diff --git a/src/include/common/unicode_limits.h b/src/include/common/unicode_limits.h
new file mode 100644
index 00000000000..59e307857ed
--- /dev/null
+++ b/src/include/common/unicode_limits.h
@@ -0,0 +1,39 @@
+/*-------------------------------------------------------------------------
+ *
+ * unicode_limits.h
+ * Useful limits calculated from Unicode data.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/unicode_limits.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef UNICODE_LIMITS_H
+#define UNICODE_LIMITS_H
+
+/*
+ * File auto-generated by src/common/unicode/generate-unicode_case_table.pl,
+ * do not edit.
+ */
+
+/*
+ * The maximum number of codepoints that can result from case mapping
+ * of a single code point. See Unicode section 5.18 "Case Mappings".
+ */
+#define UNICODE_MAX_CASEMAP_CODEPOINTS 3
+
+/*
+ * The maximum expansion factor of a UTF8 string due to case mapping.
+ */
+#define UTF8_MAX_CASEMAP_EXPANSION 3
+
+/*
+ * The maximum number of UTF8 bytes needed to store the result of case
+ * mapping a single code point.
+ */
+#define UTF8_CASEMAP_BUFSZ 6
+
+#endif
--
2.43.0
