Hi again,
Attached is v3 of the patch (now with a new title).
This version addresses the five remaining mismatches reported in my previous
email. These were caused by Unicode Corrigendum #4, which changed the
normalization mappings of five characters in Unicode 4.0. Since SASLprep (RFC
4013) requires Unicode 3.2, the patch now maps these characters to their
original Unicode 3.2 values before normalization, as described in Unicode
Standard Annex #15, section 11.3.
I've also made a few additional changes:
- Moved the Unicode 3.2 unassigned code point check into the existing mapping
loop, avoiding an additional pass over the input.
- Added regression tests for the five affected characters, bringing the total
to 18 test cases.
- Made the regression test ordering independent of the database collation.
I reran the comparison against Python's Unicode 3.2 reference implementation
using the same test script as before.
The results are now:
- Original PostgreSQL implementation: 104 mismatches
- Initial patch: 704 mismatches
- v2: 5 mismatches
- v3: 0 mismatches
Checked 1,112,210 cases; mismatches: 0
The updated patch includes the five additional regression tests. The standalone
Python test script attached to my previous email remains unchanged.
I would appreciate any review or feedback.
Thanks!
Regards,
Robert I Nilsson
From 738654b42bfcc373774bdd7a25ac152a831ea0b5 Mon Sep 17 00:00:00 2001
From: Robert I Nilsson <[email protected]>
Date: Tue, 10 Nov 2026 17:36:05 +0100
Subject: [PATCH v3] Fix SASLprep validation and Unicode 3.2 compatibility
SASLprep requires prohibited output characters and bidirectional
properties to be checked after Unicode normalization. However,
pg_saslprep() performs these checks on the original input, which
can cause valid strings to be rejected and invalid strings to be
accepted.
Additionally, SASLprep uses Unicode 3.2 to identify unassigned
code points, while PostgreSQL's normalizer uses newer Unicode data.
This can cause code points unassigned in Unicode 3.2 to normalize
into assigned characters, bypassing the unassigned check.
Furthermore, Unicode Corrigendum #4 changed the normalization mappings
of five characters in Unicode 4.0. Since SASLprep (RFC 4013) requires
Unicode 3.2, these characters must use their original mappings rather
than those in PostgreSQL's newer Unicode normalization tables.
Reject unassigned Unicode 3.2 code points before normalization,
restore the original mappings for the five affected characters,
then perform prohibited output and bidirectional checks on the
normalized result.
Add regression tests covering these cases.
Author: Robert I Nilsson <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/common/saslprep.c | 62 ++++++++++++++---
.../test_saslprep/expected/test_saslprep.out | 67 +++++++++++++++++++
.../test_saslprep/sql/test_saslprep.sql | 53 +++++++++++++++
3 files changed, 174 insertions(+), 8 deletions(-)
diff --git a/src/common/saslprep.c b/src/common/saslprep.c
index a2bb60ac0cb..b94d723ea75 100644
--- a/src/common/saslprep.c
+++ b/src/common/saslprep.c
@@ -1049,6 +1049,7 @@ pg_saslprep(const char *input, char **output)
char32_t *input_chars = NULL;
char32_t *output_chars = NULL;
int input_size;
+ int output_size;
char *result;
int result_size;
int count;
@@ -1098,6 +1099,15 @@ pg_saslprep(const char *input, char **output)
{
char32_t code = input_chars[i];
+ /*
+ * Reject code points unassigned in Unicode 3.2 before normalization.
+ * The normalizer uses newer Unicode data and may map such code
+ * points to characters that were already assigned in Unicode 3.2.
+ * See Unicode Standard Annex #15, section 11.3 for details.
+ */
+ if (IS_CODE_IN_TABLE(code, unassigned_codepoint_ranges))
+ goto prohibited;
+
if (IS_CODE_IN_TABLE(code, non_ascii_space_ranges))
input_chars[count++] = 0x0020;
else if (IS_CODE_IN_TABLE(code, commonly_mapped_to_nothing_ranges))
@@ -1105,7 +1115,36 @@ pg_saslprep(const char *input, char **output)
/* map to nothing */
}
else
+ {
+ /*
+ * PostgreSQL's Unicode normalization tables include the corrections
+ * introduced by Unicode Corrigendum #4. SASLprep (RFC 4013), however,
+ * requires Unicode 3.2 normalization, which predates these corrections.
+ * Map the five affected characters to their original Unicode 3.2
+ * values before normalization.
+ * See Unicode Standard Annex #15, section 11.3 for details.
+ */
+ switch (code)
+ {
+ case 0x2F868:
+ code = 0x2136A;
+ break;
+ case 0x2F874:
+ code = 0x5F33;
+ break;
+ case 0x2F91F:
+ code = 0x43AB;
+ break;
+ case 0x2F95F:
+ code = 0x7AAE;
+ break;
+ case 0x2F9BF:
+ code = 0x4D57;
+ break;
+ }
+
input_chars[count++] = code;
+ }
}
input_chars[count] = (char32_t) '\0';
input_size = count;
@@ -1121,13 +1160,20 @@ pg_saslprep(const char *input, char **output)
if (!output_chars)
goto oom;
+ output_size = 0;
+ for (wp = output_chars; *wp; wp++)
+ output_size++;
+
+ if (output_size == 0)
+ goto prohibited; /* don't allow empty password, after NFKC */
+
/*
* 3) Prohibit -- Check for any characters that are not allowed in the
* output. If any are found, return an error.
*/
- for (i = 0; i < input_size; i++)
+ for (i = 0; i < output_size; i++)
{
- char32_t code = input_chars[i];
+ char32_t code = output_chars[i];
if (IS_CODE_IN_TABLE(code, prohibited_output_ranges))
goto prohibited;
@@ -1157,9 +1203,9 @@ pg_saslprep(const char *input, char **output)
* MUST be the last character of the string."
*/
contains_RandALCat = false;
- for (i = 0; i < input_size; i++)
+ for (i = 0; i < output_size; i++)
{
- char32_t code = input_chars[i];
+ char32_t code = output_chars[i];
if (IS_CODE_IN_TABLE(code, RandALCat_codepoint_ranges))
{
@@ -1170,12 +1216,12 @@ pg_saslprep(const char *input, char **output)
if (contains_RandALCat)
{
- char32_t first = input_chars[0];
- char32_t last = input_chars[input_size - 1];
+ char32_t first = output_chars[0];
+ char32_t last = output_chars[output_size - 1];
- for (i = 0; i < input_size; i++)
+ for (i = 0; i < output_size; i++)
{
- char32_t code = input_chars[i];
+ char32_t code = output_chars[i];
if (IS_CODE_IN_TABLE(code, LCat_codepoint_ranges))
goto prohibited;
diff --git a/src/test/modules/test_saslprep/expected/test_saslprep.out b/src/test/modules/test_saslprep/expected/test_saslprep.out
index 92f93365343..285c2987c98 100644
--- a/src/test/modules/test_saslprep/expected/test_saslprep.out
+++ b/src/test/modules/test_saslprep/expected/test_saslprep.out
@@ -149,4 +149,71 @@ SELECT
<DEL> | \x7f | (,PROHIBITED)
(128 rows)
+-- Test SASLprep validation before and after normalization.
+-- Prohibited output characters and bidi rules must be checked
+-- after normalization, while code points unassigned in Unicode
+-- 3.2 must be rejected before normalization.
+WITH cases(name, src, expected_status, expected_res) AS (
+ VALUES
+ -- Characters that become valid after normalization.
+ ('combining grave', 'cd80', 'SUCCESS', 'cc80'),
+ ('combining acute', 'cd81', 'SUCCESS', 'cc81'),
+ ('grave after a', '61cd80', 'SUCCESS', 'c3a0'),
+ -- Normalization can increase the number of code points.
+ ('latin ligature ff', 'efac80', 'SUCCESS', '6666'),
+ -- Normalization introduces prohibited bidi sequences.
+ ('hebrew presentation', 'efac9d', 'PROHIBITED', NULL),
+ ('hebrew shin', 'efacaa', 'PROHIBITED', NULL),
+ ('arabic ligature', 'efb19b', 'PROHIBITED', NULL),
+ -- Normalization changes bidi properties (L to R).
+ ('alef after latin', '41e284b5', 'PROHIBITED', NULL),
+ ('alef in RTL', 'd790e284b5d791', 'SUCCESS', 'd790d790d791'),
+ -- Normalization introduces L characters into an RTL string.
+ ('rupee in RTL', 'd790e282a8d791', 'PROHIBITED', NULL),
+ -- Code points unassigned in Unicode 3.2 that normalize
+ -- to assigned characters using newer Unicode data.
+ ('unassigned Greek', 'cfb9', 'PROHIBITED', NULL),
+ ('unassigned modifier', 'e1b4ac', 'PROHIBITED', NULL),
+ -- Control case: valid RTL string containing a digit.
+ ('valid RTL digits', 'd79031d791', 'SUCCESS', 'd79031d791'),
+ -- Unicode 3.2, Corrigendum #4
+ ('U+2f868 from U+2136a to U+36fc', 'f0afa1a8' , 'SUCCESS', 'f0a18daa'),
+ ('U+2f874 from U+5f33 to U+5f53', 'f0afa1b4' , 'SUCCESS', 'e5bcb3'),
+ ('U+2f91f from U+43ab to U+243ab', 'f0afa49f' , 'SUCCESS', 'e48eab'),
+ ('U+2f95f from U+7aae to U+7aee', 'f0afa59f' , 'SUCCESS', 'e7aaae'),
+ ('U+2f9bf from U+4d57 to U+45d7', 'f0afa6bf' , 'SUCCESS', 'e4b597')
+)
+SELECT name,
+ actual.status,
+ encode(actual.res, 'hex') AS result,
+ actual.status = expected_status
+ AND encode(actual.res, 'hex')
+ IS NOT DISTINCT FROM expected_res AS passed
+FROM cases
+CROSS JOIN LATERAL test_saslprep(
+ decode(cases.src, 'hex')
+) AS actual
+ORDER BY name COLLATE "C";
+ name | status | result | passed
+--------------------------------+------------+--------------+--------
+ U+2f868 from U+2136a to U+36fc | SUCCESS | f0a18daa | t
+ U+2f874 from U+5f33 to U+5f53 | SUCCESS | e5bcb3 | t
+ U+2f91f from U+43ab to U+243ab | SUCCESS | e48eab | t
+ U+2f95f from U+7aae to U+7aee | SUCCESS | e7aaae | t
+ U+2f9bf from U+4d57 to U+45d7 | SUCCESS | e4b597 | t
+ alef after latin | PROHIBITED | | t
+ alef in RTL | SUCCESS | d790d790d791 | t
+ arabic ligature | PROHIBITED | | t
+ combining acute | SUCCESS | cc81 | t
+ combining grave | SUCCESS | cc80 | t
+ grave after a | SUCCESS | c3a0 | t
+ hebrew presentation | PROHIBITED | | t
+ hebrew shin | PROHIBITED | | t
+ latin ligature ff | SUCCESS | 6666 | t
+ rupee in RTL | PROHIBITED | | t
+ unassigned Greek | PROHIBITED | | t
+ unassigned modifier | PROHIBITED | | t
+ valid RTL digits | SUCCESS | d79031d791 | t
+(18 rows)
+
DROP EXTENSION test_saslprep;
diff --git a/src/test/modules/test_saslprep/sql/test_saslprep.sql b/src/test/modules/test_saslprep/sql/test_saslprep.sql
index 00bad48eca7..ea6024709af 100644
--- a/src/test/modules/test_saslprep/sql/test_saslprep.sql
+++ b/src/test/modules/test_saslprep/sql/test_saslprep.sql
@@ -16,4 +16,57 @@ SELECT
test_saslprep(set_byte('\x00'::bytea, 0, a)) AS saslprep
FROM generate_series(0,127) AS a;
+-- Test SASLprep validation before and after normalization.
+-- Prohibited output characters and bidi rules must be checked
+-- after normalization, while code points unassigned in Unicode
+-- 3.2 must be rejected before normalization.
+WITH cases(name, src, expected_status, expected_res) AS (
+ VALUES
+ -- Characters that become valid after normalization.
+ ('combining grave', 'cd80', 'SUCCESS', 'cc80'),
+ ('combining acute', 'cd81', 'SUCCESS', 'cc81'),
+ ('grave after a', '61cd80', 'SUCCESS', 'c3a0'),
+
+ -- Normalization can increase the number of code points.
+ ('latin ligature ff', 'efac80', 'SUCCESS', '6666'),
+
+ -- Normalization introduces prohibited bidi sequences.
+ ('hebrew presentation', 'efac9d', 'PROHIBITED', NULL),
+ ('hebrew shin', 'efacaa', 'PROHIBITED', NULL),
+ ('arabic ligature', 'efb19b', 'PROHIBITED', NULL),
+
+ -- Normalization changes bidi properties (L to R).
+ ('alef after latin', '41e284b5', 'PROHIBITED', NULL),
+ ('alef in RTL', 'd790e284b5d791', 'SUCCESS', 'd790d790d791'),
+
+ -- Normalization introduces L characters into an RTL string.
+ ('rupee in RTL', 'd790e282a8d791', 'PROHIBITED', NULL),
+
+ -- Code points unassigned in Unicode 3.2 that normalize
+ -- to assigned characters using newer Unicode data.
+ ('unassigned Greek', 'cfb9', 'PROHIBITED', NULL),
+ ('unassigned modifier', 'e1b4ac', 'PROHIBITED', NULL),
+
+ -- Control case: valid RTL string containing a digit.
+ ('valid RTL digits', 'd79031d791', 'SUCCESS', 'd79031d791'),
+
+ -- Unicode 3.2, Corrigendum #4
+ ('U+2f868 from U+2136a to U+36fc', 'f0afa1a8' , 'SUCCESS', 'f0a18daa'),
+ ('U+2f874 from U+5f33 to U+5f53', 'f0afa1b4' , 'SUCCESS', 'e5bcb3'),
+ ('U+2f91f from U+43ab to U+243ab', 'f0afa49f' , 'SUCCESS', 'e48eab'),
+ ('U+2f95f from U+7aae to U+7aee', 'f0afa59f' , 'SUCCESS', 'e7aaae'),
+ ('U+2f9bf from U+4d57 to U+45d7', 'f0afa6bf' , 'SUCCESS', 'e4b597')
+)
+SELECT name,
+ actual.status,
+ encode(actual.res, 'hex') AS result,
+ actual.status = expected_status
+ AND encode(actual.res, 'hex')
+ IS NOT DISTINCT FROM expected_res AS passed
+FROM cases
+CROSS JOIN LATERAL test_saslprep(
+ decode(cases.src, 'hex')
+) AS actual
+ORDER BY name COLLATE "C";
+
DROP EXTENSION test_saslprep;
--
2.53.0
