dbatomic commented on code in PR #46180: URL: https://github.com/apache/spark/pull/46180#discussion_r1592436706
########## common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java: ########## @@ -117,76 +119,438 @@ public Collation( } /** - * Constructor with comparators that are inherited from the given collator. + * collation id (32-bit integer) layout: + * bit 31: 0 = predefined collation, 1 = user-defined collation + * bit 30-29: 00 = utf8-binary, 01 = ICU, 10 = indeterminate (without spec implementation) + * bit 28: 0 for utf8-binary / 0 = case-sensitive, 1 = case-insensitive for ICU + * bit 27: 0 for utf8-binary / 0 = accent-sensitive, 1 = accent-insensitive for ICU + * bit 26-25: zeroes, reserved for punctuation sensitivity + * bit 24-23: zeroes, reserved for first letter preference + * bit 22-21: 00 = unspecified, 01 = to-lower, 10 = to-upper + * bit 20-19: zeroes, reserved for space trimming + * bit 18-17: zeroes, reserved for version + * bit 16-12: zeroes + * bit 11-0: zeroes for utf8-binary / locale id for ICU */ - public Collation( - String collationName, - Collator collator, - String version, - boolean supportsBinaryEquality, - boolean supportsBinaryOrdering, - boolean supportsLowercaseEquality) { - this( - collationName, - collator, - (s1, s2) -> collator.compare(s1.toString(), s2.toString()), - version, - s -> (long)collator.getCollationKey(s.toString()).hashCode(), - supportsBinaryEquality, - supportsBinaryOrdering, - supportsLowercaseEquality); + private abstract static class CollationSpec { + protected enum ImplementationProvider { + UTF8_BINARY, ICU, INDETERMINATE + } + + protected enum CaseSensitivity { + CS, CI + } + + protected enum AccentSensitivity { + AS, AI + } + + protected enum CaseConversion { + UNSPECIFIED, LCASE, UCASE + } + + protected static final int IMPLEMENTATION_PROVIDER_OFFSET = 29; + protected static final int IMPLEMENTATION_PROVIDER_MASK = 0b11; + protected static final int CASE_SENSITIVITY_OFFSET = 28; + protected static final int CASE_SENSITIVITY_MASK = 0b1; + protected static final int ACCENT_SENSITIVITY_OFFSET = 27; + protected static final int ACCENT_SENSITIVITY_MASK = 0b1; + protected static final int CASE_CONVERSION_OFFSET = 21; + protected static final int CASE_CONVERSION_MASK = 0b11; + protected static final int LOCALE_OFFSET = 0; + protected static final int LOCALE_MASK = 0x0FFF; + + protected static final int INDETERMINATE_COLLATION_ID = + ImplementationProvider.INDETERMINATE.ordinal() << IMPLEMENTATION_PROVIDER_OFFSET; + + protected final CaseSensitivity caseSensitivity; + protected final AccentSensitivity accentSensitivity; + protected final CaseConversion caseConversion; + protected final String locale; + protected final int collationId; + + protected CollationSpec( + String locale, + CaseSensitivity caseSensitivity, + AccentSensitivity accentSensitivity, + CaseConversion caseConversion) { + this.locale = locale; + this.caseSensitivity = caseSensitivity; + this.accentSensitivity = accentSensitivity; + this.caseConversion = caseConversion; + this.collationId = getCollationId(); + } + + private static final Map<Integer, Collation> collationMap = new ConcurrentHashMap<>(); + + public static Collation fetchCollation(int collationId) throws SparkException { + if (collationMap.containsKey(collationId)) { + return collationMap.get(collationId); + } else { + CollationSpec spec; + int implementationProviderOrdinal = + (collationId >> IMPLEMENTATION_PROVIDER_OFFSET) & IMPLEMENTATION_PROVIDER_MASK; + if (implementationProviderOrdinal >= ImplementationProvider.values().length) { + throw SparkException.internalError("Invalid implementation provider"); Review Comment: Can you just add in error message that this has something to do with collations? e.g. "Invalid collation implementation provider" so it's easier to root cause if we ever hit this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org