mihailom-db commented on code in PR #46180: URL: https://github.com/apache/spark/pull/46180#discussion_r1595376570
########## common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java: ########## @@ -117,76 +119,438 @@ public Collation( } /** - * Constructor with comparators that are inherited from the given collator. + * collation id (32-bit integer) layout: + * bit 31: 0 = predefined collation, 1 = user-defined collation + * bit 30-29: 00 = utf8-binary, 01 = ICU, 10 = indeterminate (without spec implementation) + * bit 28: 0 for utf8-binary / 0 = case-sensitive, 1 = case-insensitive for ICU + * bit 27: 0 for utf8-binary / 0 = accent-sensitive, 1 = accent-insensitive for ICU + * bit 26-25: zeroes, reserved for punctuation sensitivity + * bit 24-23: zeroes, reserved for first letter preference + * bit 22-21: 00 = unspecified, 01 = to-lower, 10 = to-upper + * bit 20-19: zeroes, reserved for space trimming + * bit 18-17: zeroes, reserved for version + * bit 16-12: zeroes + * bit 11-0: zeroes for utf8-binary / locale id for ICU */ - public Collation( - String collationName, - Collator collator, - String version, - boolean supportsBinaryEquality, - boolean supportsBinaryOrdering, - boolean supportsLowercaseEquality) { - this( - collationName, - collator, - (s1, s2) -> collator.compare(s1.toString(), s2.toString()), - version, - s -> (long)collator.getCollationKey(s.toString()).hashCode(), - supportsBinaryEquality, - supportsBinaryOrdering, - supportsLowercaseEquality); + private abstract static class CollationSpec { + protected enum ImplementationProvider { + UTF8_BINARY, ICU, INDETERMINATE + } + + protected enum CaseSensitivity { + CS, CI + } + + protected enum AccentSensitivity { + AS, AI + } + + protected enum CaseConversion { + UNSPECIFIED, LCASE, UCASE + } + + protected static final int IMPLEMENTATION_PROVIDER_OFFSET = 29; + protected static final int IMPLEMENTATION_PROVIDER_MASK = 0b11; + protected static final int CASE_SENSITIVITY_OFFSET = 28; + protected static final int CASE_SENSITIVITY_MASK = 0b1; + protected static final int ACCENT_SENSITIVITY_OFFSET = 27; + protected static final int ACCENT_SENSITIVITY_MASK = 0b1; + protected static final int CASE_CONVERSION_OFFSET = 21; + protected static final int CASE_CONVERSION_MASK = 0b11; + protected static final int LOCALE_OFFSET = 0; + protected static final int LOCALE_MASK = 0x0FFF; + + protected static final int INDETERMINATE_COLLATION_ID = + ImplementationProvider.INDETERMINATE.ordinal() << IMPLEMENTATION_PROVIDER_OFFSET; + + protected final CaseSensitivity caseSensitivity; + protected final AccentSensitivity accentSensitivity; + protected final CaseConversion caseConversion; + protected final String locale; + protected final int collationId; + + protected CollationSpec( + String locale, + CaseSensitivity caseSensitivity, + AccentSensitivity accentSensitivity, + CaseConversion caseConversion) { + this.locale = locale; + this.caseSensitivity = caseSensitivity; + this.accentSensitivity = accentSensitivity; + this.caseConversion = caseConversion; + this.collationId = getCollationId(); + } + + private static final Map<Integer, Collation> collationMap = new ConcurrentHashMap<>(); + + public static Collation fetchCollation(int collationId) throws SparkException { + if (collationMap.containsKey(collationId)) { + return collationMap.get(collationId); + } else { + CollationSpec spec; + int implementationProviderOrdinal = + (collationId >> IMPLEMENTATION_PROVIDER_OFFSET) & IMPLEMENTATION_PROVIDER_MASK; + if (implementationProviderOrdinal >= ImplementationProvider.values().length) { + throw SparkException.internalError("Invalid implementation provider"); + } else { + ImplementationProvider implementationProvider = ImplementationProvider.values()[ + implementationProviderOrdinal]; + if (implementationProvider == ImplementationProvider.UTF8_BINARY) { + spec = CollationSpecUTF8Binary.fromCollationId(collationId); + } else if (implementationProvider == ImplementationProvider.ICU) { + spec = CollationSpecICU.fromCollationId(collationId); + } else { + throw SparkException.internalError("Cannot instantiate indeterminate collation"); Review Comment: I believe for now it is not possible to create `INDETERMINATE` collation. What we do need is to be able to alias the values with this collation in select queries (this is a print of result of for example concat function). This is something tricky which I would say we could deal with after this PR, as we need to allow printing, but not storing of the INDETERMINATE collation, but for now I believe it is sufficient to just block users from creating INDETERMINATE collation explicitly with COLLATE in SQL. Same blocking should apply to pyspark and dataframe API. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org