This is an automated email from the ASF dual-hosted git repository. jhyde pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 2a96512c352bda4a5d9c0c80730f5c115ac363d6 Author: Julian Hyde <[email protected]> AuthorDate: Tue Aug 22 23:22:13 2023 -0700 [CALCITE-5914] Cache compiled regular expressions in SQL function runtime Functions/operators that now cache include REGEXP_CONTAINS, PARSE_URL, SIMILAR TO, LIKE, ILIKE, RLIKE, posix regex, PARSE_TIMESTAMP, FORMAT_DATE, TO_CHAR. Introduce `class RegexFunction` and use it to implement variants of REGEXP_REPLACE. Add CalciteSystemProperty.FUNCTION_LEVEL_CACHE_MAX_SIZE to limit the number of entries in a function's cache. Close apache/calcite#3394 --- .../calcite/adapter/enumerable/RexImpTable.java | 271 ++++----- .../calcite/config/CalciteSystemProperty.java | 46 +- .../org/apache/calcite/rex/RexInterpreter.java | 19 +- .../main/java/org/apache/calcite/runtime/Like.java | 12 +- .../org/apache/calcite/runtime/SqlFunctions.java | 647 ++++++++++++++------- .../org/apache/calcite/util/BuiltInMethod.java | 69 ++- .../apache/calcite/util/format/FormatModel.java | 3 + .../apache/calcite/util/format/FormatModels.java | 4 +- .../org/apache/calcite/test/SqlFunctionsTest.java | 119 ++-- .../main/java/org/apache/calcite/linq4j/Ord.java | 19 +- 10 files changed, 721 insertions(+), 488 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index df3073699f..bfc6342a35 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -385,6 +385,8 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.MULTISET_INTERSECT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.MULTISET_INTERSECT_DISTINCT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.MULTISET_UNION; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.MULTISET_UNION_DISTINCT; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.NEGATED_POSIX_REGEX_CASE_INSENSITIVE; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.NEGATED_POSIX_REGEX_CASE_SENSITIVE; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.NEXT_VALUE; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.NOT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.NOT_EQUALS; @@ -397,6 +399,8 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.OVERLAY; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.PI; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.PLUS; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.POSITION; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.POSIX_REGEX_CASE_INSENSITIVE; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.POWER; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RADIANS; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RAND; @@ -562,9 +566,9 @@ public class RexImpTable { defineMethod(REVERSE, BuiltInMethod.REVERSE.method, NullPolicy.STRICT); defineMethod(LEVENSHTEIN, BuiltInMethod.LEVENSHTEIN.method, NullPolicy.STRICT); defineMethod(SPLIT, BuiltInMethod.SPLIT.method, NullPolicy.STRICT); - defineMethod(PARSE_URL, BuiltInMethod.PARSE_URL.method, NullPolicy.STRICT); - defineMethod(REGEXP_CONTAINS, BuiltInMethod.REGEXP_CONTAINS.method, - NullPolicy.STRICT); + defineReflective(PARSE_URL, BuiltInMethod.PARSE_URL2.method, + BuiltInMethod.PARSE_URL3.method); + defineReflective(REGEXP_CONTAINS, BuiltInMethod.REGEXP_CONTAINS.method); map.put(TRIM, new TrimImplementor()); @@ -603,8 +607,10 @@ public class RexImpTable { map.put(LOG, new LogImplementor()); map.put(LOG10, new LogImplementor()); - map.put(RAND, new RandImplementor()); - map.put(RAND_INTEGER, new RandIntegerImplementor()); + defineReflective(RAND, BuiltInMethod.RAND.method, + BuiltInMethod.RAND_SEED.method); + defineReflective(RAND_INTEGER, BuiltInMethod.RAND_INTEGER.method, + BuiltInMethod.RAND_INTEGER_SEED.method); defineMethod(ACOS, BuiltInMethod.ACOS.method, NullPolicy.STRICT); defineMethod(ACOSH, BuiltInMethod.ACOSH.method, NullPolicy.STRICT); @@ -721,17 +727,13 @@ public class RexImpTable { NullPolicy.STRICT); // Datetime parsing methods - defineMethod(PARSE_DATE, BuiltInMethod.PARSE_DATE.method, - NullPolicy.STRICT); - defineMethod(PARSE_DATETIME, BuiltInMethod.PARSE_DATETIME.method, - NullPolicy.STRICT); - defineMethod(PARSE_TIME, BuiltInMethod.PARSE_TIME.method, - NullPolicy.STRICT); - defineMethod(PARSE_TIMESTAMP, BuiltInMethod.PARSE_TIMESTAMP.method, - NullPolicy.STRICT); + defineReflective(PARSE_DATE, BuiltInMethod.PARSE_DATE.method); + defineReflective(PARSE_DATETIME, BuiltInMethod.PARSE_DATETIME.method); + defineReflective(PARSE_TIME, BuiltInMethod.PARSE_TIME.method); + defineReflective(PARSE_TIMESTAMP, BuiltInMethod.PARSE_TIMESTAMP.method); // Datetime formatting methods - defineMethod(TO_CHAR, BuiltInMethod.TO_CHAR.method, NullPolicy.STRICT); + defineReflective(TO_CHAR, BuiltInMethod.TO_CHAR.method); final FormatDatetimeImplementor datetimeFormatImpl = new FormatDatetimeImplementor(); map.put(FORMAT_DATE, datetimeFormatImpl); @@ -747,34 +749,31 @@ public class RexImpTable { map.put(IS_FALSE, new IsFalseImplementor()); map.put(IS_NOT_FALSE, new IsNotFalseImplementor()); - // LIKE, ILIKE and SIMILAR - map.put(LIKE, - new MethodImplementor(BuiltInMethod.LIKE.method, NullPolicy.STRICT, - false)); - map.put(ILIKE, - new MethodImplementor(BuiltInMethod.ILIKE.method, NullPolicy.STRICT, - false)); - map.put(RLIKE, - new MethodImplementor(BuiltInMethod.RLIKE.method, NullPolicy.STRICT, - false)); - map.put(SIMILAR_TO, - new MethodImplementor(BuiltInMethod.SIMILAR.method, NullPolicy.STRICT, - false)); + // LIKE, ILIKE, RLIKE and SIMILAR + defineReflective(LIKE, BuiltInMethod.LIKE.method, + BuiltInMethod.LIKE_ESCAPE.method); + defineReflective(ILIKE, BuiltInMethod.ILIKE.method, + BuiltInMethod.ILIKE_ESCAPE.method); + defineReflective(RLIKE, BuiltInMethod.RLIKE.method); + defineReflective(SIMILAR_TO, BuiltInMethod.SIMILAR.method, + BuiltInMethod.SIMILAR_ESCAPE.method); // POSIX REGEX - final MethodImplementor posixRegexImplementorCaseSensitive = - new PosixRegexMethodImplementor(true); - final MethodImplementor posixRegexImplementorCaseInsensitive = - new PosixRegexMethodImplementor(false); - map.put(SqlStdOperatorTable.POSIX_REGEX_CASE_INSENSITIVE, - posixRegexImplementorCaseInsensitive); - map.put(SqlStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE, - posixRegexImplementorCaseSensitive); - map.put(SqlStdOperatorTable.NEGATED_POSIX_REGEX_CASE_INSENSITIVE, - NotImplementor.of(posixRegexImplementorCaseInsensitive)); - map.put(SqlStdOperatorTable.NEGATED_POSIX_REGEX_CASE_SENSITIVE, - NotImplementor.of(posixRegexImplementorCaseSensitive)); - map.put(REGEXP_REPLACE, new RegexpReplaceImplementor()); + ReflectiveImplementor insensitiveImplementor = + defineReflective(POSIX_REGEX_CASE_INSENSITIVE, + BuiltInMethod.POSIX_REGEX_INSENSITIVE.method); + ReflectiveImplementor sensitiveImplementor = + defineReflective(POSIX_REGEX_CASE_SENSITIVE, + BuiltInMethod.POSIX_REGEX_SENSITIVE.method); + map.put(NEGATED_POSIX_REGEX_CASE_INSENSITIVE, + NotImplementor.of(insensitiveImplementor)); + map.put(NEGATED_POSIX_REGEX_CASE_SENSITIVE, + NotImplementor.of(sensitiveImplementor)); + defineReflective(REGEXP_REPLACE, + BuiltInMethod.REGEXP_REPLACE3.method, + BuiltInMethod.REGEXP_REPLACE4.method, + BuiltInMethod.REGEXP_REPLACE5.method, + BuiltInMethod.REGEXP_REPLACE6.method); // Multisets & arrays defineMethod(CARDINALITY, BuiltInMethod.COLLECTION_SIZE.method, @@ -1062,6 +1061,14 @@ public class RexImpTable { map.put(operator, new MethodImplementor(method, nullPolicy, false)); } + private ReflectiveImplementor defineReflective(SqlOperator operator, + Method... methods) { + final ReflectiveImplementor implementor = + new ReflectiveImplementor(ImmutableList.copyOf(methods)); + map.put(operator, implementor); + return implementor; + } + private void defineUnary(SqlOperator operator, ExpressionType expressionType, NullPolicy nullPolicy, @Nullable String backupMethodName) { map.put(operator, new UnaryImplementor(expressionType, nullPolicy, backupMethodName)); @@ -2478,7 +2485,7 @@ public class RexImpTable { // Because BigQuery treats all int types as aliases for BIGINT (Java's long) // they can all be converted to LONG to minimize entries in the SqlFunctions class. - private Expression convertType(Expression arg, RexNode node) { + static Expression convertType(Expression arg, RexNode node) { if (SqlTypeName.INT_TYPES.contains(node.getType().getSqlTypeName())) { return Expressions.convert_(arg, long.class); } else { @@ -2659,9 +2666,12 @@ public class RexImpTable { * {@code FORMAT_TIME} and {@code FORMAT_DATETIME} functions. */ private static class FormatDatetimeImplementor - extends AbstractRexCallImplementor { + extends ReflectiveImplementor { FormatDatetimeImplementor() { - super("formatDatetime", NullPolicy.STRICT, false); + super( + ImmutableList.of(BuiltInMethod.FORMAT_DATE.method, + BuiltInMethod.FORMAT_TIME.method, + BuiltInMethod.FORMAT_TIMESTAMP.method)); } @Override Expression implementSafe(final RexToLixTranslator translator, @@ -2679,7 +2689,8 @@ public class RexImpTable { default: method = BuiltInMethod.FORMAT_TIMESTAMP.method; } - return Expressions.call(method, translator.getRoot(), operand0, operand1); + return implementSafe(method, + ImmutableList.of(translator.getRoot(), operand0, operand1)); } } @@ -2687,13 +2698,13 @@ public class RexImpTable { private static class MethodImplementor extends AbstractRexCallImplementor { protected final Method method; - MethodImplementor(String variableName, Method method, - @Nullable NullPolicy nullPolicy, boolean harmonize) { + MethodImplementor(String variableName, Method method, NullPolicy nullPolicy, + boolean harmonize) { super(variableName, nullPolicy, harmonize); this.method = method; } - MethodImplementor(Method method, @Nullable NullPolicy nullPolicy, + MethodImplementor(Method method, NullPolicy nullPolicy, boolean harmonize) { this("method_call", method, nullPolicy, harmonize); } @@ -2725,25 +2736,6 @@ public class RexImpTable { } } - /** Implementor for {@link org.apache.calcite.sql.fun.SqlPosixRegexOperator}s. */ - private static class PosixRegexMethodImplementor extends MethodImplementor { - protected final boolean caseSensitive; - - PosixRegexMethodImplementor(boolean caseSensitive) { - super(BuiltInMethod.POSIX_REGEX.method, NullPolicy.STRICT, false); - this.caseSensitive = caseSensitive; - } - - @Override Expression implementSafe(RexToLixTranslator translator, - RexCall call, List<Expression> argValueList) { - assert argValueList.size() == 2; - // Add extra parameter (caseSensitive boolean flag), required by SqlFunctions#posixRegex. - final List<Expression> newOperands = new ArrayList<>(argValueList); - newOperands.add(Expressions.constant(caseSensitive)); - return super.implementSafe(translator, call, newOperands); - } - } - /** * Implementor for JSON_VALUE function, convert to solid format * "JSON_VALUE(json_doc, path, empty_behavior, empty_default, error_behavior, error default)" @@ -3473,7 +3465,7 @@ public class RexImpTable { private final AbstractRexCallImplementor implementor; private NotImplementor(AbstractRexCallImplementor implementor) { - super("not", implementor.getNullPolicy(), false); + super("not", implementor.nullPolicy, false); this.implementor = implementor; } @@ -3713,13 +3705,13 @@ public class RexImpTable { /** Variable name should be meaningful. It helps us debug issues. */ final String variableName; - final @Nullable NullPolicy nullPolicy; - private final boolean harmonize; + final NullPolicy nullPolicy; + final boolean harmonize; AbstractRexCallImplementor(String variableName, - @Nullable NullPolicy nullPolicy, boolean harmonize) { + NullPolicy nullPolicy, boolean harmonize) { this.variableName = requireNonNull(variableName, "variableName"); - this.nullPolicy = nullPolicy; + this.nullPolicy = requireNonNull(nullPolicy, "nullPolicy"); this.harmonize = harmonize; } @@ -3741,19 +3733,9 @@ public class RexImpTable { return new RexToLixTranslator.Result(isNullVariable, valueVariable); } - // Variable name facilitates reasoning about issues when necessary - String getVariableName() { - return variableName; - } - - @Nullable NullPolicy getNullPolicy() { - return nullPolicy; - } - /** Figures out conditional expression according to NullPolicy. */ Expression getCondition(final List<Expression> argIsNullList) { if (argIsNullList.isEmpty() - || nullPolicy == null || nullPolicy == NullPolicy.NONE) { return FALSE_EXPR; } @@ -3807,7 +3789,7 @@ public class RexImpTable { convertedCallValue); final ParameterExpression value = Expressions.parameter(convertedCallValue.getType(), - translator.getBlockBuilder().newName(getVariableName() + "_value")); + translator.getBlockBuilder().newName(variableName + "_value")); translator.getBlockBuilder().add( Expressions.declare(Modifier.FINAL, value, valueExpression)); return value; @@ -3822,7 +3804,7 @@ public class RexImpTable { final RexToLixTranslator translator, final ParameterExpression value) { final ParameterExpression isNullVariable = Expressions.parameter(Boolean.TYPE, - translator.getBlockBuilder().newName(getVariableName() + "_isNull")); + translator.getBlockBuilder().newName(variableName + "_isNull")); final Expression isNullExpression = translator.checkNull(value); translator.getBlockBuilder().add( Expressions.declare(Modifier.FINAL, isNullVariable, isNullExpression)); @@ -3872,18 +3854,21 @@ public class RexImpTable { /** Under null check, it is safe to unbox the operands before entering the * implementor. */ private List<Expression> unboxIfNecessary(final List<Expression> argValueList) { - List<Expression> unboxValueList = argValueList; - if (nullPolicy == NullPolicy.STRICT || nullPolicy == NullPolicy.ANY - || nullPolicy == NullPolicy.SEMI_STRICT) { - unboxValueList = argValueList.stream() - .map(AbstractRexCallImplementor::unboxExpression) - .collect(Collectors.toList()); - } - if (nullPolicy == NullPolicy.ARG0 && !argValueList.isEmpty()) { - final Expression unboxArg0 = unboxExpression(unboxValueList.get(0)); - unboxValueList.set(0, unboxArg0); + switch (nullPolicy) { + case STRICT: + case ANY: + case SEMI_STRICT: + return Util.transform(argValueList, + AbstractRexCallImplementor::unboxExpression); + case ARG0: + if (!argValueList.isEmpty()) { + final Expression unboxArg0 = unboxExpression(argValueList.get(0)); + argValueList.set(0, unboxArg0); + } + // fall through + default: + return argValueList; } - return unboxValueList; } private static Expression unboxExpression(final Expression argValue) { @@ -3944,11 +3929,11 @@ public class RexImpTable { final Expression valueExpression = nullAs.handle(callExpression); final ParameterExpression valueVariable = Expressions.parameter(valueExpression.getType(), - translator.getBlockBuilder().newName(getVariableName() + "_value")); + translator.getBlockBuilder().newName(variableName + "_value")); final Expression isNullExpression = translator.checkNull(valueVariable); final ParameterExpression isNullVariable = Expressions.parameter(Boolean.TYPE, - translator.getBlockBuilder().newName(getVariableName() + "_isNull")); + translator.getBlockBuilder().newName(variableName + "_isNull")); translator.getBlockBuilder().add( Expressions.declare(Modifier.FINAL, valueVariable, valueExpression)); translator.getBlockBuilder().add( @@ -3997,11 +3982,11 @@ public class RexImpTable { final Expression valueExpression = nullAs.handle(callExpression); final ParameterExpression valueVariable = Expressions.parameter(valueExpression.getType(), - translator.getBlockBuilder().newName(getVariableName() + "_value")); + translator.getBlockBuilder().newName(variableName + "_value")); final Expression isNullExpression = translator.checkNull(valueExpression); final ParameterExpression isNullVariable = Expressions.parameter(Boolean.TYPE, - translator.getBlockBuilder().newName(getVariableName() + "_isNull")); + translator.getBlockBuilder().newName(variableName + "_isNull")); translator.getBlockBuilder().add( Expressions.declare(Modifier.FINAL, valueVariable, valueExpression)); translator.getBlockBuilder().add( @@ -4110,70 +4095,48 @@ public class RexImpTable { } /** - * Implementation that calls a given {@link java.lang.reflect.Method}. + * Implementation that a {@link java.lang.reflect.Method}. + * + * <p>If there are several methods in the list, calls the first that has the + * right number of arguments. * * <p>When method is not static, a new instance of the required class is * created. */ private static class ReflectiveImplementor extends AbstractRexCallImplementor { - protected final Method method; + protected final ImmutableList<? extends Method> methods; - ReflectiveImplementor(Method method, @Nullable NullPolicy nullPolicy) { - super("reflective_" + method.getName(), nullPolicy, false); - this.method = requireNonNull(method, "method"); + ReflectiveImplementor(List<? extends Method> methods) { + super("reflective_" + methods.get(0).getName(), NullPolicy.STRICT, false); + this.methods = ImmutableList.copyOf(methods); } @Override Expression implementSafe(RexToLixTranslator translator, RexCall call, List<Expression> argValueList) { + for (Method method : methods) { + if (method.getParameterCount() == argValueList.size()) { + return implementSafe(method, argValueList); + } + } + throw new IllegalArgumentException("no matching method"); + } + + protected MethodCallExpression implementSafe(Method method, + List<Expression> argValueList) { List<Expression> argValueList0 = - EnumUtils.fromInternal(method.getParameterTypes(), argValueList); + EnumUtils.fromInternal(method.getParameterTypes(), + argValueList); if (isStatic(method)) { return Expressions.call(method, argValueList0); } else { - // The UDF class must have a public zero-args constructor. - // Assume that the validator checked already. - final Expression target = Expressions.new_(method.getDeclaringClass()); + // The class must have a public zero-args constructor. + final Expression target = + Expressions.new_(method.getDeclaringClass()); return Expressions.call(target, method, argValueList0); } } } - /** Implementor for the {@code RAND} function. */ - private static class RandImplementor extends AbstractRexCallImplementor { - private final AbstractRexCallImplementor[] implementors = { - new ReflectiveImplementor(BuiltInMethod.RAND.method, nullPolicy), - new ReflectiveImplementor(BuiltInMethod.RAND_SEED.method, nullPolicy) - }; - - RandImplementor() { - super("rand", NullPolicy.STRICT, false); - } - - @Override Expression implementSafe(final RexToLixTranslator translator, - final RexCall call, final List<Expression> argValueList) { - return implementors[call.getOperands().size()] - .implementSafe(translator, call, argValueList); - } - } - - /** Implementor for the {@code RAND_INTEGER} function. */ - private static class RandIntegerImplementor extends AbstractRexCallImplementor { - private final AbstractRexCallImplementor[] implementors = { - new ReflectiveImplementor(BuiltInMethod.RAND_INTEGER.method, nullPolicy), - new ReflectiveImplementor(BuiltInMethod.RAND_INTEGER_SEED.method, nullPolicy) - }; - - RandIntegerImplementor() { - super("rand_integer", NullPolicy.STRICT, false); - } - - @Override Expression implementSafe(final RexToLixTranslator translator, - final RexCall call, final List<Expression> argValueList) { - return implementors[call.getOperands().size() - 1] - .implementSafe(translator, call, argValueList); - } - } - /** Implementor for the {@code PI} operator. */ private static class PiImplementor extends AbstractRexCallImplementor { PiImplementor() { @@ -4282,26 +4245,6 @@ public class RexImpTable { } } - /** Implementor for the {@code REGEXP_REPLACE} function. */ - private static class RegexpReplaceImplementor extends AbstractRexCallImplementor { - private final AbstractRexCallImplementor[] implementors = { - new ReflectiveImplementor(BuiltInMethod.REGEXP_REPLACE3.method, nullPolicy), - new ReflectiveImplementor(BuiltInMethod.REGEXP_REPLACE4.method, nullPolicy), - new ReflectiveImplementor(BuiltInMethod.REGEXP_REPLACE5.method, nullPolicy), - new ReflectiveImplementor(BuiltInMethod.REGEXP_REPLACE6.method, nullPolicy), - }; - - RegexpReplaceImplementor() { - super("regexp_replace", NullPolicy.STRICT, false); - } - - @Override Expression implementSafe(RexToLixTranslator translator, - RexCall call, List<Expression> argValueList) { - return implementors[call.getOperands().size() - 3] - .implementSafe(translator, call, argValueList); - } - } - /** Implementor for the {@code DEFAULT} function. */ private static class DefaultImplementor extends AbstractRexCallImplementor { DefaultImplementor() { @@ -4423,7 +4366,7 @@ public class RexImpTable { BlockBuilder lambdaBuilder = new BlockBuilder(); final ParameterExpression leftExpr = Expressions.parameter(left.getType(), - translator.getBlockBuilder().newName("_" + getVariableName() + "_left_value")); + translator.getBlockBuilder().newName("_" + variableName + "_left_value")); // left should have final modifier otherwise it can not be passed to lambda translator.getBlockBuilder().add(Expressions.declare(Modifier.FINAL, leftExpr, left)); RexNode leftRex = call.getOperands().get(0); diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 0e7281f11a..963626fd10 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -251,7 +251,8 @@ public final class CalciteSystemProperty<T> { booleanProperty("calcite.test.redis", true); /** - * Whether to use Docker containers (https://www.testcontainers.org/) in tests. + * Whether to use + * <a href="https://www.testcontainers.org/";>Docker containers</a> in tests. * * <p>If the property is set to <code>true</code>, affected tests will attempt * to start Docker containers; when Docker is not available tests fallback to @@ -303,7 +304,7 @@ public final class CalciteSystemProperty<T> { * The name of the default national character set. * * <p>It is used with the N'string' construct in - * {@link org.apache.calcite.sql.SqlLiteral#SqlLiteral} + * {@link org.apache.calcite.sql.SqlLiteral} * and may be different from the {@link #DEFAULT_CHARSET}. */ // TODO review zabetak: @@ -315,7 +316,7 @@ public final class CalciteSystemProperty<T> { * The name of the default collation. * * <p>It is used in {@link org.apache.calcite.sql.SqlCollation} and - * {@link org.apache.calcite.sql.SqlLiteral#SqlLiteral}. + * {@link org.apache.calcite.sql.SqlLiteral}. */ // TODO review zabetak: // What happens if a wrong value is specified? @@ -328,7 +329,7 @@ public final class CalciteSystemProperty<T> { * tertiary, identical. * * <p>It is used in {@link org.apache.calcite.sql.SqlCollation} and - * {@link org.apache.calcite.sql.SqlLiteral#SqlLiteral}. + * {@link org.apache.calcite.sql.SqlLiteral}. */ // TODO review zabetak: // What happens if a wrong value is specified? @@ -364,7 +365,7 @@ public final class CalciteSystemProperty<T> { * <p>Setting this property to 0 disables the cache. */ public static final CalciteSystemProperty<Integer> BINDABLE_CACHE_MAX_SIZE = - intProperty("calcite.bindable.cache.maxSize", 0, v -> v >= 0 && v <= Integer.MAX_VALUE); + intProperty("calcite.bindable.cache.maxSize", 0, v -> v >= 0); /** * The concurrency level of the cache used for storing Bindable objects, @@ -380,15 +381,44 @@ public final class CalciteSystemProperty<T> { * {@link #BINDABLE_CACHE_MAX_SIZE} set to 0. */ public static final CalciteSystemProperty<Integer> BINDABLE_CACHE_CONCURRENCY_LEVEL = - intProperty("calcite.bindable.cache.concurrencyLevel", 1, - v -> v >= 1 && v <= Integer.MAX_VALUE); + intProperty("calcite.bindable.cache.concurrencyLevel", 1, v -> v >= 1); + + /** + * The maximum number of items in a function-level cache. + * + * <p>A few SQL functions have expensive processing that, if its results are + * cached, can be reused by future calls to the function. One such function + * is {@code RLIKE}, whose arguments are a regular expression and a string. + * The regular expression needs to be compiled to a + * {@link java.util.regex.Pattern}. Compilation is expensive, and within a + * particular query, the arguments are often the same string, or a small + * number of distinct strings, so caching makes sense. + * + * <p>Therefore, functions such as {@code RLIKE}, {@code SIMILAR TO}, + * {@code PARSE_URL}, {@code PARSE_TIMESTAMP}, {@code FORMAT_DATE} have a + * function-level cache. The cache is created in the code generated for the + * query, at the call site of the function, and expires when the query has + * finished executing. Such caches do not need time-based expiration, but + * we need to cap the size of the cache to deal with scenarios such as a + * billion-row table where every row has a distinct regular expression. + * + * <p>Because of how Calcite generates and executes code in Enumerable + * convention, each function object is used from a single thread. Therefore, + * non thread-safe objects such as {@link java.text.DateFormat} can be safely + * cached. + * + * <p>The value of this parameter limits the size of every function-level + * cache in Calcite. The default value is 1,000. + */ + public static final CalciteSystemProperty<Integer> FUNCTION_LEVEL_CACHE_MAX_SIZE = + intProperty("calcite.function.cache.maxSize", 0, v -> v >= 0); private static CalciteSystemProperty<Boolean> booleanProperty(String key, boolean defaultValue) { // Note that "" -> true (convenient for command-lines flags like '-Dflag') return new CalciteSystemProperty<>(key, v -> v == null ? defaultValue - : "".equals(v) || Boolean.parseBoolean(v)); + : v.isEmpty() || Boolean.parseBoolean(v)); } private static CalciteSystemProperty<Integer> intProperty(String key, int defaultValue) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java b/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java index 80adc5ca5f..c495dddd35 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java +++ b/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java @@ -68,6 +68,13 @@ public class RexInterpreter implements RexVisitor<Comparable> { SqlKind.DIVIDE, SqlKind.COALESCE, SqlKind.CEIL, SqlKind.FLOOR, SqlKind.EXTRACT); + private final SqlFunctions.LikeFunction likeFunction = + new SqlFunctions.LikeFunction(); + private final SqlFunctions.SimilarFunction similarFunction = + new SqlFunctions.SimilarFunction(); + private final SqlFunctions.SimilarEscapeFunction similarEscapeFunction = + new SqlFunctions.SimilarEscapeFunction(); + private final Map<RexNode, Comparable> environment; /** Creates an interpreter. @@ -245,7 +252,7 @@ public class RexInterpreter implements RexVisitor<Comparable> { return DateTimeUtils.unixDateExtract(timeUnitRange, v2); } - private static Comparable like(List<Comparable> values) { + private Comparable like(List<Comparable> values) { if (containsNull(values)) { return N; } @@ -253,17 +260,17 @@ public class RexInterpreter implements RexVisitor<Comparable> { final NlsString pattern = (NlsString) values.get(1); switch (values.size()) { case 2: - return SqlFunctions.like(value.getValue(), pattern.getValue()); + return likeFunction.like(value.getValue(), pattern.getValue()); case 3: final NlsString escape = (NlsString) values.get(2); - return SqlFunctions.like(value.getValue(), pattern.getValue(), + return likeFunction.like(value.getValue(), pattern.getValue(), escape.getValue()); default: throw new AssertionError(); } } - private static Comparable similar(List<Comparable> values) { + private Comparable similar(List<Comparable> values) { if (containsNull(values)) { return N; } @@ -271,10 +278,10 @@ public class RexInterpreter implements RexVisitor<Comparable> { final NlsString pattern = (NlsString) values.get(1); switch (values.size()) { case 2: - return SqlFunctions.similar(value.getValue(), pattern.getValue()); + return similarFunction.similar(value.getValue(), pattern.getValue()); case 3: final NlsString escape = (NlsString) values.get(2); - return SqlFunctions.similar(value.getValue(), pattern.getValue(), + return similarEscapeFunction.similar(value.getValue(), pattern.getValue(), escape.getValue()); default: throw new AssertionError(); diff --git a/core/src/main/java/org/apache/calcite/runtime/Like.java b/core/src/main/java/org/apache/calcite/runtime/Like.java index 2fa8665629..ac074afa3d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Like.java +++ b/core/src/main/java/org/apache/calcite/runtime/Like.java @@ -310,16 +310,22 @@ public class Like { return javaPattern.toString(); } - static java.util.regex.Pattern posixRegexToPattern(String regex, boolean caseSensitive) { + static java.util.regex.Pattern posixRegexToPattern(String regex, + boolean caseSensitive) { + int flags = caseSensitive ? 0 : java.util.regex.Pattern.CASE_INSENSITIVE; + return posixRegexToPattern(regex, flags); + } + + static java.util.regex.Pattern posixRegexToPattern(String regex, int flags) { // Replace existing character classes with java equivalent ones String originalRegex = regex; String[] existingExpressions = Arrays.stream(POSIX_CHARACTER_CLASSES) - .filter(v -> originalRegex.contains(v.toLowerCase(Locale.ROOT))).toArray(String[]::new); + .filter(v -> originalRegex.contains(v.toLowerCase(Locale.ROOT))) + .toArray(String[]::new); for (String v : existingExpressions) { regex = regex.replace(v.toLowerCase(Locale.ROOT), "\\p{" + v + "}"); } - int flags = caseSensitive ? 0 : java.util.regex.Pattern.CASE_INSENSITIVE; return java.util.regex.Pattern.compile(regex, flags); } } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 6837fbb9dc..648093b57f 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -27,6 +27,7 @@ import org.apache.calcite.linq4j.CartesianProductEnumerator; import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.function.Deterministic; import org.apache.calcite.linq4j.function.Experimental; import org.apache.calcite.linq4j.function.Function1; @@ -43,6 +44,7 @@ import org.apache.calcite.util.TimestampWithTimeZoneString; import org.apache.calcite.util.Unsafe; import org.apache.calcite.util.Util; import org.apache.calcite.util.format.FormatElement; +import org.apache.calcite.util.format.FormatModel; import org.apache.calcite.util.format.FormatModels; import org.apache.commons.codec.DecoderException; @@ -55,7 +57,11 @@ import org.apache.commons.text.similarity.LevenshteinDistance; import com.google.common.base.Splitter; import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.UncheckedExecutionException; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; @@ -72,6 +78,7 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.sql.SQLException; +import java.sql.Time; import java.sql.Timestamp; import java.text.DateFormat; import java.text.DecimalFormat; @@ -106,14 +113,18 @@ import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BinaryOperator; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import static org.apache.calcite.config.CalciteSystemProperty.FUNCTION_LEVEL_CACHE_MAX_SIZE; import static org.apache.calcite.linq4j.Nullness.castNonNull; import static org.apache.calcite.util.Static.RESOURCE; @@ -354,54 +365,87 @@ public class SqlFunctions { return DigestUtils.sha512Hex(string.getBytes()); } - /** SQL {@code REGEXP_CONTAINS(value, regexp)} function. - * Throws a runtime exception for invalid regular expressions.*/ - public static boolean regexpContains(String value, String regex) { - try { - // Uses java.util.regex as a standard for regex processing - // in Calcite instead of RE2 used by BigQuery/GoogleSQL - Pattern regexp = Pattern.compile(regex); - return regexp.matcher(value).find(); - } catch (PatternSyntaxException ex) { - throw RESOURCE.invalidInputForRegexpContains(ex.getMessage().replace("\r\n", " ") - .replace("\n", " ").replace("\r", " ")).ex(); + /** State for {@code REGEXP_CONTAINS}, {@code REGEXP_REPLACE}, {@code RLIKE}. + * + * <p>Marked deterministic so that the code generator instantiates one once + * per query, not once per row. */ + @Deterministic + public static class RegexFunction { + /** Cache key. */ + private static class Key extends Ord<String> { + Key(int flags, String regex) { + super(flags, regex); + } + + @SuppressWarnings("MagicConstant") + Pattern toPattern() { + return Pattern.compile(e, i); + } } - } - /** SQL {@code REGEXP_REPLACE} function with 3 arguments. */ - public static String regexpReplace(String s, String regex, - String replacement) { - return regexpReplace(s, regex, replacement, 1, 0, null); - } + private final LoadingCache<Key, Pattern> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build(CacheLoader.from(Key::toPattern)); - /** SQL {@code REGEXP_REPLACE} function with 4 arguments. */ - public static String regexpReplace(String s, String regex, String replacement, - int pos) { - return regexpReplace(s, regex, replacement, pos, 0, null); - } + /** SQL {@code REGEXP_CONTAINS(value, regexp)} function. + * Throws a runtime exception for invalid regular expressions.*/ + public boolean regexpContains(String value, String regex) { + final Pattern pattern; + try { + // Uses java.util.regex as a standard for regex processing + // in Calcite instead of RE2 used by BigQuery/GoogleSQL + pattern = cache.getUnchecked(new Key(0, regex)); + } catch (UncheckedExecutionException e) { + if (e.getCause() instanceof PatternSyntaxException) { + throw RESOURCE.invalidInputForRegexpContains( + stripLineEndings( + requireNonNull(e.getCause().getMessage(), "message"))).ex(); + } + throw e; + } + return pattern.matcher(value).find(); + } - /** SQL {@code REGEXP_REPLACE} function with 5 arguments. */ - public static String regexpReplace(String s, String regex, String replacement, - int pos, int occurrence) { - return regexpReplace(s, regex, replacement, pos, occurrence, null); - } + private static String stripLineEndings(String message) { + return message.replace("\r\n", " ") + .replace("\n", " ") + .replace("\r", " "); + } - /** SQL {@code REGEXP_REPLACE} function with 6 arguments. */ - public static String regexpReplace(String s, String regex, String replacement, - int pos, int occurrence, @Nullable String matchType) { - if (pos < 1 || pos > s.length()) { - throw RESOURCE.invalidInputForRegexpReplace(Integer.toString(pos)).ex(); + /** SQL {@code REGEXP_REPLACE} function with 3 arguments. */ + public String regexpReplace(String s, String regex, + String replacement) { + return regexpReplace(s, regex, replacement, 1, 0, null); } - final int flags = makeRegexpFlags(matchType); - final Pattern pattern = Pattern.compile(regex, flags); + /** SQL {@code REGEXP_REPLACE} function with 4 arguments. */ + public String regexpReplace(String s, String regex, String replacement, + int pos) { + return regexpReplace(s, regex, replacement, pos, 0, null); + } - return Unsafe.regexpReplace(s, pattern, replacement, pos, occurrence); - } + /** SQL {@code REGEXP_REPLACE} function with 5 arguments. */ + public String regexpReplace(String s, String regex, String replacement, + int pos, int occurrence) { + return regexpReplace(s, regex, replacement, pos, occurrence, null); + } + + /** SQL {@code REGEXP_REPLACE} function with 6 arguments. */ + public String regexpReplace(String s, String regex, String replacement, + int pos, int occurrence, @Nullable String matchType) { + if (pos < 1 || pos > s.length()) { + throw RESOURCE.invalidInputForRegexpReplace(Integer.toString(pos)).ex(); + } - private static int makeRegexpFlags(@Nullable String stringFlags) { - int flags = 0; - if (stringFlags != null) { + final int flags = matchType == null ? 0 : makeRegexpFlags(matchType); + final Pattern pattern = cache.getUnchecked(new Key(flags, regex)); + + return Unsafe.regexpReplace(s, pattern, replacement, pos, occurrence); + } + + private static int makeRegexpFlags(String stringFlags) { + int flags = 0; for (int i = 0; i < stringFlags.length(); ++i) { switch (stringFlags.charAt(i)) { case 'i': @@ -420,8 +464,13 @@ public class SqlFunctions { throw RESOURCE.invalidInputForRegexpReplace(stringFlags).ex(); } } + return flags; + } + + /** SQL {@code RLIKE} function. */ + public boolean rlike(String s, String pattern) { + return cache.getUnchecked(new Key(0, pattern)).matcher(s).find(); } - return flags; } /** SQL {@code LPAD(string, integer, string)} function. */ @@ -1069,78 +1118,76 @@ public class SqlFunctions { } } - /** SQL {@code PARSE_URL(urlStr, partToExtract, keyToExtract)} function. */ - public static @Nullable String parseUrl(@Nullable String urlStr, - @Nullable String partToExtract, @Nullable String keyToExtract) { - if (partToExtract == null || !partToExtract.equals("QUERY")) { - return null; + /** State for {@code PARSE_URL}. */ + @Deterministic + public static class ParseUrlFunction { + static Pattern keyToPattern(String keyToExtract) { + return Pattern.compile("(&|^)" + keyToExtract + "=([^&]*)"); } - String query = parseUrl(urlStr, partToExtract); - if (query == null) { - return null; - } + private final LoadingCache<String, Pattern> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build(CacheLoader.from(ParseUrlFunction::keyToPattern)); - Pattern p = Pattern.compile("(&|^)" + keyToExtract + "=([^&]*)"); - Matcher m = p.matcher(query); - return m.find() ? m.group(2) : null; - } + /** SQL {@code PARSE_URL(urlStr, partToExtract, keyToExtract)} function. */ + public @Nullable String parseUrl(String urlStr, String partToExtract, + String keyToExtract) { + if (!partToExtract.equals("QUERY")) { + return null; + } - /** SQL {@code PARSE_URL(urlStr, partToExtract)} function. */ - public static @Nullable String parseUrl(@Nullable String urlStr, - @Nullable String partToExtract) { - if (urlStr == null || partToExtract == null) { - return null; - } + String query = parseUrl(urlStr, partToExtract); + if (query == null) { + return null; + } - URI uri; - try { - uri = new URI(urlStr); - } catch (URISyntaxException e) { - return null; + Pattern p = cache.getUnchecked(keyToExtract); + Matcher m = p.matcher(query); + return m.find() ? m.group(2) : null; } - String extractValue; - PartToExtract part; - try { - part = PartToExtract.valueOf(partToExtract); - } catch (IllegalArgumentException e) { - return null; - } + /** SQL {@code PARSE_URL(urlStr, partToExtract)} function. */ + public @Nullable String parseUrl(String urlStr, String partToExtract) { + URI uri; + try { + uri = new URI(urlStr); + } catch (URISyntaxException e) { + return null; + } - switch (part) { - case HOST: - extractValue = uri.getHost(); - break; - case PATH: - extractValue = uri.getRawPath(); - break; - case QUERY: - extractValue = uri.getRawQuery(); - break; - case REF: - extractValue = uri.getRawFragment(); - break; - case PROTOCOL: - extractValue = uri.getScheme(); - break; - case FILE: - if (uri.getRawQuery() != null) { - extractValue = uri.getRawPath() + "?" + uri.getRawQuery(); - } else { - extractValue = uri.getRawPath(); + PartToExtract part; + try { + part = PartToExtract.valueOf(partToExtract); + } catch (IllegalArgumentException e) { + return null; + } + + switch (part) { + case HOST: + return uri.getHost(); + case PATH: + return uri.getRawPath(); + case QUERY: + return uri.getRawQuery(); + case REF: + return uri.getRawFragment(); + case PROTOCOL: + return uri.getScheme(); + case FILE: + if (uri.getRawQuery() != null) { + return uri.getRawPath() + "?" + uri.getRawQuery(); + } else { + return uri.getRawPath(); + } + case AUTHORITY: + return uri.getRawAuthority(); + case USERINFO: + return uri.getRawUserInfo(); + default: + return null; } - break; - case AUTHORITY: - extractValue = uri.getRawAuthority(); - break; - case USERINFO: - extractValue = uri.getRawUserInfo(); - break; - default: - extractValue = null; } - return extractValue; } /** SQL {@code RTRIM} function applied to string. */ @@ -1259,50 +1306,136 @@ public class SqlFunctions { .concat(s.substring(start - 1 + length)); } - /** SQL {@code LIKE} function. */ - public static boolean like(String s, String pattern) { - final String regex = Like.sqlToRegexLike(pattern, null); - return Pattern.matches(regex, s); - } + /** State for {@code LIKE}, {@code ILIKE}. */ + @Deterministic + public static class LikeFunction { + /** Key for cache of compiled regular expressions. */ + private static final class Key { + final String pattern; + final @Nullable String escape; + final int flags; + + Key(String pattern, @Nullable String escape, int flags) { + this.pattern = pattern; + this.escape = escape; + this.flags = flags; + } - /** SQL {@code LIKE} function with escape. */ - public static boolean like(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexLike(pattern, escape); - return Pattern.matches(regex, s); - } + @Override public int hashCode() { + return pattern.hashCode() + ^ (escape == null ? 0 : escape.hashCode()) + ^ flags; + } - /** SQL {@code ILIKE} function. */ - public static boolean ilike(String s, String pattern) { - final String regex = Like.sqlToRegexLike(pattern, null); - return Pattern.compile(regex, Pattern.CASE_INSENSITIVE).matcher(s).matches(); - } + @Override public boolean equals(@Nullable Object obj) { + return this == obj + || obj instanceof Key + && pattern.equals(((Key) obj).pattern) + && Objects.equals(escape, ((Key) obj).escape) + && flags == ((Key) obj).flags; + } - /** SQL {@code ILIKE} function with escape. */ - public static boolean ilike(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexLike(pattern, escape); - return Pattern.compile(regex, Pattern.CASE_INSENSITIVE).matcher(s).matches(); - } + Pattern toPattern() { + String regex = Like.sqlToRegexLike(pattern, escape); + return Pattern.compile(regex, flags); + } + } - /** SQL {@code RLIKE} function. */ - public static boolean rlike(String s, String pattern) { - return Pattern.compile(pattern).matcher(s).find(); - } + private final LoadingCache<Key, Pattern> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build(CacheLoader.from(Key::toPattern)); + + /** SQL {@code LIKE} function. */ + public boolean like(String s, String pattern) { + final Key key = new Key(pattern, null, 0); + return cache.getUnchecked(key).matcher(s).matches(); + } + + /** SQL {@code LIKE} function with escape. */ + public boolean like(String s, String pattern, String escape) { + final Key key = new Key(pattern, escape, 0); + return cache.getUnchecked(key).matcher(s).matches(); + } + + /** SQL {@code ILIKE} function. */ + public boolean ilike(String s, String pattern) { + final Key key = new Key(pattern, null, Pattern.CASE_INSENSITIVE); + return cache.getUnchecked(key).matcher(s).matches(); + } - /** SQL {@code SIMILAR} function. */ - public static boolean similar(String s, String pattern) { - final String regex = Like.sqlToRegexSimilar(pattern, null); - return Pattern.matches(regex, s); + /** SQL {@code ILIKE} function with escape. */ + public boolean ilike(String s, String pattern, String escape) { + final Key key = new Key(pattern, escape, Pattern.CASE_INSENSITIVE); + return cache.getUnchecked(key).matcher(s).matches(); + } } - /** SQL {@code SIMILAR} function with escape. */ - public static boolean similar(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexSimilar(pattern, escape); - return Pattern.matches(regex, s); + /** State for {@code SIMILAR} function. */ + @Deterministic + public static class SimilarFunction { + private final LoadingCache<String, Pattern> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build( + CacheLoader.from(pattern -> + Pattern.compile(Like.sqlToRegexSimilar(pattern, null)))); + + /** SQL {@code SIMILAR} function. */ + public boolean similar(String s, String pattern) { + return cache.getUnchecked(pattern).matcher(s).matches(); + } + } + + /** State for {@code SIMILAR} function with escape. */ + public static class SimilarEscapeFunction { + /** Cache key. */ + private static class Key extends MapEntry<String, String> { + Key(String formatModel, String format) { + super(formatModel, format); + } + + Pattern toPattern() { + return Pattern.compile(Like.sqlToRegexSimilar(t, u)); + } + } + + private final LoadingCache<Key, Pattern> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build(CacheLoader.from(Key::toPattern)); + + /** SQL {@code SIMILAR} function with escape. */ + public boolean similar(String s, String pattern, String escape) { + return cache.getUnchecked(new Key(pattern, escape)) + .matcher(s).matches(); + } } - public static boolean posixRegex(String s, String regex, boolean caseSensitive) { - final Pattern pattern = Like.posixRegexToPattern(regex, caseSensitive); - return pattern.matcher(s).find(); + /** State for posix regex function. */ + @Deterministic + public static class PosixRegexFunction { + private final LoadingCache<Ord<String>, Pattern> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build( + CacheLoader.from(pattern -> + Like.posixRegexToPattern(pattern.e, pattern.i))); + + boolean posixRegex(String s, String regex, int flags) { + final Ord<String> key = Ord.of(flags, regex); + return cache.getUnchecked(key).matcher(s).find(); + } + + /** Posix regex, case-insensitive. */ + public boolean posixRegexInsensitive(String s, String regex) { + return posixRegex(s, regex, Pattern.CASE_INSENSITIVE); + } + + /** Posix regex, case-sensitive. */ + public boolean posixRegexSensitive(String s, String regex) { + return posixRegex(s, regex, 0); + } } // = @@ -3339,97 +3472,176 @@ public class SqlFunctions { .toString(); } - private static String internalFormatDatetime(String fmtString, java.util.Date date) { - StringBuilder sb = new StringBuilder(); - List<FormatElement> elements = FormatModels.BIG_QUERY.parse(fmtString); - elements.forEach(ele -> ele.format(sb, date)); - return sb.toString(); - } + /** State for {@code FORMAT_DATE}, {@code FORMAT_TIMESTAMP}, + * {@code FORMAT_DATETIME}, {@code FORMAT_TIME}, {@code TO_CHAR} functions. */ + @Deterministic + public static class DateFormatFunction { + /** Work space for various functions. Clear it before you use it. */ + final StringBuilder sb = new StringBuilder(); + + /** Cache key. */ + private static class Key extends MapEntry<FormatModel, String> { + Key(FormatModel formatModel, String format) { + super(formatModel, format); + } + } - public static String formatTimestamp(DataContext ctx, String fmtString, long timestamp) { - return internalFormatDatetime(fmtString, internalToTimestamp(timestamp)); - } + private final LoadingCache<Key, List<FormatElement>> formatCache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build(CacheLoader.from(key -> key.t.parseNoCache(key.u))); - public static String toChar(long timestamp, String pattern) { - List<FormatElement> elements = FormatModels.POSTGRESQL.parse(pattern); - StringBuilder sb = new StringBuilder(); - elements.forEach(ele -> ele.format(sb, internalToTimestamp(timestamp))); - return sb.toString().trim(); - } + /** Given a format string and a format model, calls an action with the + * list of elements obtained by parsing that format string. */ + protected final void withElements(FormatModel formatModel, String format, + Consumer<List<FormatElement>> consumer) { + List<FormatElement> elements = + formatCache.getUnchecked(new Key(formatModel, format)); + consumer.accept(elements); + } - public static String formatDate(DataContext ctx, String fmtString, int date) { - return internalFormatDatetime(fmtString, internalToDate(date)); - } + private String internalFormatDatetime(String fmtString, + java.util.Date date) { + sb.setLength(0); + withElements(FormatModels.BIG_QUERY, fmtString, elements -> + elements.forEach(element -> element.format(sb, date))); + return sb.toString(); + } - public static String formatTime(DataContext ctx, String fmtString, int time) { - return internalFormatDatetime(fmtString, internalToTime(time)); - } + public String formatTimestamp(DataContext ctx, String fmtString, + long timestamp) { + return internalFormatDatetime(fmtString, internalToTimestamp(timestamp)); + } - private static String parseDatetimePattern(String fmtString) { - StringBuilder sb = new StringBuilder(); - List<FormatElement> elements = FormatModels.BIG_QUERY.parse(fmtString); - elements.forEach(ele -> ele.toPattern(sb)); - return sb.toString(); - } + public String toChar(long timestamp, String pattern) { + final Timestamp sqlTimestamp = internalToTimestamp(timestamp); + sb.setLength(0); + withElements(FormatModels.POSTGRESQL, pattern, elements -> + elements.forEach(element -> element.format(sb, sqlTimestamp))); + return sb.toString().trim(); + } - private static long internalParseDatetime(String fmtString, String datetime) { - return internalParseDatetime(fmtString, datetime, - DateTimeUtils.DEFAULT_ZONE); - } - - private static long internalParseDatetime(String fmt, String datetime, - TimeZone tz) { - final String javaFmt = parseDatetimePattern(fmt); - // TODO: make Locale configurable. ENGLISH set for weekday parsing (e.g. - // Thursday, Friday). - final DateFormat parser = new SimpleDateFormat(javaFmt, Locale.ENGLISH); - final ParsePosition pos = new ParsePosition(0); - parser.setLenient(false); - parser.setCalendar(Calendar.getInstance(tz, Locale.ROOT)); - Date parsed = parser.parse(datetime, pos); - // Throw if either the parse was unsuccessful, or the format string did not - // contain enough elements to parse the datetime string completely. - if (pos.getErrorIndex() >= 0 || pos.getIndex() != datetime.length()) { - SQLException e = - new SQLException( - String.format(Locale.ROOT, - "Invalid format: '%s' for datetime string: '%s'.", fmt, - datetime)); - throw Util.toUnchecked(e); + public String formatDate(DataContext ctx, String fmtString, int date) { + return internalFormatDatetime(fmtString, internalToDate(date)); } - // Suppress the Errorprone warning "[JavaUtilDate] Date has a bad API that - // leads to bugs; prefer java.time.Instant or LocalDate" because we know - // what we're doing. - @SuppressWarnings("JavaUtilDate") - final long millisSinceEpoch = parsed.getTime(); - return millisSinceEpoch; - } - public static int parseDate(String fmtString, String date) { - final long millisSinceEpoch = internalParseDatetime(fmtString, date); - return toInt(new java.sql.Date(millisSinceEpoch)); + public String formatTime(DataContext ctx, String fmtString, int time) { + return internalFormatDatetime(fmtString, internalToTime(time)); + } } - public static long parseDatetime(String fmtString, String datetime) { - final long millisSinceEpoch = internalParseDatetime(fmtString, datetime); - return toLong(new java.sql.Timestamp(millisSinceEpoch)); - } + /** State for {@code PARSE_DATE}, {@code PARSE_TIMESTAMP}, + * {@code PARSE_DATETIME}, {@code PARSE_TIME} functions. */ + @Deterministic + public static class DateParseFunction { + /** Use a {@link DateFormatFunction} for its cache of parsed + * format strings. */ + final DateFormatFunction f = new DateFormatFunction(); + + private final LoadingCache<Key, DateFormat> cache = + CacheBuilder.newBuilder() + .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) + .build(CacheLoader.from(key -> key.toDateFormat(f))); + + private <T> T withParser(String fmt, String timeZone, + Function<DateFormat, T> action) { + final DateFormat dateFormat = cache.getUnchecked(new Key(fmt, timeZone)); + return action.apply(dateFormat); + } + + private long internalParseDatetime(String fmtString, String datetime) { + return internalParseDatetime(fmtString, datetime, + DateTimeUtils.DEFAULT_ZONE.getID()); + } + + private long internalParseDatetime(String fmt, String datetime, + String timeZone) { + final ParsePosition pos = new ParsePosition(0); + Date parsed = + withParser(fmt, timeZone, parser -> parser.parse(datetime, pos)); + // Throw if either the parse was unsuccessful, or the format string did + // not contain enough elements to parse the datetime string completely. + if (pos.getErrorIndex() >= 0 || pos.getIndex() != datetime.length()) { + SQLException e = + new SQLException( + String.format(Locale.ROOT, + "Invalid format: '%s' for datetime string: '%s'.", fmt, + datetime)); + throw Util.toUnchecked(e); + } + // Suppress the Errorprone warning "[JavaUtilDate] Date has a bad API that + // leads to bugs; prefer java.time.Instant or LocalDate" because we know + // what we're doing. + @SuppressWarnings("JavaUtilDate") + final long millisSinceEpoch = parsed.getTime(); + return millisSinceEpoch; + } - public static int parseTime(String fmtString, String time) { - final long millisSinceEpoch = internalParseDatetime(fmtString, time); - return toInt(new java.sql.Time(millisSinceEpoch)); - } + public int parseDate(String fmtString, String date) { + final long millisSinceEpoch = internalParseDatetime(fmtString, date); + return toInt(new java.sql.Date(millisSinceEpoch)); + } - public static long parseTimestamp(String fmtString, String timestamp) { - return parseTimestamp(fmtString, timestamp, "UTC"); - } + public long parseDatetime(String fmtString, String datetime) { + final long millisSinceEpoch = internalParseDatetime(fmtString, datetime); + return toLong(new Timestamp(millisSinceEpoch)); + } - public static long parseTimestamp(String fmtString, String timestamp, - String timeZone) { - TimeZone tz = TimeZone.getTimeZone(timeZone); - final long millisSinceEpoch = - internalParseDatetime(fmtString, timestamp, tz); - return toLong(new java.sql.Timestamp(millisSinceEpoch), tz); + public int parseTime(String fmtString, String time) { + final long millisSinceEpoch = internalParseDatetime(fmtString, time); + return toInt(new Time(millisSinceEpoch)); + } + + public long parseTimestamp(String fmtString, String timestamp) { + return parseTimestamp(fmtString, timestamp, "UTC"); + } + + public long parseTimestamp(String fmtString, String timestamp, + String timeZone) { + TimeZone tz = TimeZone.getTimeZone(timeZone); + final long millisSinceEpoch = + internalParseDatetime(fmtString, timestamp, timeZone); + return toLong(new java.sql.Timestamp(millisSinceEpoch), tz); + } + + /** Key for cache of parsed format strings. */ + private static final class Key { + final String fmt; + final String timeZone; + + Key(String fmt, String timeZone) { + this.fmt = fmt; + this.timeZone = timeZone; + } + + @Override public int hashCode() { + return fmt.hashCode() + + timeZone.hashCode() * 37; + } + + @Override public boolean equals(@Nullable Object obj) { + return this == obj + || obj instanceof Key + && fmt.equals(((Key) obj).fmt) + && timeZone.equals(((Key) obj).timeZone); + } + + DateFormat toDateFormat(DateFormatFunction f) { + f.sb.setLength(0); + f.withElements(FormatModels.BIG_QUERY, fmt, elements -> + elements.forEach(ele -> ele.toPattern(f.sb))); + final String javaFmt = f.sb.toString(); + + // TODO: make Locale configurable. ENGLISH set for weekday + // parsing (e.g. Thursday, Friday). + final DateFormat parser = + new SimpleDateFormat(javaFmt, Locale.ENGLISH); + parser.setLenient(false); + TimeZone tz = TimeZone.getTimeZone(timeZone); + parser.setCalendar(Calendar.getInstance(tz, Locale.ROOT)); + return parser; + } + } } /** @@ -5141,8 +5353,8 @@ public class SqlFunctions { SCALAR, LIST, MAP } - /** Type of part to extract passed into {@link #parseUrl}. */ - public enum PartToExtract { + /** Type of part to extract passed into {@link ParseUrlFunction#parseUrl}. */ + private enum PartToExtract { HOST, PATH, QUERY, @@ -5152,5 +5364,4 @@ public class SqlFunctions { AUTHORITY, USERINFO; } - } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index a4b5e56881..fe50d2c8cc 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -361,8 +361,8 @@ public enum BuiltInMethod { EXP(SqlFunctions.class, "exp", double.class), MOD(SqlFunctions.class, "mod", long.class, long.class), POWER(SqlFunctions.class, "power", double.class, double.class), - REGEXP_CONTAINS(SqlFunctions.class, "regexpContains", String.class, - String.class), + REGEXP_CONTAINS(SqlFunctions.RegexFunction.class, "regexpContains", + String.class, String.class), REPEAT(SqlFunctions.class, "repeat", String.class, int.class), SPACE(SqlFunctions.class, "space", int.class), SPLIT(SqlFunctions.class, "split", String.class), @@ -442,7 +442,10 @@ public enum BuiltInMethod { STRING_CONCAT(SqlFunctions.class, "concat", String.class, String.class), STRING_CONCAT_WITH_NULL(SqlFunctions.class, "concatWithNull", String.class, String.class), - PARSE_URL(SqlFunctions.class, "parseUrl", String.class, String.class, String.class), + PARSE_URL2(SqlFunctions.ParseUrlFunction.class, "parseUrl", String.class, + String.class), + PARSE_URL3(SqlFunctions.ParseUrlFunction.class, "parseUrl", String.class, + String.class, String.class), MULTI_STRING_CONCAT(SqlFunctions.class, "concatMulti", String[].class), MULTI_STRING_CONCAT_WITH_NULL(SqlFunctions.class, "concatMultiWithNull", String[].class), @@ -505,19 +508,30 @@ public enum BuiltInMethod { TRANSLATE3(SqlFunctions.class, "translate3", String.class, String.class, String.class), LTRIM(SqlFunctions.class, "ltrim", String.class), RTRIM(SqlFunctions.class, "rtrim", String.class), - LIKE(SqlFunctions.class, "like", String.class, String.class), - ILIKE(SqlFunctions.class, "ilike", String.class, String.class), - RLIKE(SqlFunctions.class, "rlike", String.class, String.class), - SIMILAR(SqlFunctions.class, "similar", String.class, String.class), - POSIX_REGEX(SqlFunctions.class, "posixRegex", String.class, String.class, boolean.class), - REGEXP_REPLACE3(SqlFunctions.class, "regexpReplace", String.class, + LIKE(SqlFunctions.LikeFunction.class, "like", String.class, String.class), + LIKE_ESCAPE(SqlFunctions.LikeFunction.class, "like", String.class, String.class, String.class), - REGEXP_REPLACE4(SqlFunctions.class, "regexpReplace", String.class, - String.class, String.class, int.class), - REGEXP_REPLACE5(SqlFunctions.class, "regexpReplace", String.class, - String.class, String.class, int.class, int.class), - REGEXP_REPLACE6(SqlFunctions.class, "regexpReplace", String.class, - String.class, String.class, int.class, int.class, String.class), + ILIKE(SqlFunctions.LikeFunction.class, "ilike", String.class, String.class), + ILIKE_ESCAPE(SqlFunctions.LikeFunction.class, "ilike", String.class, + String.class, String.class), + RLIKE(SqlFunctions.RegexFunction.class, "rlike", String.class, String.class), + SIMILAR(SqlFunctions.SimilarFunction.class, "similar", String.class, + String.class), + SIMILAR_ESCAPE(SqlFunctions.SimilarEscapeFunction.class, "similar", + String.class, String.class, String.class), + POSIX_REGEX_INSENSITIVE(SqlFunctions.PosixRegexFunction.class, + "posixRegexInsensitive", String.class, String.class), + POSIX_REGEX_SENSITIVE(SqlFunctions.PosixRegexFunction.class, + "posixRegexSensitive", String.class, String.class), + REGEXP_REPLACE3(SqlFunctions.RegexFunction.class, "regexpReplace", + String.class, String.class, String.class), + REGEXP_REPLACE4(SqlFunctions.RegexFunction.class, "regexpReplace", + String.class, String.class, String.class, int.class), + REGEXP_REPLACE5(SqlFunctions.RegexFunction.class, "regexpReplace", + String.class, String.class, String.class, int.class, int.class), + REGEXP_REPLACE6(SqlFunctions.RegexFunction.class, "regexpReplace", + String.class, String.class, String.class, int.class, int.class, + String.class), IS_TRUE(SqlFunctions.class, "isTrue", Boolean.class), IS_NOT_FALSE(SqlFunctions.class, "isNotFalse", Boolean.class), NOT(SqlFunctions.class, "not", Boolean.class), @@ -580,19 +594,22 @@ public enum BuiltInMethod { DATETIME(SqlFunctions.class, "datetime", long.class), TIME(SqlFunctions.class, "time", int.class), TIMESTAMP(SqlFunctions.class, "timestamp", long.class), - PARSE_DATE(SqlFunctions.class, "parseDate", String.class, String.class), - PARSE_DATETIME(SqlFunctions.class, "parseDatetime", String.class, + PARSE_DATE(SqlFunctions.DateParseFunction.class, "parseDate", String.class, String.class), - PARSE_TIME(SqlFunctions.class, "parseTime", String.class, String.class), - PARSE_TIMESTAMP(SqlFunctions.class, "parseTimestamp", String.class, + PARSE_DATETIME(SqlFunctions.DateParseFunction.class, "parseDatetime", + String.class, String.class), + PARSE_TIME(SqlFunctions.DateParseFunction.class, "parseTime", String.class, String.class), - FORMAT_TIMESTAMP(SqlFunctions.class, "formatTimestamp", DataContext.class, - String.class, long.class), - TO_CHAR(SqlFunctions.class, "toChar", long.class, String.class), - FORMAT_DATE(SqlFunctions.class, "formatDate", DataContext.class, String.class, - int.class), - FORMAT_TIME(SqlFunctions.class, "formatTime", DataContext.class, String.class, - int.class), + PARSE_TIMESTAMP(SqlFunctions.DateParseFunction.class, "parseTimestamp", + String.class, String.class), + FORMAT_TIMESTAMP(SqlFunctions.DateFormatFunction.class, "formatTimestamp", + DataContext.class, String.class, long.class), + TO_CHAR(SqlFunctions.DateFormatFunction.class, "toChar", long.class, + String.class), + FORMAT_DATE(SqlFunctions.DateFormatFunction.class, "formatDate", + DataContext.class, String.class, int.class), + FORMAT_TIME(SqlFunctions.DateFormatFunction.class, "formatTime", + DataContext.class, String.class, int.class), UNIX_DATE_TO_STRING(DateTimeUtils.class, "unixDateToString", int.class), UNIX_TIME_TO_STRING(DateTimeUtils.class, "unixTimeToString", int.class), UNIX_TIMESTAMP_TO_STRING(DateTimeUtils.class, "unixTimestampToString", diff --git a/core/src/main/java/org/apache/calcite/util/format/FormatModel.java b/core/src/main/java/org/apache/calcite/util/format/FormatModel.java index 39bac2f053..248ae5accb 100644 --- a/core/src/main/java/org/apache/calcite/util/format/FormatModel.java +++ b/core/src/main/java/org/apache/calcite/util/format/FormatModel.java @@ -34,4 +34,7 @@ public interface FormatModel { /** Parses a format string using element identifiers supplied by * {@code format}. */ List<FormatElement> parse(String format); + + /** As {@link #parse}, but does not cache. */ + List<FormatElement> parseNoCache(String format); } diff --git a/core/src/main/java/org/apache/calcite/util/format/FormatModels.java b/core/src/main/java/org/apache/calcite/util/format/FormatModels.java index 545e315b58..2839e661b7 100644 --- a/core/src/main/java/org/apache/calcite/util/format/FormatModels.java +++ b/core/src/main/java/org/apache/calcite/util/format/FormatModels.java @@ -237,7 +237,7 @@ public class FormatModels { return elementMap; } - private List<FormatElement> internalParse(String format) { + @Override public List<FormatElement> parseNoCache(String format) { final ImmutableList.Builder<FormatElement> elements = ImmutableList.builder(); final Matcher matcher = pattern.matcher(format); @@ -263,7 +263,7 @@ public class FormatModels { } @Override public List<FormatElement> parse(String format) { - return memoizedElements.computeIfAbsent(format, this::internalParse); + return memoizedElements.computeIfAbsent(format, this::parseNoCache); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index e6eb29b938..e308297898 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -58,15 +58,11 @@ import static org.apache.calcite.runtime.SqlFunctions.lower; import static org.apache.calcite.runtime.SqlFunctions.ltrim; import static org.apache.calcite.runtime.SqlFunctions.md5; import static org.apache.calcite.runtime.SqlFunctions.position; -import static org.apache.calcite.runtime.SqlFunctions.posixRegex; -import static org.apache.calcite.runtime.SqlFunctions.regexpContains; -import static org.apache.calcite.runtime.SqlFunctions.regexpReplace; import static org.apache.calcite.runtime.SqlFunctions.rtrim; import static org.apache.calcite.runtime.SqlFunctions.sha1; import static org.apache.calcite.runtime.SqlFunctions.sha256; import static org.apache.calcite.runtime.SqlFunctions.sha512; import static org.apache.calcite.runtime.SqlFunctions.toBase64; -import static org.apache.calcite.runtime.SqlFunctions.toChar; import static org.apache.calcite.runtime.SqlFunctions.toInt; import static org.apache.calcite.runtime.SqlFunctions.toIntOptional; import static org.apache.calcite.runtime.SqlFunctions.toLong; @@ -216,76 +212,86 @@ class SqlFunctionsTest { } @Test void testPosixRegex() { - assertThat(posixRegex("abc", "abc", true), is(true)); - assertThat(posixRegex("abc", "^a", true), is(true)); - assertThat(posixRegex("abc", "(b|d)", true), is(true)); - assertThat(posixRegex("abc", "^(b|c)", true), is(false)); + final SqlFunctions.PosixRegexFunction f = + new SqlFunctions.PosixRegexFunction(); + assertThat(f.posixRegexSensitive("abc", "abc"), is(true)); + assertThat(f.posixRegexSensitive("abc", "^a"), is(true)); + assertThat(f.posixRegexSensitive("abc", "(b|d)"), is(true)); + assertThat(f.posixRegexSensitive("abc", "^(b|c)"), is(false)); - assertThat(posixRegex("abc", "ABC", false), is(true)); - assertThat(posixRegex("abc", "^A", false), is(true)); - assertThat(posixRegex("abc", "(B|D)", false), is(true)); - assertThat(posixRegex("abc", "^(B|C)", false), is(false)); + assertThat(f.posixRegexInsensitive("abc", "ABC"), is(true)); + assertThat(f.posixRegexInsensitive("abc", "^A"), is(true)); + assertThat(f.posixRegexInsensitive("abc", "(B|D)"), is(true)); + assertThat(f.posixRegexInsensitive("abc", "^(B|C)"), is(false)); - assertThat(posixRegex("abc", "^[[:xdigit:]]$", false), is(false)); - assertThat(posixRegex("abc", "^[[:xdigit:]]+$", false), is(true)); - assertThat(posixRegex("abcq", "^[[:xdigit:]]+$", false), is(false)); + assertThat(f.posixRegexInsensitive("abc", "^[[:xdigit:]]$"), is(false)); + assertThat(f.posixRegexInsensitive("abc", "^[[:xdigit:]]+$"), is(true)); + assertThat(f.posixRegexInsensitive("abcq", "^[[:xdigit:]]+$"), is(false)); - assertThat(posixRegex("abc", "[[:xdigit:]]", false), is(true)); - assertThat(posixRegex("abc", "[[:xdigit:]]+", false), is(true)); - assertThat(posixRegex("abcq", "[[:xdigit:]]", false), is(true)); + assertThat(f.posixRegexInsensitive("abc", "[[:xdigit:]]"), is(true)); + assertThat(f.posixRegexInsensitive("abc", "[[:xdigit:]]+"), is(true)); + assertThat(f.posixRegexInsensitive("abcq", "[[:xdigit:]]"), is(true)); } @Test void testRegexpContains() { + final SqlFunctions.RegexFunction f = new SqlFunctions.RegexFunction(); + + // Use same regex; should hit cache + assertThat(f.regexpContains("abcdef", "abz*"), is(true)); + assertThat(f.regexpContains("zabzz", "abz*"), is(true)); + assertThat(f.regexpContains("zazbbzz", "abz*"), is(false)); + try { - regexpContains("abc def ghi", "(abc"); - fail("'regexp_contains' on an invalid regex input '(abc' is not possible"); + final boolean b = f.regexpContains("abc def ghi", "(abc"); + fail("expected error, got " + b); } catch (RuntimeException e) { - assertThat( - e.getMessage(), is("Invalid regular expression for REGEXP_CONTAINS: 'Unclosed " + assertThat(e.getMessage(), + is("Invalid regular expression for REGEXP_CONTAINS: 'Unclosed " + "group near index 4 (abc'")); } try { - regexpContains("abc def ghi", "[z-a]"); - fail("'regexp_contains' on an invalid regex input '[z-a]' is not possible"); + final boolean b = f.regexpContains("abc def ghi", "[z-a]"); + fail("expected error, got " + b); } catch (RuntimeException e) { - assertThat( - e.getMessage(), is("Invalid regular expression for REGEXP_CONTAINS: 'Illegal " + assertThat(e.getMessage(), + is("Invalid regular expression for REGEXP_CONTAINS: 'Illegal " + "character range near index" + " 3 [z-a] ^'")); } try { - regexpContains("abc def ghi", "{2,1}"); - fail("'regexp_contains' on an invalid regex input '{2,1}' is not possible"); + final boolean b = f.regexpContains("abc def ghi", "{2,1}"); + fail("expected error, got " + b); } catch (RuntimeException e) { - assertThat( - e.getMessage(), is("Invalid regular expression for REGEXP_CONTAINS: 'Illegal " + assertThat(e.getMessage(), + is("Invalid regular expression for REGEXP_CONTAINS: 'Illegal " + "repetition range near " + "index 4 {2,1} ^'")); } } @Test void testRegexpReplace() { - assertThat(regexpReplace("a b c", "b", "X"), is("a X c")); - assertThat(regexpReplace("abc def ghi", "[g-z]+", "X"), is("abc def X")); - assertThat(regexpReplace("abc def ghi", "[a-z]+", "X"), is("X X X")); - assertThat(regexpReplace("a b c", "a|b", "X"), is("X X c")); - assertThat(regexpReplace("a b c", "y", "X"), is("a b c")); - - assertThat(regexpReplace("100-200", "(\\d+)", "num"), is("num-num")); - assertThat(regexpReplace("100-200", "(\\d+)", "###"), is("###-###")); - assertThat(regexpReplace("100-200", "(-)", "###"), is("100###200")); - - assertThat(regexpReplace("abc def ghi", "[a-z]+", "X", 1), is("X X X")); - assertThat(regexpReplace("abc def ghi", "[a-z]+", "X", 2), is("aX X X")); - assertThat(regexpReplace("abc def ghi", "[a-z]+", "X", 1, 3), + final SqlFunctions.RegexFunction f = new SqlFunctions.RegexFunction(); + assertThat(f.regexpReplace("a b c", "b", "X"), is("a X c")); + assertThat(f.regexpReplace("abc def ghi", "[g-z]+", "X"), is("abc def X")); + assertThat(f.regexpReplace("abc def ghi", "[a-z]+", "X"), is("X X X")); + assertThat(f.regexpReplace("a b c", "a|b", "X"), is("X X c")); + assertThat(f.regexpReplace("a b c", "y", "X"), is("a b c")); + + assertThat(f.regexpReplace("100-200", "(\\d+)", "num"), is("num-num")); + assertThat(f.regexpReplace("100-200", "(\\d+)", "###"), is("###-###")); + assertThat(f.regexpReplace("100-200", "(-)", "###"), is("100###200")); + + assertThat(f.regexpReplace("abc def ghi", "[a-z]+", "X", 1), is("X X X")); + assertThat(f.regexpReplace("abc def ghi", "[a-z]+", "X", 2), is("aX X X")); + assertThat(f.regexpReplace("abc def ghi", "[a-z]+", "X", 1, 3), is("abc def X")); - assertThat(regexpReplace("abc def GHI", "[a-z]+", "X", 1, 3, "c"), + assertThat(f.regexpReplace("abc def GHI", "[a-z]+", "X", 1, 3, "c"), is("abc def GHI")); - assertThat(regexpReplace("abc def GHI", "[a-z]+", "X", 1, 3, "i"), + assertThat(f.regexpReplace("abc def GHI", "[a-z]+", "X", 1, 3, "i"), is("abc def X")); try { - regexpReplace("abc def ghi", "[a-z]+", "X", 0); + f.regexpReplace("abc def ghi", "[a-z]+", "X", 0); fail("'regexp_replace' on an invalid pos is not possible"); } catch (CalciteException e) { assertThat(e.getMessage(), @@ -293,7 +299,7 @@ class SqlFunctionsTest { } try { - regexpReplace("abc def ghi", "[a-z]+", "X", 1, 3, "WWW"); + f.regexpReplace("abc def ghi", "[a-z]+", "X", 1, 3, "WWW"); fail("'regexp_replace' on an invalid matchType is not possible"); } catch (CalciteException e) { assertThat(e.getMessage(), @@ -1492,24 +1498,23 @@ class SqlFunctionsTest { String pattern1 = "YYYY-MM-DD HH24:MI:SS.MS"; String pattern2 = "Day, DD HH12:MI:SS"; - assertThat( - toChar(0, pattern1), + final SqlFunctions.DateFormatFunction f = + new SqlFunctions.DateFormatFunction(); + assertThat(f.toChar(0, pattern1), is("1970-01-01 00:00:00.000")); - assertThat( - toChar(0, pattern2), + assertThat(f.toChar(0, pattern2), is("Thursday, 01 12:00:00")); - assertThat( - toChar(timestampStringToUnixDate("2014-09-30 15:28:27.356"), pattern1), + final long ts0 = timestampStringToUnixDate("2014-09-30 15:28:27.356"); + assertThat(f.toChar(ts0, pattern1), is("2014-09-30 15:28:27.356")); - assertThat( - toChar(timestampStringToUnixDate("2014-09-30 15:28:27.356"), pattern2), + assertThat(f.toChar(ts0, pattern2), is("Tuesday, 30 03:28:27")); - assertThat( - toChar(timestampStringToUnixDate("1500-04-30 12:00:00.123"), pattern1), + final long ts1 = timestampStringToUnixDate("1500-04-30 12:00:00.123"); + assertThat(f.toChar(ts1, pattern1), is("1500-04-30 12:00:00.123")); } diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java index 7e92686a1f..f30c7ba256 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java @@ -18,10 +18,13 @@ package org.apache.calcite.linq4j; import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.AbstractList; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.RandomAccess; import java.util.function.ObjIntConsumer; @@ -49,6 +52,17 @@ public class Ord<E> implements Map.Entry<Integer, E> { return new Ord<>(n, e); } + @Override public int hashCode() { + return Objects.hash(e, i); + } + + @Override public boolean equals(@Nullable Object obj) { + return this == obj + || obj instanceof Ord + && i == ((Ord<?>) obj).i + && Objects.equals(e, ((Ord<?>) obj).e); + } + /** * Creates an iterable of {@code Ord}s over an iterable. */ @@ -99,6 +113,7 @@ public class Ord<E> implements Map.Entry<Integer, E> { * <p>Given the array ["a", "b", "c"], returns (2, "c") then (1, "b") then * (0, "a"). */ + @SafeVarargs // heap pollution is not possible because we only read public static <E> Iterable<Ord<E>> reverse(E... elements) { return reverse(ImmutableList.copyOf(elements)); } @@ -121,10 +136,6 @@ public class Ord<E> implements Map.Entry<Integer, E> { @Override public Ord<E> next() { return Ord.of(i, elementList.get(i--)); } - - @Override public void remove() { - throw new UnsupportedOperationException("remove"); - } }; }
