Jackie-Jiang commented on code in PR #17659:
URL: https://github.com/apache/pinot/pull/17659#discussion_r2867098855
##########
pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java:
##########
@@ -169,6 +169,7 @@ public enum TransformFunctionType {
// Special functions
VALUE_IN("valueIn", ReturnTypes.ARG0,
OperandTypes.variadic(SqlOperandCountRanges.from(2))),
+ EVAL_MV("evalMv", ReturnTypes.ARG0,
OperandTypes.family(List.of(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER))),
Review Comment:
Should we call it `FILTER_MV`? It is an extra filter on MV entries
##########
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/EvalMvScalarFunction.java:
##########
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.function.scalar;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.PinotScalarFunction;
+import org.apache.pinot.common.function.sql.PinotSqlFunction;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import
org.apache.pinot.core.operator.transform.function.EvalMvPredicateEvaluator;
+import org.apache.pinot.spi.annotations.ScalarFunction;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+
+
+@ScalarFunction(names = {"evalMv"})
+public class EvalMvScalarFunction implements PinotScalarFunction {
+ private static final Map<ColumnDataType, FunctionInfo>
TYPE_FUNCTION_INFO_MAP =
+ new EnumMap<>(ColumnDataType.class);
+ private static final ConcurrentMap<CacheKey, EvalMvPredicateEvaluator>
EVALUATOR_CACHE =
Review Comment:
Should we use a cache with bounded size?
##########
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/EvalMvScalarFunction.java:
##########
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.function.scalar;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.PinotScalarFunction;
+import org.apache.pinot.common.function.sql.PinotSqlFunction;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import
org.apache.pinot.core.operator.transform.function.EvalMvPredicateEvaluator;
+import org.apache.pinot.spi.annotations.ScalarFunction;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+
+
+@ScalarFunction(names = {"evalMv"})
+public class EvalMvScalarFunction implements PinotScalarFunction {
Review Comment:
Is scalar function version useful? Where do you plan to apply it?
##########
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/EvalMvScalarFunction.java:
##########
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.function.scalar;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.PinotScalarFunction;
+import org.apache.pinot.common.function.sql.PinotSqlFunction;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import
org.apache.pinot.core.operator.transform.function.EvalMvPredicateEvaluator;
+import org.apache.pinot.spi.annotations.ScalarFunction;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+
+
+@ScalarFunction(names = {"evalMv"})
+public class EvalMvScalarFunction implements PinotScalarFunction {
+ private static final Map<ColumnDataType, FunctionInfo>
TYPE_FUNCTION_INFO_MAP =
+ new EnumMap<>(ColumnDataType.class);
+ private static final ConcurrentMap<CacheKey, EvalMvPredicateEvaluator>
EVALUATOR_CACHE =
+ new ConcurrentHashMap<>();
+
+ static {
+ try {
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.INT_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
int[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.LONG_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
long[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.FLOAT_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
float[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.DOUBLE_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
double[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.STRING_ARRAY,
Review Comment:
BYTES_ARRAY is missing
##########
pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/EvalMvPredicateEvaluator.java:
##########
@@ -0,0 +1,473 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.common.request.context.FilterContext;
+import org.apache.pinot.common.request.context.RequestContextUtils;
+import org.apache.pinot.common.request.context.predicate.Predicate;
+import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
+import
org.apache.pinot.core.operator.filter.predicate.PredicateEvaluatorProvider;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.sql.parsers.CalciteSqlParser;
+
+
+public final class EvalMvPredicateEvaluator {
+ private static final EnumSet<Predicate.Type> SUPPORTED_PREDICATE_TYPES =
+ EnumSet.of(Predicate.Type.EQ, Predicate.Type.NOT_EQ, Predicate.Type.IN,
Predicate.Type.NOT_IN,
+ Predicate.Type.RANGE, Predicate.Type.REGEXP_LIKE);
+
+ private final EvalNode _root;
+
+ private EvalMvPredicateEvaluator(EvalNode root) {
+ _root = root;
+ }
+
+ public static EvalMvPredicateEvaluator forPredicate(String predicate,
DataType dataType,
+ @Nullable Dictionary dictionary, @Nullable String expectedColumn) {
Review Comment:
Since the column is provided as the first argument, here we can enforce a
placeholder name (e.g. `v`) to simplify the handling. We always check if lhs is
`v`
##########
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/EvalMvScalarFunction.java:
##########
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.function.scalar;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.PinotScalarFunction;
+import org.apache.pinot.common.function.sql.PinotSqlFunction;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import
org.apache.pinot.core.operator.transform.function.EvalMvPredicateEvaluator;
+import org.apache.pinot.spi.annotations.ScalarFunction;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+
+
+@ScalarFunction(names = {"evalMv"})
+public class EvalMvScalarFunction implements PinotScalarFunction {
+ private static final Map<ColumnDataType, FunctionInfo>
TYPE_FUNCTION_INFO_MAP =
+ new EnumMap<>(ColumnDataType.class);
+ private static final ConcurrentMap<CacheKey, EvalMvPredicateEvaluator>
EVALUATOR_CACHE =
+ new ConcurrentHashMap<>();
+
+ static {
+ try {
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.INT_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
int[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.LONG_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
long[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.FLOAT_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
float[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.DOUBLE_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
double[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.STRING_ARRAY,
Review Comment:
Seems we need to also fix other array functions. Let's open a separate PR to
fix them
##########
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/EvalMvScalarFunction.java:
##########
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.function.scalar;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.PinotScalarFunction;
+import org.apache.pinot.common.function.sql.PinotSqlFunction;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import
org.apache.pinot.core.operator.transform.function.EvalMvPredicateEvaluator;
+import org.apache.pinot.spi.annotations.ScalarFunction;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+
+
+@ScalarFunction(names = {"evalMv"})
+public class EvalMvScalarFunction implements PinotScalarFunction {
+ private static final Map<ColumnDataType, FunctionInfo>
TYPE_FUNCTION_INFO_MAP =
+ new EnumMap<>(ColumnDataType.class);
+ private static final ConcurrentMap<CacheKey, EvalMvPredicateEvaluator>
EVALUATOR_CACHE =
+ new ConcurrentHashMap<>();
+
+ static {
+ try {
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.INT_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
int[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.LONG_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
long[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.FLOAT_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
float[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.DOUBLE_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
double[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(ColumnDataType.STRING_ARRAY,
+ new FunctionInfo(EvalMvScalarFunction.class.getMethod("evalMv",
String[].class, String.class),
+ EvalMvScalarFunction.class, false));
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public String getName() {
+ return "evalMv";
+ }
+
+ @Override
+ public Set<String> getNames() {
+ return Set.of("evalMv");
+ }
+
+ @Nullable
+ @Override
+ public PinotSqlFunction toPinotSqlFunction() {
+ // Should already be registered in PinotOperatorTable by the transform
function implementation
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public FunctionInfo getFunctionInfo(ColumnDataType[] argumentTypes) {
+ if (argumentTypes.length != 2) {
+ return null;
+ }
+ if (argumentTypes[1] != ColumnDataType.STRING) {
+ return null;
+ }
+ return TYPE_FUNCTION_INFO_MAP.get(argumentTypes[0]);
Review Comment:
Look up stored type
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
