This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0f439bb1ca [vectorized](udf) java udf support map type (#22059)
0f439bb1ca is described below
commit 0f439bb1caeff927712a8ed597c624b0bae8a87c
Author: Mryange <[email protected]>
AuthorDate: Tue Jul 25 11:56:20 2023 +0800
[vectorized](udf) java udf support map type (#22059)
---
be/src/vec/functions/function_java_udf.cpp | 53 ++++++-
be/src/vec/functions/function_java_udf.h | 1 +
.../apache/doris/common/jni/utils/UdfUtils.java | 48 ++++++-
.../java/org/apache/doris/udf/BaseExecutor.java | 119 ++++++++++++++++
.../java/org/apache/doris/udf/UdfExecutor.java | 152 ++++++++++++++++++++-
.../java/org/apache/doris/catalog/MapType.java | 9 ++
.../main/java/org/apache/doris/catalog/Type.java | 2 +
.../apache/doris/analysis/CreateFunctionStmt.java | 4 +
.../java/org/apache/doris/catalog/ColumnType.java | 15 +-
.../data/javaudf_p0/test_javaudf_map.out | 10 ++
.../java/org/apache/doris/udf/MapIntIntTest.java | 35 +++++
.../java/org/apache/doris/udf/MapStrStrTest.java | 36 +++++
.../suites/javaudf_p0/test_javaudf_map.groovy | 85 ++++++++++++
13 files changed, 560 insertions(+), 9 deletions(-)
diff --git a/be/src/vec/functions/function_java_udf.cpp
b/be/src/vec/functions/function_java_udf.cpp
index cd992b84e1..0df2806026 100644
--- a/be/src/vec/functions/function_java_udf.cpp
+++ b/be/src/vec/functions/function_java_udf.cpp
@@ -32,6 +32,7 @@
#include "util/jni-util.h"
#include "vec/columns/column.h"
#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
#include "vec/columns/column_vector.h"
@@ -71,7 +72,8 @@ Status JavaFunctionCall::open(FunctionContext* context,
FunctionContext::Functio
jni_env->executor_cl, "convertBasicArguments",
"(IZIJJJ)[Ljava/lang/Object;");
jni_env->executor_convert_array_argument_id = env->GetMethodID(
jni_env->executor_cl, "convertArrayArguments",
"(IZIJJJJJ)[Ljava/lang/Object;");
-
+ jni_env->executor_convert_map_argument_id = env->GetMethodID(
+ jni_env->executor_cl, "convertMapArguments",
"(IZIJJJJJJJJ)[Ljava/lang/Object;");
jni_env->executor_result_basic_batch_id = env->GetMethodID(
jni_env->executor_cl, "copyBatchBasicResult",
"(ZI[Ljava/lang/Object;JJJ)V");
jni_env->executor_result_array_batch_id = env->GetMethodID(
@@ -148,6 +150,7 @@ Status JavaFunctionCall::execute(FunctionContext* context,
Block& block,
ColumnPtr null_cols[arg_size];
jclass obj_class = env->FindClass("[Ljava/lang/Object;");
jclass arraylist_class = env->FindClass("Ljava/util/ArrayList;");
+ // jclass hashmap_class = env->FindClass("Ljava/util/HashMap;");
jobjectArray arg_objects = env->NewObjectArray(arg_size, obj_class,
nullptr);
int64_t nullmap_address = 0;
for (size_t arg_idx = 0; arg_idx < arg_size; ++arg_idx) {
@@ -218,6 +221,54 @@ Status JavaFunctionCall::execute(FunctionContext* context,
Block& block,
jni_env->executor_convert_array_argument_id, arg_idx,
arg_column_nullable,
num_rows, nullmap_address, offset_address,
nested_nullmap_address,
nested_data_address, nested_offset_address);
+ } else if (data_cols[arg_idx]->is_column_map()) {
+ const ColumnMap* map_col = assert_cast<const
ColumnMap*>(data_cols[arg_idx].get());
+ auto offset_address =
+
reinterpret_cast<int64_t>(map_col->get_offsets_column().get_raw_data().data);
+ const ColumnNullable& map_key_column_nullable =
+ assert_cast<const ColumnNullable&>(map_col->get_keys());
+ auto key_data_column_null_map =
map_key_column_nullable.get_null_map_column_ptr();
+ auto key_data_column =
map_key_column_nullable.get_nested_column_ptr();
+
+ auto key_nested_nullmap_address = reinterpret_cast<int64_t>(
+
check_and_get_column<ColumnVector<UInt8>>(key_data_column_null_map)
+ ->get_data()
+ .data());
+ int64_t key_nested_data_address = 0, key_nested_offset_address = 0;
+ if (key_data_column->is_column_string()) {
+ const ColumnString* col = assert_cast<const
ColumnString*>(key_data_column.get());
+ key_nested_data_address =
reinterpret_cast<int64_t>(col->get_chars().data());
+ key_nested_offset_address =
reinterpret_cast<int64_t>(col->get_offsets().data());
+ } else {
+ key_nested_data_address =
+
reinterpret_cast<int64_t>(key_data_column->get_raw_data().data);
+ }
+
+ const ColumnNullable& map_value_column_nullable =
+ assert_cast<const ColumnNullable&>(map_col->get_values());
+ auto value_data_column_null_map =
map_value_column_nullable.get_null_map_column_ptr();
+ auto value_data_column =
map_value_column_nullable.get_nested_column_ptr();
+ auto value_nested_nullmap_address = reinterpret_cast<int64_t>(
+
check_and_get_column<ColumnVector<UInt8>>(value_data_column_null_map)
+ ->get_data()
+ .data());
+ int64_t value_nested_data_address = 0, value_nested_offset_address
= 0;
+ // array type need pass address: [nullmap_address],
offset_address, nested_nullmap_address,
nested_data_address/nested_char_address,nested_offset_address
+ if (value_data_column->is_column_string()) {
+ const ColumnString* col = assert_cast<const
ColumnString*>(value_data_column.get());
+ value_nested_data_address =
reinterpret_cast<int64_t>(col->get_chars().data());
+ value_nested_offset_address =
reinterpret_cast<int64_t>(col->get_offsets().data());
+ } else {
+ value_nested_data_address =
+
reinterpret_cast<int64_t>(value_data_column->get_raw_data().data);
+ }
+ arr_obj = (jobjectArray)env->CallNonvirtualObjectMethod(
+ jni_ctx->executor, jni_env->executor_cl,
+ jni_env->executor_convert_map_argument_id, arg_idx,
arg_column_nullable,
+ num_rows, nullmap_address, offset_address,
key_nested_nullmap_address,
+ key_nested_data_address, key_nested_offset_address,
+ value_nested_nullmap_address, value_nested_data_address,
+ value_nested_offset_address);
} else {
return Status::InvalidArgument(
strings::Substitute("Java UDF doesn't support type $0 now
!",
diff --git a/be/src/vec/functions/function_java_udf.h
b/be/src/vec/functions/function_java_udf.h
index ba17942cce..4398fa038d 100644
--- a/be/src/vec/functions/function_java_udf.h
+++ b/be/src/vec/functions/function_java_udf.h
@@ -99,6 +99,7 @@ private:
jmethodID executor_evaluate_id;
jmethodID executor_convert_basic_argument_id;
jmethodID executor_convert_array_argument_id;
+ jmethodID executor_convert_map_argument_id;
jmethodID executor_result_basic_batch_id;
jmethodID executor_result_array_batch_id;
jmethodID executor_close_id;
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
index 09e0dc3804..c546750bcf 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
@@ -18,6 +18,7 @@
package org.apache.doris.common.jni.utils;
import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@@ -52,7 +53,7 @@ import java.time.LocalDateTime;
import java.util.Set;
public class UdfUtils {
- private static final Logger LOG = Logger.getLogger(UdfUtils.class);
+ public static final Logger LOG = Logger.getLogger(UdfUtils.class);
public static final Unsafe UNSAFE;
private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L;
public static final long BYTE_ARRAY_OFFSET;
@@ -95,15 +96,16 @@ public class UdfUtils {
DECIMAL32("DECIMAL32", TPrimitiveType.DECIMAL32, 4),
DECIMAL64("DECIMAL64", TPrimitiveType.DECIMAL64, 8),
DECIMAL128("DECIMAL128", TPrimitiveType.DECIMAL128I, 16),
- ARRAY_TYPE("ARRAY_TYPE", TPrimitiveType.ARRAY, 0);
-
+ ARRAY_TYPE("ARRAY_TYPE", TPrimitiveType.ARRAY, 0),
+ MAP_TYPE("MAP_TYPE", TPrimitiveType.MAP, 0);
private final String description;
private final TPrimitiveType thriftType;
private final int len;
private int precision;
private int scale;
private Type itemType;
-
+ private Type keyType;
+ private Type valueType;
JavaUdfDataType(String description, TPrimitiveType thriftType, int
len) {
this.description = description;
this.thriftType = thriftType;
@@ -153,6 +155,8 @@ public class UdfUtils {
JavaUdfDataType.DECIMAL128);
} else if (c == java.util.ArrayList.class) {
return Sets.newHashSet(JavaUdfDataType.ARRAY_TYPE);
+ } else if (c == java.util.HashMap.class) {
+ return Sets.newHashSet(JavaUdfDataType.MAP_TYPE);
}
return Sets.newHashSet(JavaUdfDataType.INVALID_TYPE);
}
@@ -192,6 +196,22 @@ public class UdfUtils {
public void setItemType(Type type) {
this.itemType = type;
}
+
+ public Type getKeyType() {
+ return keyType;
+ }
+
+ public Type getValueType() {
+ return valueType;
+ }
+
+ public void setKeyType(Type type) {
+ this.keyType = type;
+ }
+
+ public void setValueType(Type type) {
+ this.valueType = type;
+ }
}
public static Pair<Type, Integer> fromThrift(TTypeDesc typeDesc, int
nodeIdx) throws InternalException {
@@ -232,6 +252,14 @@ public class UdfUtils {
nodeIdx = childType.second;
break;
}
+ case MAP: {
+ Preconditions.checkState(nodeIdx + 1 <
typeDesc.getTypesSize());
+ Pair<Type, Integer> keyType = fromThrift(typeDesc, nodeIdx +
1);
+ Pair<Type, Integer> valueType = fromThrift(typeDesc, nodeIdx +
1 + keyType.value());
+ type = new MapType(keyType.key(), valueType.key());
+ nodeIdx = 1 + keyType.value() + valueType.value();
+ break;
+ }
default:
throw new InternalException("Return type " + node.getType() +
" is not supported now!");
@@ -307,6 +335,14 @@ public class UdfUtils {
result.setPrecision(arrType.getItemType().getPrecision());
result.setScale(((ScalarType)
arrType.getItemType()).getScalarScale());
}
+ } else if (retType.isMapType()) {
+ MapType mapType = (MapType) retType;
+ result.setKeyType(mapType.getKeyType());
+ result.setValueType(mapType.getValueType());
+ if (mapType.getKeyType().isDatetimeV2() ||
mapType.getKeyType().isDecimalV3()) {
+ result.setPrecision(mapType.getKeyType().getPrecision());
+ result.setScale(((ScalarType)
mapType.getKeyType()).getScalarScale());
+ }
}
return Pair.of(res.length != 0, result);
}
@@ -332,6 +368,10 @@ public class UdfUtils {
} else if (parameterTypes[finalI].isArrayType()) {
ArrayType arrType = (ArrayType) parameterTypes[finalI];
inputArgTypes[i].setItemType(arrType.getItemType());
+ } else if (parameterTypes[finalI].isMapType()) {
+ MapType mapType = (MapType) parameterTypes[finalI];
+ inputArgTypes[i].setKeyType(mapType.getKeyType());
+ inputArgTypes[i].setValueType(mapType.getValueType());
}
if (res.length == 0) {
return Pair.of(false, inputArgTypes);
diff --git
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
index ef405197d6..abbddab454 100644
---
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
+++
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
@@ -1202,4 +1202,123 @@ public abstract class BaseExecutor {
}
return argument;
}
+
+ public Object[] convertMapArg(PrimitiveType type, int argIdx, boolean
isNullable, int rowStart, int rowEnd,
+ long nullMapAddr,
+ long offsetsAddr, long nestedNullMapAddr, long dataAddr, long
strOffsetAddr) {
+ Object[] argument = (Object[]) Array.newInstance(ArrayList.class,
rowEnd - rowStart);
+ for (int row = rowStart; row < rowEnd; ++row) {
+ long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L
* (row - 1));
+ long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L *
(row));
+ int currentRowNum = (int) (offsetEnd - offsetStart);
+ switch (type) {
+ case BOOLEAN: {
+ argument[row
+ - rowStart] = UdfConvert
+ .convertArrayBooleanArg(row,
currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case TINYINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayTinyIntArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case SMALLINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArraySmallIntArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case INT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayIntArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case BIGINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayBigIntArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case LARGEINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayLargeIntArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case FLOAT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayFloatArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DOUBLE: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDoubleArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case CHAR:
+ case VARCHAR:
+ case STRING: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayStringArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr,
strOffsetAddr);
+ break;
+ }
+ case DATE: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DATETIME: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateTimeArg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DATEV2: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateV2Arg(row, currentRowNum,
offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DATETIMEV2: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateTimeV2Arg(row, currentRowNum,
offsetStart, isNullable,
+ nullMapAddr, nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DECIMALV2:
+ case DECIMAL128: {
+ argument[row - rowStart] = UdfConvert
+
.convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum,
+ offsetStart, isNullable, nullMapAddr,
nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DECIMAL32: {
+ argument[row - rowStart] = UdfConvert
+
.convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum,
+ offsetStart, isNullable, nullMapAddr,
nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DECIMAL64: {
+ argument[row - rowStart] = UdfConvert
+
.convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum,
+ offsetStart, isNullable, nullMapAddr,
nestedNullMapAddr, dataAddr);
+ break;
+ }
+ default: {
+ LOG.info("Not support: " + argTypes[argIdx]);
+ Preconditions.checkState(false, "Not support type " +
argTypes[argIdx].toString());
+ break;
+ }
+ }
+ }
+ return argument;
+ }
}
diff --git
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
index 50528d007b..34333ba1eb 100644
---
a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
+++
b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
@@ -17,6 +17,7 @@
package org.apache.doris.udf;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Pair;
import org.apache.doris.common.exception.UdfRuntimeException;
@@ -36,14 +37,20 @@ import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.MalformedURLException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
import java.util.ArrayList;
+import java.util.HashMap;
public class UdfExecutor extends BaseExecutor {
- private static final Logger LOG = Logger.getLogger(UdfExecutor.class);
+ // private static final java.util.logging.Logger LOG =
+ // Logger.getLogger(UdfExecutor.class);
+ public static final Logger LOG = Logger.getLogger(UdfExecutor.class);
// setup by init() and cleared by close()
private Method method;
- // Pre-constructed input objects for the UDF. This minimizes object
creation overhead
+ // Pre-constructed input objects for the UDF. This minimizes object
creation
+ // overhead
// as these objects are reused across calls to evaluate().
private Object[] inputObjects;
@@ -119,13 +126,76 @@ public class UdfExecutor extends BaseExecutor {
return convertBasicArg(true, argIdx, isNullable, 0, numRows,
nullMapAddr, columnAddr, strOffsetAddr);
}
-
public Object[] convertArrayArguments(int argIdx, boolean isNullable, int
numRows, long nullMapAddr,
long offsetsAddr, long nestedNullMapAddr, long dataAddr, long
strOffsetAddr) {
return convertArrayArg(argIdx, isNullable, 0, numRows, nullMapAddr,
offsetsAddr, nestedNullMapAddr, dataAddr,
strOffsetAddr);
}
+ public Object[] convertMapArguments(int argIdx, boolean isNullable, int
numRows, long nullMapAddr,
+ long offsetsAddr, long keyNestedNullMapAddr, long keyDataAddr,
long keyStrOffsetAddr,
+ long valueNestedNullMapAddr, long valueDataAddr, long
valueStrOffsetAddr) {
+ PrimitiveType keyType =
argTypes[argIdx].getKeyType().getPrimitiveType();
+ PrimitiveType valueType =
argTypes[argIdx].getValueType().getPrimitiveType();
+ Object[] keyCol = convertMapArg(keyType, argIdx, isNullable, 0,
numRows, nullMapAddr, offsetsAddr,
+ keyNestedNullMapAddr, keyDataAddr,
+ keyStrOffsetAddr);
+ Object[] valueCol = convertMapArg(valueType, argIdx, isNullable, 0,
numRows, nullMapAddr, offsetsAddr,
+ valueNestedNullMapAddr, valueDataAddr,
+ valueStrOffsetAddr);
+ switch (keyType) {
+ case BOOLEAN: {
+ return new HashMapBuilder<Boolean>().get(keyCol, valueCol,
valueType);
+ }
+ case TINYINT: {
+ return new HashMapBuilder<Byte>().get(keyCol, valueCol,
valueType);
+ }
+ case SMALLINT: {
+ return new HashMapBuilder<Short>().get(keyCol, valueCol,
valueType);
+ }
+ case INT: {
+ return new HashMapBuilder<Integer>().get(keyCol, valueCol,
valueType);
+ }
+ case BIGINT: {
+ return new HashMapBuilder<Long>().get(keyCol, valueCol,
valueType);
+ }
+ case LARGEINT: {
+ return new HashMapBuilder<BigInteger>().get(keyCol, valueCol,
valueType);
+ }
+ case FLOAT: {
+ return new HashMapBuilder<Float>().get(keyCol, valueCol,
valueType);
+ }
+ case DOUBLE: {
+ return new HashMapBuilder<Double>().get(keyCol, valueCol,
valueType);
+ }
+ case CHAR:
+ case VARCHAR:
+ case STRING: {
+ return new HashMapBuilder<String>().get(keyCol, valueCol,
valueType);
+ }
+ case DATEV2:
+ case DATE: {
+ return new HashMapBuilder<LocalDate>().get(keyCol, valueCol,
valueType);
+ }
+ case DATETIMEV2:
+ case DATETIME: {
+ return new HashMapBuilder<LocalDateTime>().get(keyCol,
valueCol, valueType);
+ }
+ case DECIMAL32:
+ case DECIMAL64:
+ case DECIMALV2:
+ case DECIMAL128: {
+ return new HashMapBuilder<BigDecimal>().get(keyCol, valueCol,
valueType);
+ }
+ default: {
+ LOG.info("Not support: " + keyType);
+ Preconditions.checkState(false, "Not support type " +
keyType.toString());
+ break;
+ }
+ }
+ return null;
+ }
+
/**
* Evaluates the UDF with 'args' as the input to the UDF.
*/
@@ -503,4 +573,80 @@ public class UdfExecutor extends BaseExecutor {
throw new UdfRuntimeException("Unable to call create UDF
instance.", e);
}
}
+
+ public static class HashMapBuilder<keyType> {
+ public Object[] get(Object[] keyCol, Object[] valueCol, PrimitiveType
valueType) {
+ switch (valueType) {
+ case BOOLEAN: {
+ return new BuildMapFromType<keyType,
Boolean>().get(keyCol, valueCol);
+ }
+ case TINYINT: {
+ return new BuildMapFromType<keyType, Byte>().get(keyCol,
valueCol);
+ }
+ case SMALLINT: {
+ return new BuildMapFromType<keyType, Short>().get(keyCol,
valueCol);
+ }
+ case INT: {
+ return new BuildMapFromType<keyType,
Integer>().get(keyCol, valueCol);
+ }
+ case BIGINT: {
+ return new BuildMapFromType<keyType, Long>().get(keyCol,
valueCol);
+ }
+ case LARGEINT: {
+ return new BuildMapFromType<keyType,
BigInteger>().get(keyCol, valueCol);
+ }
+ case FLOAT: {
+ return new BuildMapFromType<keyType, Float>().get(keyCol,
valueCol);
+ }
+ case DOUBLE: {
+ return new BuildMapFromType<keyType, Double>().get(keyCol,
valueCol);
+ }
+ case CHAR:
+ case VARCHAR:
+ case STRING: {
+ return new BuildMapFromType<keyType, String>().get(keyCol,
valueCol);
+ }
+ case DATEV2:
+ case DATE: {
+ return new BuildMapFromType<keyType,
LocalDate>().get(keyCol, valueCol);
+ }
+ case DATETIMEV2:
+ case DATETIME: {
+ return new BuildMapFromType<keyType,
LocalDateTime>().get(keyCol, valueCol);
+ }
+ case DECIMAL32:
+ case DECIMAL64:
+ case DECIMALV2:
+ case DECIMAL128: {
+ return new BuildMapFromType<keyType,
BigDecimal>().get(keyCol, valueCol);
+ }
+ default: {
+ LOG.info("Not support: " + valueType);
+ Preconditions.checkState(false, "Not support type " +
valueType.toString());
+ break;
+ }
+ }
+ return null;
+ }
+ }
+
+ public static class BuildMapFromType<T1, T2> {
+ public Object[] get(Object[] keyCol, Object[] valueCol) {
+ Object[] retHashMap = new HashMap[keyCol.length];
+ for (int colIdx = 0; colIdx < keyCol.length; colIdx++) {
+ HashMap<T1, T2> hashMap = new HashMap<>();
+ ArrayList<T1> keys = (ArrayList<T1>) (keyCol[colIdx]);
+ ArrayList<T2> values = (ArrayList<T2>) (valueCol[colIdx]);
+ for (int i = 0; i < keys.size(); i++) {
+ T1 key = keys.get(i);
+ T2 value = values.get(i);
+ if (!hashMap.containsKey(key)) {
+ hashMap.put(key, value);
+ }
+ }
+ retHashMap[colIdx] = hashMap;
+ }
+ return retHashMap;
+ }
+ }
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
index 8b7a83bc93..8227770636 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
@@ -63,6 +63,15 @@ public class MapType extends Type {
this.isValueContainsNull = true;
}
+ public MapType(Type keyType, Type valueType, boolean keyContainsNull,
boolean valueContainsNull) {
+ Preconditions.checkNotNull(keyType);
+ Preconditions.checkNotNull(valueType);
+ this.keyType = keyType;
+ this.isKeyContainsNull = keyContainsNull;
+ this.valueType = valueType;
+ this.isValueContainsNull = valueContainsNull;
+ }
+
@Override
public PrimitiveType getPrimitiveType() {
return PrimitiveType.MAP;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index db3bb398ab..c57ad04958 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -38,6 +38,7 @@ import java.math.BigInteger;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -258,6 +259,7 @@ public abstract class Type {
.put(PrimitiveType.DECIMAL64,
Sets.newHashSet(BigDecimal.class))
.put(PrimitiveType.DECIMAL128,
Sets.newHashSet(BigDecimal.class))
.put(PrimitiveType.ARRAY, Sets.newHashSet(ArrayList.class))
+ .put(PrimitiveType.MAP, Sets.newHashSet(HashMap.class))
.build();
public static ArrayList<ScalarType> getIntegerTypes() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
index 06422e9b35..d00318d0a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Function;
import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.ScalarFunction;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@@ -577,6 +578,9 @@ public class CreateFunctionStmt extends DdlStmt {
} else if (expType instanceof ArrayType) {
ArrayType arrayType = (ArrayType) expType;
javaTypes =
Type.PrimitiveTypeToJavaClassType.get(arrayType.getPrimitiveType());
+ } else if (expType instanceof MapType) {
+ MapType mapType = (MapType) expType;
+ javaTypes =
Type.PrimitiveTypeToJavaClassType.get(mapType.getPrimitiveType());
} else {
throw new AnalysisException(
String.format("Method '%s' in class '%s' does not support
type '%s'",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
index 10f88eb27f..d4813dbc82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
@@ -166,7 +166,7 @@ public abstract class ColumnType {
}
public static void write(DataOutput out, Type type) throws IOException {
- Preconditions.checkArgument(type.isScalarType() || type.isArrayType(),
+ Preconditions.checkArgument(type.isScalarType() || type.isArrayType()
|| type.isMapType(),
"only support scalar type and array serialization");
if (type.isScalarType()) {
ScalarType scalarType = (ScalarType) type;
@@ -181,6 +181,13 @@ public abstract class ColumnType {
Text.writeString(out, arrayType.getPrimitiveType().name());
write(out, arrayType.getItemType());
out.writeBoolean(arrayType.getContainsNull());
+ } else if (type.isMapType()) {
+ MapType mapType = (MapType) type;
+ Text.writeString(out, mapType.getPrimitiveType().name());
+ write(out, mapType.getKeyType());
+ write(out, mapType.getValueType());
+ out.writeBoolean(mapType.getIsKeyContainsNull());
+ out.writeBoolean(mapType.getIsValueContainsNull());
}
}
@@ -190,6 +197,12 @@ public abstract class ColumnType {
Type itermType = read(in);
boolean containsNull = in.readBoolean();
return ArrayType.create(itermType, containsNull);
+ } else if (primitiveType == PrimitiveType.MAP) {
+ Type keyType = read(in);
+ Type valueType = read(in);
+ boolean keyContainsNull = in.readBoolean();
+ boolean valueContainsNull = in.readBoolean();
+ return new MapType(keyType, valueType, keyContainsNull,
valueContainsNull);
} else {
int scale = in.readInt();
int precision = in.readInt();
diff --git a/regression-test/data/javaudf_p0/test_javaudf_map.out
b/regression-test/data/javaudf_p0/test_javaudf_map.out
new file mode 100644
index 0000000000..7c7cf58b2f
--- /dev/null
+++ b/regression-test/data/javaudf_p0/test_javaudf_map.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_1 --
+{1:1, 10:1, 100:1} 111
+{2:1, 20:1, 200:1, 2000:1} 2222
+{3:1} 3
+
+-- !select_2 --
+{"114":"514", "1919":"810"} 1145141919810
+{"a":"bc", "def":"g", "hij":"k"} abcdefghijk
+
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java
new file mode 100644
index 0000000000..ae1fb9bc77
--- /dev/null
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.util.*;
+
+public class MapIntIntTest extends UDF {
+ public Integer evaluate(HashMap<Integer, Integer> hashMap) {
+ Integer mul = 0;
+ for (Map.Entry<Integer, Integer> entry : hashMap.entrySet()) {
+ Integer key = entry.getKey();
+ Integer value = entry.getValue();
+ mul += key * value;
+ }
+ return mul;
+ }
+}
+
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java
new file mode 100644
index 0000000000..fd5ac9c20f
--- /dev/null
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.util.*;
+
+public class MapStrStrTest extends UDF {
+ public String evaluate(HashMap<String, String> hashMap) {
+ StringBuffer sb = new StringBuffer();
+ for (Map.Entry<String, String> entry : hashMap.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ sb.append((key + value));
+ }
+ String ans = sb.toString();
+ return ans;
+ }
+}
+
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_map.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_map.groovy
new file mode 100644
index 0000000000..4f7b89fd92
--- /dev/null
+++ b/regression-test/suites/javaudf_p0/test_javaudf_map.groovy
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_javaudf_map") {
+ def jarPath =
"""${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar"""
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ try_sql("DROP FUNCTION IF EXISTS udfii(Map<INT, INT>);")
+ try_sql("DROP FUNCTION IF EXISTS udfss(Map<String, String>);")
+ try_sql("DROP TABLE IF EXISTS map_ii")
+ try_sql("DROP TABLE IF EXISTS map_ss")
+ sql """
+ CREATE TABLE IF NOT EXISTS map_ii (
+ `id` INT(11) NULL COMMENT "",
+ `m` Map<INT, INT> NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
+ );
+ """
+ sql """ """
+ sql """ INSERT INTO map_ii VALUES(1, {1:1,10:1,100:1}); """
+ sql """ INSERT INTO map_ii VALUES(2, {2:1,20:1,200:1,2000:1}); """
+ sql """ INSERT INTO map_ii VALUES(3, {3:1}); """
+ sql """ DROP FUNCTION IF EXISTS udfii(Map<INT, INT>); """
+ sql """ CREATE FUNCTION udfii(Map<INT, INT>) RETURNS INT PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MapIntIntTest",
+ "type"="JAVA_UDF"
+ ); """
+
+
+ qt_select_1 """ select m,udfii(m) from map_ii order by id; """
+
+ sql """ CREATE TABLE IF NOT EXISTS map_ss (
+ `id` INT(11) NULL COMMENT "",
+ `m` Map<String, String> NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
+ ); """
+ sql """ INSERT INTO map_ss VALUES(1, {"114":"514","1919":"810"});
"""
+ sql """ INSERT INTO map_ss VALUES(2, {"a":"bc","def":"g","hij":"k"});
"""
+ sql """ DROP FUNCTION IF EXISTS udfss(Map<String, String>); """
+
+ sql """ CREATE FUNCTION udfss(Map<String, String>) RETURNS STRING
PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MapStrStrTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select_2 """ select m,udfss(m) from map_ss order by id; """
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udfii(Map<INT, INT>);")
+ try_sql("DROP FUNCTION IF EXISTS udfss(Map<String, String>);")
+ try_sql("DROP TABLE IF EXISTS map_ii")
+ try_sql("DROP TABLE IF EXISTS map_ss")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
