Hisoka-X commented on code in PR #9881:
URL: https://github.com/apache/seatunnel/pull/9881#discussion_r2382435151
##########
seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRow.java:
##########
@@ -342,6 +346,13 @@ private int getBytesForValue(Object v) {
}
return rowSize;
default:
+ if (v.getClass().isArray() && v instanceof Object[]) {
Review Comment:
ditto
##########
seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/ArrayFunction.java:
##########
@@ -101,36 +95,31 @@ public static Object[] array(List<Object> args) {
}
public static ArrayType castArrayTypeMapping(Function function,
SeaTunnelRowType inputRowType) {
- return castArrayTypeMapping(getFunctionArgs(function, inputRowType));
- }
+ List<Expression> expressions = CommonFunction.getExpressions(function);
- public static ArrayType castArrayTypeMapping(List<Class<?>> args) {
- if (args == null || args.isEmpty()) {
+ if (expressions.isEmpty()) {
return ArrayType.STRING_ARRAY_TYPE;
}
- Class<?> arrayType = getClassType(args);
- return getSeaTunnelDataType(arrayType);
+ SeaTunnelDataType<?> elementType = null;
+ for (Expression expr : expressions) {
+ SeaTunnelDataType<?> t = CommonFunction.toType(expr, inputRowType);
+ elementType = CommonFunction.unifyCollectionType(elementType, t);
+ }
+ if (elementType == null) {
+ elementType = BasicType.STRING_TYPE;
+ }
+ return createArrayType(elementType);
}
- private static ArrayType getSeaTunnelDataType(Class<?> clazz) {
- String className = clazz.getSimpleName();
- switch (className) {
- case "Integer":
- return ArrayType.INT_ARRAY_TYPE;
- case "Double":
- return ArrayType.DOUBLE_ARRAY_TYPE;
- case "Boolean":
- return ArrayType.BOOLEAN_ARRAY_TYPE;
- case "Long":
- return ArrayType.LONG_ARRAY_TYPE;
- case "float":
- return ArrayType.FLOAT_ARRAY_TYPE;
- case "short":
- return ArrayType.SHORT_ARRAY_TYPE;
- default:
- return ArrayType.STRING_ARRAY_TYPE;
- }
+ static ArrayType createArrayType(SeaTunnelDataType<?> elementType) {
+ if (elementType == BasicType.INT_TYPE) return ArrayType.INT_ARRAY_TYPE;
+ if (elementType == BasicType.LONG_TYPE) return
ArrayType.LONG_ARRAY_TYPE;
+ if (elementType == BasicType.DOUBLE_TYPE) return
ArrayType.DOUBLE_ARRAY_TYPE;
+ if (elementType == BasicType.FLOAT_TYPE) return
ArrayType.FLOAT_ARRAY_TYPE;
+ if (elementType == BasicType.SHORT_TYPE) return
ArrayType.SHORT_ARRAY_TYPE;
+ if (elementType == BasicType.BOOLEAN_TYPE) return
ArrayType.BOOLEAN_ARRAY_TYPE;
+ return ArrayType.STRING_ARRAY_TYPE;
Review Comment:
So if `elementType` is `ArrayType.STRING_ARRAY_TYPE`. The method will return
`ARRAY<STRING>`? Why not `ARRAY<ARRAY<STRING>>`? We should support create any
array type. So I think you should do some upgrade for ArrayType class.
##########
seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRow.java:
##########
@@ -237,7 +237,11 @@ private int getBytesForArray(Object v,
SeaTunnelDataType<?> dataType) {
return getArrayMapNotNullSize(v);
case NULL:
default:
- return 0;
+ int total = 0;
+ for (Object elem : (Object[]) v) {
+ total += getBytesForValue(elem, dataType);
+ }
+ return total;
Review Comment:
why put it into default case. We should put it into the real datatype. For
example, case ARRAY.
##########
seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/nested_type.conf:
##########
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+######
+###### This config file is a demonstration of streaming processing in
seatunnel config
+######
+
+env {
+ job.mode = "BATCH"
+}
+
+source {
+ FakeSource {
+ plugin_output = "fake"
+ row.num = 1
+ string.template = ["nestedType"]
+ schema = {
+ fields {
+ name = "string"
+ }
+ }
+ }
+}
+
+transform {
+ Sql {
+ plugin_input = "fake"
+ plugin_output = "tmp_nested"
+ query = """
+ select
+ ARRAY(ARRAY(ARRAY(1,2,3), ARRAY(4,5,6), ARRAY(ARRAY(1,2,3),
ARRAY(4,5,6)), ARRAY(3, 4))) as arr_of_arr,
+ MAP('k', MAP('k', MAP('k', 1))) as map_of_map,
+ ARRAY(MAP('k', 1), MAP('k2', ARRAY(1, 2))) as arr_of_map,
+ MAP('k', ARRAY(1, 2)) as map_of_arr
+ from dual
+ """
+ }
+}
+
+sink {
+ Console {
+ plugin_input = "tmp_nested"
+ }
Review Comment:
Please use assert sink to verify th data type.
##########
seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java:
##########
@@ -521,15 +524,22 @@ private SeaTunnelDataType<?> getFunctionType(Function
function) {
}
private static List<Expression> getExpressions(Function function) {
- ExpressionList parameters = function.getParameters();
+ ExpressionList<Expression> parameters =
+ (ExpressionList<Expression>) function.getParameters();
if (parameters == null) {
throw new TransformException(
CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
function.getName() + " function requires at least one
parameter");
}
- List<Expression> expressions = parameters.getExpressions();
- if (expressions == null || expressions.isEmpty()) {
+ List<Expression> expressions = new ArrayList<>();
+ if (function.getParameters() != null) {
Review Comment:
```suggestion
if (parameters != null) {
```
##########
seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CommonFunction.java:
##########
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.sql.zeta.functions;
+
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.MapType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.exception.CommonError;
+
+import net.sf.jsqlparser.expression.DoubleValue;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.expression.LongValue;
+import net.sf.jsqlparser.expression.NullValue;
+import net.sf.jsqlparser.expression.StringValue;
+import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import net.sf.jsqlparser.schema.Column;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CommonFunction {
+ private CommonFunction() {}
+
+ public static SeaTunnelDataType toType(Expression expression,
SeaTunnelRowType rowType) {
Review Comment:
I can't get the method behavior for from method name. We should rename it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
