This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/object_type by this push:
new c93161e555e add read_object scalar function
c93161e555e is described below
commit c93161e555ef64a4cf533550e9a9fa53d5183d01
Author: shuwenwei <[email protected]>
AuthorDate: Thu Jul 3 17:13:59 2025 +0800
add read_object scalar function
---
.../relational/ColumnTransformerBuilder.java | 21 ++++
.../builtin/helper/ReadObjectFunctionHelper.java | 80 +++++++++++++++
.../relational/metadata/TableMetadataImpl.java | 24 ++++-
.../unary/scalar/ReadObjectColumnTransformer.java | 114 +++++++++++++++++++++
.../relational/TableBuiltinScalarFunction.java | 1 +
5 files changed, 238 insertions(+), 2 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
index d0b7e35e94c..48586f6a34c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
@@ -133,6 +133,7 @@ import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Lo
import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrim2ColumnTransformer;
import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrimColumnTransformer;
import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RadiansColumnTransformer;
+import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReadObjectColumnTransformer;
import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLike2ColumnTransformer;
import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLikeColumnTransformer;
import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Replace2ColumnTransformer;
@@ -199,6 +200,7 @@ import static
org.apache.tsfile.read.common.type.BooleanType.BOOLEAN;
import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE;
import static org.apache.tsfile.read.common.type.IntType.INT32;
import static org.apache.tsfile.read.common.type.LongType.INT64;
+import static org.apache.tsfile.read.common.type.ObjectType.OBJECT;
import static org.apache.tsfile.read.common.type.StringType.STRING;
public class ColumnTransformerBuilder
@@ -999,6 +1001,25 @@ public class ColumnTransformerBuilder
Type returnType = columnTransformers.get(0).getType();
return AbstractGreatestLeastColumnTransformer.getLeastColumnTransformer(
returnType, columnTransformers);
+ } else if (TableBuiltinScalarFunction.READ_OBJECT
+ .getFunctionName()
+ .equalsIgnoreCase(functionName)) {
+ ColumnTransformer first = this.process(children.get(0), context);
+ if (children.size() == 1) {
+ return new ReadObjectColumnTransformer(OBJECT, first);
+ } else if (children.size() == 2) {
+ return new ReadObjectColumnTransformer(
+ OBJECT, ((LongLiteral) children.get(1)).getParsedValue(), first);
+ } else {
+ long offset = ((LongLiteral) children.get(1)).getParsedValue();
+ long length = ((LongLiteral) children.get(2)).getParsedValue();
+ checkArgument(offset >= 0 && length >= 0);
+ return new ReadObjectColumnTransformer(
+ OBJECT,
+ ((LongLiteral) children.get(1)).getParsedValue(),
+ ((LongLiteral) children.get(2)).getParsedValue(),
+ first);
+ }
} else {
// user defined function
if (TableUDFUtils.isScalarFunction(functionName)) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReadObjectFunctionHelper.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReadObjectFunctionHelper.java
new file mode 100644
index 00000000000..c1a11d948d7
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/builtin/helper/ReadObjectFunctionHelper.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.helper;
+
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import
org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
+import
org.apache.iotdb.db.queryengine.plan.expression.multi.builtin.BuiltInScalarFunctionHelper;
+import org.apache.iotdb.db.queryengine.transformation.api.LayerReader;
+import
org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer;
+import
org.apache.iotdb.db.queryengine.transformation.dag.transformer.Transformer;
+
+import org.apache.tsfile.enums.TSDataType;
+
+import java.util.Map;
+
+import static
org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor.checkFunctionExpressionInputSize;
+
+public class ReadObjectFunctionHelper implements BuiltInScalarFunctionHelper {
+ @Override
+ public void checkBuiltInScalarFunctionInputSize(FunctionExpression
functionExpression)
+ throws SemanticException {
+ checkFunctionExpressionInputSize(
+ functionExpression.getExpressionString(),
+ functionExpression.getExpressions().size(),
+ 1,
+ 2,
+ 3);
+ }
+
+ @Override
+ public void checkBuiltInScalarFunctionInputDataType(TSDataType tsDataType)
+ throws SemanticException {
+ if (tsDataType == TSDataType.OBJECT) {
+ return;
+ }
+ throw new SemanticException(
+ "Input series of Scalar function [READ_OBJECT] only supports data type
[OBJECT]");
+ }
+
+ @Override
+ public TSDataType getBuiltInScalarFunctionReturnType(FunctionExpression
functionExpression) {
+ return TSDataType.BLOB;
+ }
+
+ @Override
+ public ColumnTransformer getBuiltInScalarFunctionColumnTransformer(
+ FunctionExpression expression, ColumnTransformer columnTransformer) {
+ return null;
+ }
+
+ @Override
+ public Transformer getBuiltInScalarFunctionTransformer(
+ FunctionExpression expression, LayerReader layerReader) {
+ return null;
+ }
+
+ @Override
+ public void appendFunctionAttributes(
+ boolean hasExpression, StringBuilder builder, Map<String, String>
functionAttributes) {
+ BuiltInScalarFunctionHelper.super.appendFunctionAttributes(
+ hasExpression, builder, functionAttributes);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
index 8934de172e9..b9e167d1b43 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
@@ -65,7 +65,7 @@ import org.apache.iotdb.udf.api.relational.ScalarFunction;
import org.apache.iotdb.udf.api.relational.TableFunction;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.read.common.type.BlobType;
+import org.apache.tsfile.read.common.type.ObjectType;
import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.TypeFactory;
@@ -79,6 +79,7 @@ import java.util.stream.Collectors;
import static
org.apache.iotdb.db.queryengine.transformation.dag.column.FailFunctionColumnTransformer.FAIL_FUNCTION_NAME;
import static org.apache.tsfile.read.common.type.BinaryType.TEXT;
+import static org.apache.tsfile.read.common.type.BlobType.BLOB;
import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN;
import static org.apache.tsfile.read.common.type.DateType.DATE;
import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE;
@@ -570,6 +571,21 @@ public class TableMetadataImpl implements Metadata {
+ " must have at least two arguments, and all type must be the
same.");
}
return argumentTypes.get(0);
+ } else if (TableBuiltinScalarFunction.READ_OBJECT
+ .getFunctionName()
+ .equalsIgnoreCase(functionName)) {
+ if (argumentTypes.isEmpty()
+ || argumentTypes.size() > 3
+ || !isIntegerNumber(argumentTypes.get(0))
+ // || !isObjectType(argumentTypes.get(0))
+ || (argumentTypes.size() >= 2 &&
!isIntegerNumber(argumentTypes.get(1)))
+ || (argumentTypes.size() >= 3 &&
!isIntegerNumber(argumentTypes.get(2)))) {
+ throw new SemanticException(
+ "Scalar function "
+ + functionName.toLowerCase(Locale.ENGLISH)
+ + " must have at 1~3 arguments, and first argument must be
OBJECT type, other arguments must be int32 or int64 type");
+ }
+ return BLOB;
}
// builtin aggregation function
@@ -962,8 +978,12 @@ public class TableMetadataImpl implements Metadata {
return TEXT.equals(type) || StringType.STRING.equals(type);
}
+ public static boolean isObjectType(Type type) {
+ return ObjectType.OBJECT.equals(type);
+ }
+
public static boolean isBlobType(Type type) {
- return BlobType.BLOB.equals(type);
+ return BLOB.equals(type);
}
public static boolean isBool(Type type) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
new file mode 100644
index 00000000000..15e82be9d8b
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar;
+
+import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
+import
org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer;
+import
org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.read.common.type.Type;
+import org.apache.tsfile.utils.Binary;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
+
+public class ReadObjectColumnTransformer extends UnaryColumnTransformer {
+
+ private long offset = -1;
+ private long length = -1;
+
+ public ReadObjectColumnTransformer(Type type, ColumnTransformer
childColumnTransformer) {
+ super(type, childColumnTransformer);
+ }
+
+ public ReadObjectColumnTransformer(
+ Type type, long offset, ColumnTransformer childColumnTransformer) {
+ super(type, childColumnTransformer);
+ this.offset = offset;
+ }
+
+ public ReadObjectColumnTransformer(
+ Type type, long offset, long length, ColumnTransformer
childColumnTransformer) {
+ super(type, childColumnTransformer);
+ this.offset = offset;
+ this.length = length;
+ }
+
+ @Override
+ protected void doTransform(Column column, ColumnBuilder columnBuilder) {
+ for (int i = 0, n = column.getPositionCount(); i < n; i++) {
+ if (!column.isNull(i)) {
+ transform(column, columnBuilder, i);
+ } else {
+ columnBuilder.appendNull();
+ }
+ }
+ }
+
+ @Override
+ protected void doTransform(Column column, ColumnBuilder columnBuilder,
boolean[] selection) {
+ for (int i = 0, n = column.getPositionCount(); i < n; i++) {
+ if (selection[i] && !column.isNull(i)) {
+ transform(column, columnBuilder, i);
+ } else {
+ columnBuilder.appendNull();
+ }
+ }
+ }
+
+ private void transform(Column column, ColumnBuilder columnBuilder, int i) {
+ // if (TSDataType.OBJECT.equals(column.getDataType())) {
+ // Binary binary = column.getBinary(i);
+ // columnBuilder.writeBinary(readObject(binary));
+ // }
+ columnBuilder.writeBinary(readObject(new Binary(new byte[0])));
+ }
+
+ private Binary readObject(Binary binary) {
+ File file = new File(getObjectPathFromBinary(binary));
+ // TODO: allocate memory
+ long fileSize = file.length();
+ if (offset >= fileSize) {
+ throw new UnsupportedOperationException("offset is greater than object
size");
+ }
+ long actualReadSize = Math.min(length, fileSize - offset);
+ if (actualReadSize > Integer.MAX_VALUE) {
+ throw new UnsupportedOperationException("Read object size is too large
(size > 2G)");
+ }
+ byte[] bytes = new byte[(int) actualReadSize];
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ try (FileChannel fileChannel = FileChannel.open(file.toPath(),
StandardOpenOption.READ)) {
+ fileChannel.read(buffer);
+ } catch (IOException e) {
+ throw new IoTDBRuntimeException(e,
TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ return new Binary(bytes);
+ }
+
+ private String getObjectPathFromBinary(Binary binary) {
+ return "/Users/shuww/Downloads/test_0703/aaa";
+ }
+}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
index 6d5b1ef4721..ced804e681c 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
@@ -67,6 +67,7 @@ public enum TableBuiltinScalarFunction {
FORMAT("format"),
GREATEST("greatest"),
LEAST("least"),
+ READ_OBJECT("read_object"),
;
private final String functionName;
