This is an automated email from the ASF dual-hosted git repository. HappenLee pushed a commit to branch paimon-blob-4.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 88c5dbcf828dee62d7f3166c2212feb6fdd7a66b Author: linzhenqi <[email protected]> AuthorDate: Thu Oct 22 15:57:46 2026 +0800 [feature](function) Add independent FL_GET_BINARY access for FILE ### What problem does this PR solve? Problem Summary: Expose stored FILE binary contents through an independent FL_GET_BINARY implementation and reject the FILE inline selector in ELEMENT_AT. Reuse the inline column when it contains no NULLs; otherwise preserve existing binary values and insert NULL for missing contents. Keep constant results constant and use the default input NULL handling. ### Release note FL_GET_BINARY(FILE) returns nullable VARBINARY from the stored inline contents. Missing contents return NULL. ELEMENT_AT(file, 'inline') reports an error; the other public FILE selectors remain supported. ### Check List (For Author) - Test: ASAN BE build; 43 FE unit tests; 9 BE unit tests with coverage; test_fl_get_binary, test_file_type, test_file_type_internal_disabled, and test_list_files regression suites; clang-format 16, clang-tidy, and header hygiene. - Behavior changed: Yes, FILE binary contents are accessed with FL_GET_BINARY. - Does this need documentation: Yes, updated in the local doris-website file-type-docs branch. (cherry picked from commit c791f63b575480d7d6f9100b7b7e8185d17cd648) --- be/src/exprs/function/function_fl_get_binary.cpp | 67 ++++++++++++ be/src/exprs/function/simple_function_factory.h | 2 + be/test/exprs/file_expr_test.cpp | 87 +++++++++++++-- .../doris/catalog/BuiltinScalarFunctions.java | 2 + .../expressions/functions/scalar/ElementAt.java | 3 + .../expressions/functions/scalar/FlGetBinary.java | 63 +++++++++++ .../expressions/visitor/ScalarFunctionVisitor.java | 5 + .../nereids/rules/rewrite/FileAccessPathTest.java | 2 +- .../functions/scalar/FlGetBinaryTest.java | 77 ++++++++++++++ .../nereids/types/FileTypeExpressionTest.java | 30 +++--- .../complex_types/test_fl_get_binary.out | 35 ++++++ .../complex_types/test_file_type.groovy | 22 ++-- .../test_file_type_internal_disabled.groovy | 4 +- .../complex_types/test_fl_get_binary.groovy | 117 +++++++++++++++++++++ 14 files changed, 480 insertions(+), 36 deletions(-) diff --git a/be/src/exprs/function/function_fl_get_binary.cpp b/be/src/exprs/function/function_fl_get_binary.cpp new file mode 100644 index 00000000000..6673a6dd66a --- /dev/null +++ b/be/src/exprs/function/function_fl_get_binary.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_file.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" + +namespace doris { + +class FunctionFlGetBinary : public IFunction { +public: + static constexpr auto name = "fl_get_binary"; + static FunctionPtr create() { return std::make_shared<FunctionFlGetBinary>(); } + String get_name() const override { return name; } + size_t get_number_of_arguments() const override { return 1; } + bool use_default_implementation_for_constants() const override { return true; } + + DataTypePtr get_return_type_impl(const DataTypes&) const override { + return make_nullable(std::make_shared<DataTypeVarbinary>()); + } + + Status execute_impl(FunctionContext*, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + const auto& file = + assert_cast<const ColumnFile&>(*block.get_by_position(arguments[0]).column); + const auto& inline_column = file.get_column_ptr(DataTypeFile::INLINE_FIELD_INDEX); + if (!inline_column->has_null()) { + block.replace_by_position(result, inline_column); + return Status::OK(); + } + + auto result_column = inline_column->clone_empty(); + result_column->reserve(input_rows_count); + for (size_t row = 0; row < input_rows_count; ++row) { + if (!inline_column->is_null_at(row)) { + result_column->insert_from(*inline_column, row); + } else { + // TODO: Read the remote file's binary contents and store them in the FILE value. + result_column->insert_default(); + } + } + block.replace_by_position(result, std::move(result_column)); + return Status::OK(); + } +}; + +void register_function_fl_get_binary(SimpleFunctionFactory& factory) { + factory.register_function<FunctionFlGetBinary>(); +} + +} // namespace doris diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index d43c6c3ca56..951dd183946 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -116,6 +116,7 @@ void register_function_compress(SimpleFunctionFactory& factory); void register_function_bit_test(SimpleFunctionFactory& factory); void register_function_dict_get(SimpleFunctionFactory& factory); void register_function_to_file(SimpleFunctionFactory& factory); +void register_function_fl_get_binary(SimpleFunctionFactory& factory); void register_function_dict_get_many(SimpleFunctionFactory& factory); void register_function_ai(SimpleFunctionFactory& factory); void register_function_score(SimpleFunctionFactory& factory); @@ -352,6 +353,7 @@ public: register_function_compress(instance); register_function_dict_get(instance); register_function_to_file(instance); + register_function_fl_get_binary(instance); register_function_dict_get_many(instance); register_function_ai(instance); register_function_score(instance); diff --git a/be/test/exprs/file_expr_test.cpp b/be/test/exprs/file_expr_test.cpp index b35e4bf1d3e..14201a6cd87 100644 --- a/be/test/exprs/file_expr_test.cpp +++ b/be/test/exprs/file_expr_test.cpp @@ -167,12 +167,14 @@ protected: auto input = make_nullable(file_type)->create_column_const(4, parent_null ? Field() : value); Block block {{input, make_nullable(file_type), "file"}, - {selector("inline", 4), text, "field"}, {nullptr, make_nullable(binary), "output"}}; - FunctionArrayElement function; - ASSERT_TRUE(function.execute_impl(nullptr, block, {0, 1}, 2, 4).ok()); - const auto& output = block.get_by_position(2).column; + auto function = SimpleFunctionFactory::instance().get_function( + "fl_get_binary", {block.get_by_position(0)}, make_nullable(binary)); + ASSERT_NE(function, nullptr); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 4).ok()); + const auto& output = block.get_by_position(1).column; ASSERT_EQ(output->size(), 4); + ASSERT_TRUE(is_column_const(*output)); for (size_t row = 0; row < 4; ++row) { check_constant_inline_row(*output, row, payload, parent_null); } @@ -202,14 +204,14 @@ TEST_F(FileExprTest, InlineGetterAndSixFieldStructRoundTripPreserveBinary) { EXPECT_EQ((*restored)[1].get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().size(), 0); EXPECT_TRUE((*restored)[2].get<TYPE_FILE>()[5].is_null()); EXPECT_TRUE(restored->is_null_at(3)); - ColumnsWithTypeAndName args {input, {selector("InLiNe", 4), text, "field"}}; - auto function = SimpleFunctionFactory::instance().get_function("element_at", args, + ColumnsWithTypeAndName args {input}; + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", args, schema.get_element(5)); ASSERT_NE(function, nullptr); EXPECT_EQ(remove_nullable(function->get_return_type())->get_primitive_type(), TYPE_VARBINARY); - Block block {args[0], args[1], {nullptr, schema.get_element(5), "output"}}; - ASSERT_TRUE(function->execute(nullptr, block, {0, 1}, 2, 4).ok()); - const auto& output = block.get_by_position(2).column; + Block block {args[0], {nullptr, schema.get_element(5), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 4).ok()); + const auto& output = block.get_by_position(1).column; EXPECT_EQ((*output)[0].get<TYPE_VARBINARY>().str(), bytes); EXPECT_FALSE(output->is_null_at(1)); EXPECT_EQ((*output)[1].get<TYPE_VARBINARY>().size(), 0); @@ -217,6 +219,73 @@ TEST_F(FileExprTest, InlineGetterAndSixFieldStructRoundTripPreserveBinary) { EXPECT_TRUE(output->is_null_at(3)); } +TEST_F(FileExprTest, BinaryGetterPreservesParentNullsAndSharesPayload) { + std::string bytes(1 << 20, '\xff'); + bytes[0] = '\0'; + auto value = file_value(); + value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); + auto values = make_nullable(file_type)->create_column(); + values->insert(value); + values->insert(value); + values->insert(value); + auto& source = assert_cast<ColumnNullable&>(*values); + source.get_null_map_data()[0] = 1; + ColumnWithTypeAndName input {std::move(values), make_nullable(file_type), "input"}; + const auto& inline_column = assert_cast<const ColumnNullable&>( + assert_cast<const ColumnFile&>(source.get_nested_column()).get_column(5)); + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 3).ok()); + const auto& output = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column); + EXPECT_TRUE(output.is_null_at(0)); + EXPECT_FALSE(output.is_null_at(1)); + EXPECT_FALSE(output.is_null_at(2)); + EXPECT_EQ(output[1].get<TYPE_VARBINARY>().str(), bytes); + EXPECT_EQ(output.get_nested_column_ptr(), inline_column.get_nested_column_ptr()); + EXPECT_FALSE(inline_column.is_null_at(0)); + EXPECT_FALSE(source.is_null_at(1)); +} + +TEST_F(FileExprTest, BinaryGetterInsertsNullForMissingInline) { + const std::string bytes("\0\xff\x80", 3); + auto value = file_value(); + value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); + auto empty = file_value(); + empty.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView()); + auto values = file_type->create_column(); + values->insert(value); + values->insert(value); + values->insert(empty); + auto& inline_column = assert_cast<ColumnNullable&>( + assert_cast<ColumnFile&>(*values).get_column(DataTypeFile::INLINE_FIELD_INDEX)); + inline_column.get_null_map_data()[1] = 1; + ColumnWithTypeAndName input {std::move(values), file_type, "input"}; + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 3).ok()); + const auto& output = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column); + EXPECT_EQ(output[0].get<TYPE_VARBINARY>().str(), bytes); + EXPECT_TRUE(output.is_null_at(1)); + EXPECT_TRUE(output.get_nested_column()[1].get<TYPE_VARBINARY>().empty()); + EXPECT_FALSE(output.is_null_at(2)); + EXPECT_TRUE(output[2].get<TYPE_VARBINARY>().empty()); + EXPECT_EQ(inline_column.get_nested_column()[1].get<TYPE_VARBINARY>().str(), bytes); +} + +TEST_F(FileExprTest, BinaryGetterHandlesEmptyColumns) { + const auto input = column(file_type, {}); + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 0).ok()); + EXPECT_EQ(block.get_by_position(1).column->size(), 0); +} + TEST_F(FileExprTest, StructCastMatchesReorderedPublicNamesAndStringFamily) { auto source_type = std::make_shared<DataTypeStruct>( DataTypes {bigint, std::make_shared<DataTypeString>(65533, TYPE_VARCHAR), bigint, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index ebe5b0103df..deaa13f42c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -214,6 +214,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Factorial; import org.apache.doris.nereids.trees.expressions.functions.scalar.Field; import org.apache.doris.nereids.trees.expressions.functions.scalar.FindInSet; import org.apache.doris.nereids.trees.expressions.functions.scalar.FirstSignificantSubdomain; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor; import org.apache.doris.nereids.trees.expressions.functions.scalar.Fmod; import org.apache.doris.nereids.trees.expressions.functions.scalar.Format; @@ -788,6 +789,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Field.class, "field"), scalar(FindInSet.class, "find_in_set"), scalar(FirstSignificantSubdomain.class, "first_significant_subdomain"), + scalar(FlGetBinary.class, "fl_get_binary"), scalar(Floor.class, "floor"), scalar(Fmod.class, "fmod"), scalar(Format.class, "format"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java index 523939fc876..99394e17953 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java @@ -123,6 +123,9 @@ public class ElementAt extends ScalarFunction if (!(getArgument(1) instanceof StringLikeLiteral)) { throw new AnalysisException("element_at over FILE requires a constant public field name"); } + if (((StringLikeLiteral) getArgument(1)).getStringValue().equalsIgnoreCase("inline")) { + throw new AnalysisException("ELEMENT_AT does not support the FILE inline field; use FL_GET_BINARY"); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinary.java new file mode 100644 index 00000000000..f9eb190fdef --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinary.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.NullOrIdenticalSignature; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Returns the binary contents of a FILE. */ +public class FlGetBinary extends ScalarFunction + implements UnaryExpression, NullOrIdenticalSignature, AlwaysNullable { + public static final List<FunctionSignature> SIGNATURES = ImmutableList.of( + FunctionSignature.ret(VarBinaryType.INSTANCE).args(FileType.INSTANCE)); + + public FlGetBinary(Expression arg) { + super("fl_get_binary", arg); + } + + private FlGetBinary(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public FlGetBinary withChildren(List<Expression> children) { + Preconditions.checkArgument(children.size() == 1); + return new FlGetBinary(getFunctionParams(children)); + } + + @Override + public List<FunctionSignature> getSignatures() { + return SIGNATURES; + } + + @Override + public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) { + return visitor.visitFlGetBinary(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 31c6de5fe02..bc5d1417ff6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -232,6 +232,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Factorial; import org.apache.doris.nereids.trees.expressions.functions.scalar.Field; import org.apache.doris.nereids.trees.expressions.functions.scalar.FindInSet; import org.apache.doris.nereids.trees.expressions.functions.scalar.FirstSignificantSubdomain; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor; import org.apache.doris.nereids.trees.expressions.functions.scalar.Fmod; import org.apache.doris.nereids.trees.expressions.functions.scalar.Format; @@ -1492,6 +1493,10 @@ public interface ScalarFunctionVisitor<R, C> { return visitScalarFunction(firstSignificantSubdomain, context); } + default R visitFlGetBinary(FlGetBinary function, C context) { + return visitScalarFunction(function, context); + } + default R visitFloor(Floor floor, C context) { return visitScalarFunction(floor, context); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java index eefea5a9fda..51036613df5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java @@ -72,7 +72,7 @@ public class FileAccessPathTest { @Test public void testCanonicalChildrenAndGetterPaths() { SlotReference file = slot("f", FileType.INSTANCE, false); - for (String name : ImmutableList.of("uri", "offset", "size", "content_type", "checksum", "inline")) { + for (String name : ImmutableList.of("uri", "offset", "size", "content_type", "checksum")) { AccessPathInfo info = prune(file, field(file, name)); Assertions.assertEquals(ImmutableList.of(data("f", name)), info.getAllAccessPaths()); Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinaryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinaryTest.java new file mode 100644 index 00000000000..086f7b089bd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinaryTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.RewriteWhenAnalyze; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.JsonType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; +import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +class FlGetBinaryTest { + @Test + void testNullableBinarySignature() { + for (Expression input : Arrays.asList(new SlotReference("file", FileType.INSTANCE), + new NullLiteral(FileType.INSTANCE), new NullLiteral())) { + Expression function = TypeCoercionUtils.processBoundFunction(new FlGetBinary(input)); + Assertions.assertEquals(VarBinaryType.INSTANCE, function.getDataType()); + Assertions.assertTrue(function.nullable()); + Assertions.assertFalse(function instanceof RewriteWhenAnalyze); + } + } + + @Test + void testRejectsOtherArgumentTypes() { + for (DataType type : Arrays.asList(IntegerType.INSTANCE, StringType.INSTANCE, JsonType.INSTANCE, + VariantType.INSTANCE, FileType.INSTANCE.publicStructType(), ArrayType.of(FileType.INSTANCE))) { + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processBoundFunction(new FlGetBinary(new SlotReference("value", type)))); + } + } + + @Test + void testFileInlineSelectorRejected() { + for (Expression input : Arrays.asList(new SlotReference("file", FileType.INSTANCE), + new NullLiteral(FileType.INSTANCE))) { + for (String name : Arrays.asList("inline", "INLINE", "InLiNe")) { + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processBoundFunction(new ElementAt(input, new StringLiteral(name)))); + Assertions.assertTrue(error.getMessage().contains("use FL_GET_BINARY")); + } + } + Expression struct = new SlotReference("value", FileType.INSTANCE.publicStructType()); + Assertions.assertEquals(VarBinaryType.INSTANCE, + TypeCoercionUtils.processBoundFunction(new ElementAt(struct, new StringLiteral("inline"))) + .getDataType()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java index 89d8ff388f6..47466b295ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java @@ -59,6 +59,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayReverseS import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy; import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf; @@ -248,9 +249,10 @@ public class FileTypeExpressionTest { Assertions.assertEquals(BigIntType.INSTANCE, TypeCoercionUtils.processBoundFunction(new ElementAt(file, new StringLiteral("SIZE"))).getDataType()); Assertions.assertTrue(new ElementAt(file, new StringLiteral("uri")).nullable()); - Assertions.assertEquals(VarBinaryType.INSTANCE, - TypeCoercionUtils.processBoundFunction(new ElementAt(file, new StringLiteral("INLINE"))).getDataType()); - Assertions.assertTrue(new ElementAt(file, new StringLiteral("inline")).nullable()); + for (String selector : Arrays.asList("inline", "INLINE")) { + assertAnalysisError("use FL_GET_BINARY", + () -> TypeCoercionUtils.processBoundFunction(new ElementAt(file, new StringLiteral(selector)))); + } for (Expression selector : Arrays.asList(new StringLiteral("path"), new IntegerLiteral(1), new SlotReference("s", StringType.INSTANCE))) { assertAnalysisError(selector instanceof StringLiteral @@ -430,9 +432,9 @@ public class FileTypeExpressionTest { + "'content_type', null, 'checksum', null, 'inline', X'00FF80') as file)"; for (String expression : Arrays.asList("coalesce(cast(null as file), " + inline + ")", "if(true, " + inline + ", null)", "ifnull(null, " + inline + ")", - "element_at(element_at(array(" + inline + ", null), 1), 'inline')", - "element_at(element_at(named_struct('asset', " + inline + "), 'asset'), 'inline')", - "element_at(element_at(map('asset', " + inline + "), 'asset'), 'inline')")) { + "fl_get_binary(element_at(array(" + inline + ", null), 1))", + "fl_get_binary(element_at(named_struct('asset', " + inline + "), 'asset'))", + "fl_get_binary(element_at(map('asset', " + inline + "), 'asset'))")) { Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) .analyze("select " + expression), expression); } @@ -483,6 +485,9 @@ public class FileTypeExpressionTest { public void testElementAtExplodeAndReverseCastFields() { Expression file = new SlotReference("f", FileType.INSTANCE); for (StructField field : FileType.INSTANCE.getFields()) { + if (field.getName().equals("inline")) { + continue; + } Expression accessed = ExpressionAnalyzer.analyzeFunction(null, null, new ElementAt(file, new StringLiteral(field.getName().toUpperCase(Locale.ROOT)))); Assertions.assertTrue(accessed instanceof ElementAt); @@ -491,10 +496,9 @@ public class FileTypeExpressionTest { Assertions.assertEquals(field.getDataType(), accessed.getDataType()); Assertions.assertTrue(accessed.nullable()); } - ElementAt inline = new ElementAt(new SlotReference("required", FileType.INSTANCE, false), - new StringLiteral("inline")); - Assertions.assertEquals(VarBinaryType.INSTANCE, inline.getDataType()); - Assertions.assertTrue(inline.nullable()); + FlGetBinary binary = new FlGetBinary(new SlotReference("required", FileType.INSTANCE, false)); + Assertions.assertEquals(VarBinaryType.INSTANCE, binary.getDataType()); + Assertions.assertTrue(binary.nullable()); for (BoundFunction explode : Arrays.asList(new ExplodeFile(file), new ExplodeFileOuter(file))) { Assertions.assertEquals(FileType.INSTANCE.publicStructType(), TypeCoercionUtils.processBoundFunction(explode).getDataType()); @@ -649,8 +653,8 @@ public class FileTypeExpressionTest { public void testSqlValueTransportAndExplodeAliases() { for (String sql : Arrays.asList( "select element_at(cast(null as file), 'uri')", - "select element_at(cast(null as file), 'inline')", - "select element_at(cast(null as file), 'INLINE')", + "select fl_get_binary(cast(null as file))", + "select fl_get_binary(null)", "select element_at(cast(null as file), 'SIZE')", "select cast(null as file) union all select cast(named_struct('uri', 'urn:x', 'offset', cast(0 " + "as bigint), 'size', cast(0 as bigint), 'content_type', '', 'checksum', '', " @@ -681,7 +685,7 @@ public class FileTypeExpressionTest { @Test public void testInlineAndExplodeSqlOutputSchema() { Plan getters = PlanChecker.from(MemoTestUtils.createConnectContext()).analyze( - "select element_at(f, 'inline'), element_at(f,'INLINE') " + "select fl_get_binary(f), FL_GET_BINARY(f) " + "from (select cast(null as file) f) t").getPlan(); Assertions.assertEquals(2, getters.getOutput().size()); getters.getOutput().forEach(output -> { diff --git a/regression-test/data/datatype_p0/complex_types/test_fl_get_binary.out b/regression-test/data/datatype_p0/complex_types/test_fl_get_binary.out new file mode 100644 index 00000000000..8bae755bfbb --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_fl_get_binary.out @@ -0,0 +1,35 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !fl_get_binary_constants_false -- +s3://bucket/a.png 2 3 image/png ETAG:abc 00FF80 + +-- !fl_get_binary_nulls_false -- +\N \N + +-- !fl_get_binary_constants_true -- +s3://bucket/a.png 2 3 image/png ETAG:abc 00FF80 + +-- !fl_get_binary_nulls_true -- +\N \N + +-- !fl_get_binary_columns_1 -- +1 s3://bucket/a.png 2 3 image/png ETAG:abc 00FF80 +2 \N \N \N \N \N \N +3 urn:empty 0 0 \N \N +4 urn:metadata \N \N \N \N \N + +-- !fl_get_binary_nested_1 -- +1 00FF80 00FF80 00FF80 +2 \N \N \N +3 +4 \N \N \N + +-- !fl_get_binary_aggregate_1 -- +3 3 2 + +-- !fl_get_binary_struct_1 -- +00FF80 + +-- !fl_get_binary_filter_1 -- +1 00FF80 +3 + diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy index beb91d4f868..9949360ea1e 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy @@ -58,8 +58,8 @@ suite("test_file_type") { qt_fields fileQuery(""" SELECT id, ELEMENT_AT(f, 'uri'), ELEMENT_AT(f, 'offset'), ELEMENT_AT(f, 'size'), ELEMENT_AT(f, 'content_type'), ELEMENT_AT(f, 'checksum'), ELEMENT_AT(f, 'UrI'), - FROM_BINARY(ELEMENT_AT(f, 'inline')), FROM_BINARY(ELEMENT_AT(f, 'INLINE')), - ELEMENT_AT(f, 'inline') IS NULL, + FROM_BINARY(FL_GET_BINARY(f)), FROM_BINARY(FL_GET_BINARY(f)), + FL_GET_BINARY(f) IS NULL, f IS NULL, f IS NOT NULL FROM test_file_type_values ORDER BY id """) @@ -149,7 +149,7 @@ suite("test_file_type") { qt_conditional_values fileQuery(""" SELECT id, COALESCE(f, ${inlineFallback}), IF(id = 2, ${inlineFallback}, f), IFNULL(f, ${inlineFallback}), CASE WHEN id = 2 THEN ${inlineFallback} ELSE f END, - FROM_BINARY(ELEMENT_AT(COALESCE(f, ${inlineFallback}), 'inline')) + FROM_BINARY(FL_GET_BINARY(COALESCE(f, ${inlineFallback}))) FROM test_file_type_values ORDER BY id """) qt_null_only_propagation fileQuery(""" @@ -175,7 +175,7 @@ suite("test_file_type") { """) qt_count_fields fileQuery(""" SELECT COUNT(ELEMENT_AT(f, 'uri')), COUNT(ELEMENT_AT(f, 'size')), - COUNT(ELEMENT_AT(f, 'inline')) FROM test_file_type_values + COUNT(FL_GET_BINARY(f)) FROM test_file_type_values """) test { sql fileQuery("SELECT ANY_VALUE(f) FROM test_file_type_values") @@ -451,9 +451,9 @@ suite("test_file_type") { } } qt_inline_struct_roundtrip fileQuery(""" - SELECT id, FROM_BINARY(ELEMENT_AT(CAST(CAST(f AS + SELECT id, FROM_BINARY(FL_GET_BINARY(CAST(CAST(f AS STRUCT<inline:VARBINARY,checksum:STRING,content_type:STRING,size:BIGINT,offset:BIGINT,uri:STRING>) - AS FILE), 'inline')) + AS FILE))) FROM test_file_type_values ORDER BY id """) qt_explicit_forward fileQuery(""" @@ -498,18 +498,18 @@ def evaluate(value): qt_inline_values fileQuery(""" SELECT id, f, ELEMENT_AT(f, 'uri'), ELEMENT_AT(f, 'size'), - FROM_BINARY(ELEMENT_AT(f, 'inline')), f IS NULL + FROM_BINARY(FL_GET_BINARY(f)), f IS NULL FROM test_file_type_inline_values ORDER BY id """) qt_inline_counts fileQuery(""" - SELECT COUNT(f), COUNT(ELEMENT_AT(f, 'uri')), COUNT(ELEMENT_AT(f, 'inline')) + SELECT COUNT(f), COUNT(ELEMENT_AT(f, 'uri')), COUNT(FL_GET_BINARY(f)) FROM test_file_type_inline_values """) qt_inline_constructors fileQuery(""" SELECT id, ARRAY(f, NULL), NAMED_STRUCT('asset', f), MAP('asset', f), - FROM_BINARY(ELEMENT_AT(ELEMENT_AT(ARRAY(f), 1), 'inline')), - FROM_BINARY(ELEMENT_AT(ELEMENT_AT(NAMED_STRUCT('asset', f), 'asset'), 'inline')), - FROM_BINARY(ELEMENT_AT(ELEMENT_AT(MAP('asset', f), 'asset'), 'inline')) + FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(ARRAY(f), 1))), + FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(NAMED_STRUCT('asset', f), 'asset'))), + FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(MAP('asset', f), 'asset'))) FROM test_file_type_inline_values ORDER BY id """) qt_inline_struct fileQuery("SELECT id, CAST(f AS ${publicStruct}) FROM test_file_type_inline_values ORDER BY id") diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy index 7fff3c0c586..0c6ab4009ab 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy @@ -67,7 +67,7 @@ suite("test_file_type_internal_disabled") { } } qt_inline """ - SELECT f, ELEMENT_AT(f, 'uri'), HEX(CAST(ELEMENT_AT(f, 'inline') AS STRING)), f IS NULL + SELECT f, ELEMENT_AT(f, 'uri'), HEX(CAST(FL_GET_BINARY(f) AS STRING)), f IS NULL FROM (SELECT CAST(NAMED_STRUCT('uri', NULL, 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', X'00FF') AS FILE) AS f) t """ @@ -77,7 +77,7 @@ suite("test_file_type_internal_disabled") { CREATE TABLE test_file_type_disabled_scalar DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1") AS SELECT 1 AS id, ELEMENT_AT(f, 'uri') AS uri, - HEX(CAST(ELEMENT_AT(f, 'inline') AS STRING)) AS inline_hex + HEX(CAST(FL_GET_BINARY(f) AS STRING)) AS inline_hex FROM (SELECT CAST(NAMED_STRUCT('uri', NULL, 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', X'00FF') AS FILE) AS f) t """ diff --git a/regression-test/suites/datatype_p0/complex_types/test_fl_get_binary.groovy b/regression-test/suites/datatype_p0/complex_types/test_fl_get_binary.groovy new file mode 100644 index 00000000000..cc1eec7131e --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_fl_get_binary.groovy @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_fl_get_binary") { + def file = """CAST(NAMED_STRUCT('uri', 's3://bucket/a.png', 'offset', CAST(2 AS BIGINT), + 'size', CAST(3 AS BIGINT), 'content_type', 'image/png', 'checksum', 'ETAG:abc', + 'inline', X'00FF80') AS FILE)""" + def foldMode = sql("SELECT @@enable_fold_constant_by_be")[0][0] + try { + for (def fold in [false, true]) { + sql "SET enable_fold_constant_by_be = ${fold}" + quickTest("fl_get_binary_constants_${fold}", """ + SELECT ELEMENT_AT(${file}, 'uri'), ELEMENT_AT(${file}, 'offset'), + ELEMENT_AT(${file}, 'size'), ELEMENT_AT(${file}, 'content_type'), + ELEMENT_AT(${file}, 'checksum'), FROM_BINARY(FL_GET_BINARY(${file})) + """) + quickTest("fl_get_binary_nulls_${fold}", """ + SELECT FL_GET_BINARY(NULL), FL_GET_BINARY(CAST(NULL AS FILE)) + """) + for (def input in [file, "CAST(NULL AS FILE)"]) { + for (def field in ["inline", "INLINE", "InLiNe"]) { + test { + sql "SELECT ELEMENT_AT(${input}, '${field}')" + exception "use FL_GET_BINARY" + } + test { + sql "SELECT (${input})['${field}']" + exception "use FL_GET_BINARY" + } + } + } + } + } finally { + sql "SET enable_fold_constant_by_be = ${foldMode}" + } + + sql "DROP TABLE IF EXISTS test_fl_get_values" + sql """ + CREATE TABLE test_fl_get_values (id INT, uri STRING, offset_value BIGINT, + size_value BIGINT, content_type STRING, checksum STRING, payload STRING) + DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1') + """ + sql """ + INSERT INTO test_fl_get_values VALUES + (1, 's3://bucket/a.png', 2, 3, 'image/png', 'ETAG:abc', '00FF80'), + (2, NULL, NULL, NULL, NULL, NULL, NULL), + (3, 'urn:empty', 0, 0, NULL, NULL, ''), + (4, 'urn:metadata', NULL, NULL, NULL, NULL, NULL) + """ + def rows = """SELECT id, CAST(IF(id = 2, NULL, NAMED_STRUCT('uri', uri, + 'offset', offset_value, 'size', size_value, 'content_type', content_type, + 'checksum', checksum, 'inline', CAST(UNHEX(payload) AS VARBINARY))) AS FILE) AS f + FROM test_fl_get_values""" + qt_fl_get_binary_columns_1 """ + SELECT id, ELEMENT_AT(f, 'uri'), ELEMENT_AT(f, 'offset'), ELEMENT_AT(f, 'size'), + ELEMENT_AT(f, 'content_type'), ELEMENT_AT(f, 'checksum'), FROM_BINARY(FL_GET_BINARY(f)) + FROM (${rows}) t ORDER BY id + """ + qt_fl_get_binary_nested_1 """ + SELECT id, FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(ARRAY(f), 1))), + FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(MAP('asset', f), 'asset'))), + FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(NAMED_STRUCT('asset', f), 'asset'))) + FROM (${rows}) t ORDER BY id + """ + qt_fl_get_binary_aggregate_1 """ + SELECT COUNT(ELEMENT_AT(f, 'uri')), SUM(ELEMENT_AT(f, 'size')), COUNT(FL_GET_BINARY(f)) + FROM (${rows}) t + """ + qt_fl_get_binary_struct_1 "SELECT FROM_BINARY(ELEMENT_AT(NAMED_STRUCT('inline', X'00FF80'), 'inline'))" + qt_fl_get_binary_filter_1 """ + SELECT id, FROM_BINARY(FL_GET_BINARY(f)) FROM (${rows}) t WHERE id IN (1, 3) ORDER BY id + """ + for (def getter in ["fl_get_binary"]) { + testFoldConst("SELECT ${getter}(${file})") + testFoldConst("SELECT ${getter}(NULL)") + explain { + sql "SELECT ${getter}(f) FROM (${rows}) t" + contains "fl_get_binary" + } + for (def argument in ["1", "'uri'", "CAST('{}' AS JSON)", "CAST('{}' AS VARIANT)", + "NAMED_STRUCT('uri', 'urn:x')", "ARRAY(${file})"]) { + test { + sql "SELECT ${getter}(${argument})" + exception "Can not find the compatibility function signature" + } + } + test { + sql "SELECT ${getter}()" + exception "Can not found function" + } + test { + sql "SELECT ${getter}(${file}, 'uri')" + exception "Can not found function" + } + } + for (def getter in ["fl_get_uri", "fl_get_offset", "fl_get_size", "fl_get_content_type", + "fl_get_checksum", "fl_get_inline"]) { + test { + sql "SELECT ${getter}(${file})" + exception "Can not found function" + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
