This is an automated email from the ASF dual-hosted git repository. HappenLee pushed a commit to branch paimon-blob-4.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit eda987fafcfb9566e5f60b76cb300df2c8c401eb Author: linzhenqi <[email protected]> AuthorDate: Thu Oct 22 16:38:20 2026 +0800 [improvement](function) Restrict FILE casts to STRUCT inputs Problem Summary: Support STRUCT to FILE casts, including matching nested containers, while rejecting FILE conversion to other types in CAST and TRY_CAST. Remove the BE FILE to STRUCT implementation and simplify the shared FE validation. Preserve NULL and identity casts. Reject string conversions of containers containing FILE, including function calls that require those conversions. FILE cannot be cast to STRUCT or other types. STRUCT to FILE remains supported for individual and nested values. CAST(NULL AS FILE) returns NULL. LAG and LEAD calls that require converting containers containing FILE to VARCHAR are rejected during analysis. - Test: Full ASAN BE and FE build; FILE BE unit tests with coverage; FileTypeExpressionTest, FileAccessPathTest, and FlGetBinaryTest; five FILE-related P0 suites; clang-format 16, clang-tidy, and build hygiene. - Behavior changed: Yes, FILE casts only accept STRUCT inputs apart from NULL and identity casts. - Does this need documentation: Yes, current and 4.x FILE and conversion documentation updated in the local doris-website file-type-docs branch. (cherry picked from commit 4ece2e53af63092ab8d8d2932b8a06aa18c6a00a) --- be/src/exprs/function/cast/function_cast.cpp | 3 +- be/src/exprs/function/cast/function_cast_file.cpp | 50 +------------------ be/test/exprs/file_expr_test.cpp | 58 +++++++++------------- .../nereids/rules/expression/check/CheckCast.java | 10 ++-- .../nereids/rules/rewrite/FileAccessPathTest.java | 17 ------- .../nereids/types/FileTypeExpressionTest.java | 15 ++++-- .../datatype_p0/complex_types/test_file_type.out | 18 +------ .../complex_types/test_file_type.groovy | 48 ++++++++++-------- .../complex_types/test_file_type_optimizer.groovy | 7 +-- 9 files changed, 72 insertions(+), 154 deletions(-) diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index e6881caa7c3..e0373a3826c 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -257,7 +257,8 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& origin_fro to_type->get_primitive_type() == TYPE_JSONB || from_type->get_primitive_type() == TYPE_VARIANT || to_type->get_primitive_type() == TYPE_VARIANT || - is_string_type(from_type->get_primitive_type()))) { + is_string_type(from_type->get_primitive_type()) || + is_string_type(to_type->get_primitive_type()))) { return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); } diff --git a/be/src/exprs/function/cast/function_cast_file.cpp b/be/src/exprs/function/cast/function_cast_file.cpp index 636a2b7991c..c05f95ec9fe 100644 --- a/be/src/exprs/function/cast/function_cast_file.cpp +++ b/be/src/exprs/function/cast/function_cast_file.cpp @@ -119,61 +119,13 @@ WrapperType create_cast_to_file(const DataTypePtr& from_type) { }; } -WrapperType create_cast_from_file(const DataTypePtr& from_type, const DataTypePtr& to_type) { - const auto& file_type = assert_cast<const DataTypeFile&>(*from_type); - const auto primitive = to_type->get_primitive_type(); - if (primitive == TYPE_STRUCT) { - const auto& structure = assert_cast<const DataTypeStruct&>(*to_type); - FileFieldMapping mapping {}; - auto status = map_file_fields(structure, mapping); - if (!status.ok()) { - return create_unsupport_wrapper(std::string(status.msg())); - } - DataTypes fields; - FileFieldMapping source_positions {}; - for (size_t file_index = 0; file_index < mapping.size(); ++file_index) { - source_positions[mapping[file_index]] = cast_set<int>(file_index); - } - for (size_t target_index = 0; target_index < source_positions.size(); ++target_index) { - const auto& field = structure.get_elements()[target_index]; - const auto file_index = cast_set<size_t>(source_positions[target_index]); - const auto field_primitive = remove_nullable(field)->get_primitive_type(); - const bool compatible = compatible_file_field_type(file_index, field_primitive); - if (!field->is_nullable() || !compatible) { - return create_unsupport_wrapper("FILE can only cast to its six-field STRUCT"); - } - fields.push_back(file_type.get_elements()[file_index]); - } - const auto structure_type = - std::make_shared<DataTypeStruct>(fields, structure.get_element_names()); - return [structure_type, to_type, source_positions]( - FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t rows, const NullMap::value_type* null_map) { - const auto source = - block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - const auto& file = assert_cast<const ColumnFile&>(*source); - Columns children; - for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { - children.push_back(file.get_column_ptr(source_positions[i])); - } - Block projected {{ColumnStruct::create(children), structure_type, "file_fields"}, - {nullptr, to_type, "result"}}; - RETURN_IF_ERROR(prepare_unpack_dictionaries(context, structure_type, to_type)( - context, projected, {0}, 1, rows, null_map)); - block.get_by_position(result).column = projected.get_by_position(1).column; - return Status::OK(); - }; - } - return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); -} - } // namespace WrapperType create_file_wrapper(const DataTypePtr& from_type, const DataTypePtr& to_type) { if (to_type->get_primitive_type() == TYPE_FILE) { return create_cast_to_file(from_type); } - return create_cast_from_file(from_type, to_type); + return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); } } // namespace doris::CastWrapper diff --git a/be/test/exprs/file_expr_test.cpp b/be/test/exprs/file_expr_test.cpp index 14201a6cd87..93210a73e05 100644 --- a/be/test/exprs/file_expr_test.cpp +++ b/be/test/exprs/file_expr_test.cpp @@ -181,30 +181,25 @@ protected: } }; -TEST_F(FileExprTest, InlineGetterAndSixFieldStructRoundTripPreserveBinary) { +TEST_F(FileExprTest, InlineGetterAndStructToFileCastPreserveBinary) { const std::string bytes("\0\xff\x80", 3); - auto value = file_value(); - value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); - auto empty = file_value(); - empty.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView()); - const auto input = column(make_nullable(file_type), {value, empty, file_value(), Field()}); + auto value = public_value("urn:binary"); + value.get<TYPE_STRUCT>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); + auto empty = public_value("urn:empty"); + empty.get<TYPE_STRUCT>()[5] = Field::create_field<TYPE_VARBINARY>(StringView()); const auto& schema = assert_cast<const DataTypeFile&>(*file_type); const auto structure_type = make_nullable( std::make_shared<DataTypeStruct>(schema.get_elements(), schema.get_element_names())); - ColumnPtr projected; - ASSERT_TRUE(cast(input, structure_type, true, projected).ok()); - ASSERT_EQ((*projected)[0].get<TYPE_STRUCT>().size(), 6); - EXPECT_EQ((*projected)[0].get<TYPE_STRUCT>()[5].get<TYPE_VARBINARY>().str(), bytes); + const auto source = + column(structure_type, {value, empty, public_value("urn:missing"), Field()}); ColumnPtr restored; - ASSERT_TRUE( - cast({projected, structure_type, "struct"}, make_nullable(file_type), true, restored) - .ok()); + ASSERT_TRUE(cast(source, make_nullable(file_type), true, restored).ok()); EXPECT_EQ((*restored)[0].get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().str(), bytes); EXPECT_FALSE((*restored)[1].get<TYPE_FILE>()[5].is_null()); EXPECT_EQ((*restored)[1].get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().size(), 0); EXPECT_TRUE((*restored)[2].get<TYPE_FILE>()[5].is_null()); EXPECT_TRUE(restored->is_null_at(3)); - ColumnsWithTypeAndName args {input}; + ColumnsWithTypeAndName args {{restored, make_nullable(file_type), "file"}}; auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", args, schema.get_element(5)); ASSERT_NE(function, nullptr); @@ -424,6 +419,11 @@ TEST_F(FileExprTest, CastRejectsScalarInputsAndFileToString) { ColumnPtr output; EXPECT_FALSE( cast(column(file_type, {file_value()}), make_nullable(text), strict, output).ok()); + const auto array_type = std::make_shared<DataTypeArray>(make_nullable(file_type)); + EXPECT_FALSE( + cast(column(array_type, {Field::create_field<TYPE_ARRAY>(Array {file_value()})}), + make_nullable(text), strict, output) + .ok()); } } @@ -481,19 +481,13 @@ TEST_F(FileExprTest, JsonCastsAreUnsupported) { } } -TEST_F(FileExprTest, ToJsonAndStructCastExposeExactlySixFields) { +TEST_F(FileExprTest, ToJsonExposesExactlySixFields) { const std::string bytes("\0\xff", 2); auto value = file_value("s3://b/key"); value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>( StringView(bytes.data(), cast_set<uint32_t>(bytes.size()))); const auto input = column(make_nullable(file_type), {value, Field()}); ColumnPtr output; - ASSERT_TRUE(cast(input, make_nullable(public_struct()), true, output).ok()); - EXPECT_TRUE(output->is_null_at(1)); - const auto public_value = (*output)[0]; - ASSERT_EQ(public_value.get_type(), TYPE_STRUCT); - EXPECT_EQ(public_value.get<TYPE_STRUCT>().size(), 6); - EXPECT_EQ(public_value.get<TYPE_STRUCT>()[0].get<TYPE_STRING>(), "s3://b/key"); const auto expected = R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AP8="})"; @@ -542,7 +536,7 @@ TEST_F(FileExprTest, ToJsonPreservesConstantFileAndNull) { } } -TEST_F(FileExprTest, ReverseStructMatchesPublicNamesAndRequiresNullableChildren) { +TEST_F(FileExprTest, FileToStructCastIsUnsupported) { const auto& file = assert_cast<const DataTypeFile&>(*file_type); ColumnPtr output; auto source_value = file_value("s3://b/key"); @@ -552,17 +546,13 @@ TEST_F(FileExprTest, ReverseStructMatchesPublicNamesAndRequiresNullableChildren) source_value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); const auto input = column(file_type, {source_value}); auto six = std::make_shared<DataTypeStruct>(file.get_elements(), file.get_element_names()); - EXPECT_TRUE(cast(input, six, true, output).ok()); + EXPECT_FALSE(cast(input, six, true, output).ok()); + EXPECT_FALSE(cast(input, six, false, output).ok()); auto reordered = std::make_shared<DataTypeStruct>( DataTypes {make_nullable(bigint), make_nullable(text), make_nullable(bigint), make_nullable(text), make_nullable(text), make_nullable(binary)}, Strings {"SIZE", "UrI", "offset", "content_type", "checksum", "inline"}); - ASSERT_TRUE(cast(input, reordered, true, output).ok()); - const auto value = (*output)[0]; - EXPECT_EQ(value.get<TYPE_STRUCT>()[0].get<TYPE_BIGINT>(), 12); - EXPECT_EQ(value.get<TYPE_STRUCT>()[1].get<TYPE_STRING>(), "s3://b/key"); - EXPECT_EQ(value.get<TYPE_STRUCT>()[2].get<TYPE_BIGINT>(), 3); - EXPECT_EQ(value.get<TYPE_STRUCT>()[5].get<TYPE_VARBINARY>().str(), bytes); + EXPECT_FALSE(cast(input, reordered, true, output).ok()); auto nonnullable = std::make_shared<DataTypeStruct>( DataTypes {text, make_nullable(bigint), make_nullable(bigint), make_nullable(text), make_nullable(text), make_nullable(binary)}, @@ -834,7 +824,7 @@ TEST_F(FileExprTest, StructCastSharesPayloadAndDetachesBeforeMutation) { EXPECT_EQ(source.get_column(5)[0].get<TYPE_VARBINARY>().str(), bytes); } -TEST_F(FileExprTest, ReverseCastsRecurseAndIdentityPreservesInline) { +TEST_F(FileExprTest, NestedFileToStructCastIsUnsupportedAndIdentityPreservesInline) { auto value = file_value("s3://b/key"); const std::string bytes("a\0b", 3); value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); @@ -845,11 +835,9 @@ TEST_F(FileExprTest, ReverseCastsRecurseAndIdentityPreservesInline) { ASSERT_TRUE(cast(input, input_type, true, output).ok()); EXPECT_EQ((*output)[0].get<TYPE_ARRAY>()[0].get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().size(), 3); - ASSERT_TRUE(cast(input, std::make_shared<DataTypeArray>(make_nullable(public_struct())), true, - output) - .ok()); - EXPECT_EQ((*output)[0].get<TYPE_ARRAY>()[0].get<TYPE_STRUCT>().size(), 6); - EXPECT_TRUE((*output)[0].get<TYPE_ARRAY>()[1].is_null()); + EXPECT_FALSE(cast(input, std::make_shared<DataTypeArray>(make_nullable(public_struct())), true, + output) + .ok()); } TEST_F(FileExprTest, ExplodeFilePublicFieldsNullAndOuterRows) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index cd878dd5fb6..45913af60b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -363,7 +363,7 @@ public class CheckCast implements ExpressionPatternRuleFactory { return checkFileCast(originalType, targetType); } if ((originalType.isJsonType() || targetType.isJsonType() || originalType.isStringLikeType() - || originalType.isVariantType() || targetType.isVariantType()) + || targetType.isStringLikeType() || originalType.isVariantType() || targetType.isVariantType()) && (originalType.typeContainsFile() || targetType.typeContainsFile())) { return false; } @@ -455,11 +455,10 @@ public class CheckCast implements ExpressionPatternRuleFactory { if (source.isNullType() || source.equals(target)) { return true; } - DataType publicType = source.isFileType() ? target : source; - if (!(publicType instanceof StructType)) { + if (!target.isFileType() || !(source instanceof StructType)) { return false; } - List<StructField> fields = ((StructType) publicType).getFields(); + List<StructField> fields = ((StructType) source).getFields(); StructType canonical = FileType.INSTANCE.publicStructType(); if (fields.size() != canonical.getFields().size()) { return false; @@ -469,10 +468,9 @@ public class CheckCast implements ExpressionPatternRuleFactory { StructField expected = canonical.getField(field.getName()); DataType fieldType = field.getDataType(); if (expected == null || !names.add(field.getName()) - || (source.isFileType() && !field.isNullable()) || !(fieldType.equals(expected.getDataType()) || (fieldType.isStringLikeType() && expected.getDataType().isStringLikeType()) - || (!source.isFileType() && fieldType.isNullType()))) { + || fieldType.isNullType())) { return false; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java index 51036613df5..9c7b3bb3159 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java @@ -122,12 +122,6 @@ public class FileAccessPathTest { Assertions.assertEquals(ImmutableList.of(data("f", "uri")), prune(file, new IsNull(identityField)).getAllAccessPaths()); - Expression structField = field(new Cast(file, FileType.INSTANCE.publicStructType(), true), "uri"); - for (Expression expression : ImmutableList.of(new Length(structField), new IsNull(structField))) { - AccessPathInfo info = prune(file, expression); - Assertions.assertEquals(ImmutableList.of(data("f")), info.getAllAccessPaths()); - Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); - } SlotReference struct = slot("s", FileType.INSTANCE.publicStructType(), true); Expression fileField = field(new Cast(struct, FileType.INSTANCE, true), "uri"); for (Expression expression : ImmutableList.of(new Length(fileField), new IsNull(fileField))) { @@ -206,17 +200,6 @@ public class FileAccessPathTest { SlotReference file = slot("f", FileType.INSTANCE, true); Assertions.assertEquals(ImmutableList.of(data("f", "size")), prune(file, field(new Cast(file, FileType.INSTANCE), "size")).getAllAccessPaths()); - Assertions.assertEquals(ImmutableList.of(data("f")), - prune(file, field(new Cast(file, FileType.INSTANCE.publicStructType(), true), "size")) - .getAllAccessPaths()); - for (Expression selector : ImmutableList.of(new StringLiteral("size"), new IntegerLiteral(3))) { - Expression accessed = TypeCoercionUtils.processBoundFunction(new ElementAt( - new Cast(file, FileType.INSTANCE.publicStructType(), true), selector)); - Assertions.assertTrue(accessed.child(0) instanceof Cast); - Assertions.assertEquals(ImmutableList.of(data("f")), prune(file, accessed).getAllAccessPaths()); - Assertions.assertEquals(ImmutableList.of(data("f")), - prune(file, new IsNull(accessed)).getAllAccessPaths()); - } SlotReference struct = slot("s", FileType.INSTANCE.publicStructType(), true); Assertions.assertEquals(ImmutableList.of(data("s")), prune(struct, field(new Cast(struct, FileType.INSTANCE, true), "size")).getAllAccessPaths()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java index 47466b295ea..9781fb512e1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java @@ -312,7 +312,7 @@ public class FileTypeExpressionTest { + "content_type:STRING,offset:BIGINT>")) { DataType value = type(source); Assertions.assertTrue(CheckCast.check(value, FileType.INSTANCE, strict), source); - Assertions.assertTrue(CheckCast.check(FileType.INSTANCE, value, strict), source); + Assertions.assertFalse(CheckCast.check(FileType.INSTANCE, value, strict), source); Assertions.assertTrue(TypeCoercionUtils.implicitCast(value, FileType.INSTANCE).isPresent()); Assertions.assertTrue(TypeCoercionUtils.implicitCast(FileType.INSTANCE, value).isPresent()); } @@ -327,6 +327,13 @@ public class FileTypeExpressionTest { } Assertions.assertTrue(CheckCast.check(type("ARRAY<STRUCT<" + fields + ">>"), type("ARRAY<FILE>"), strict)); + for (String wrapper : Arrays.asList("%s", "ARRAY<%s>", "MAP<STRING,%s>", "STRUCT<f:%s>", + "ARRAY<STRUCT<f:%s>>")) { + DataType source = type(String.format(wrapper, "STRUCT<" + fields + ">")); + DataType target = type(String.format(wrapper, "FILE")); + Assertions.assertTrue(CheckCast.check(source, target, strict), wrapper); + Assertions.assertFalse(CheckCast.check(target, source, strict), wrapper); + } Assertions.assertTrue(CheckCast.check(StringType.INSTANCE, type("ARRAY<STRUCT<" + fields + ">>"), strict)); Assertions.assertFalse(CheckCast.check(StringType.INSTANCE, type("ARRAY<FILE>"), strict)); } @@ -362,6 +369,8 @@ public class FileTypeExpressionTest { } Assertions.assertFalse(CheckCast.check(nested, VariantType.INSTANCE, strict), container + " -> VARIANT"); + Assertions.assertFalse(CheckCast.check(nested, StringType.INSTANCE, strict), + container + " -> STRING"); Assertions.assertTrue(CheckCast.check(NullType.INSTANCE, nested, strict), "NULL -> " + container); Assertions.assertTrue(CheckCast.check(nested, nested, strict), container); } @@ -482,7 +491,7 @@ public class FileTypeExpressionTest { } @Test - public void testElementAtExplodeAndReverseCastFields() { + public void testElementAtAndExplodeFields() { Expression file = new SlotReference("f", FileType.INSTANCE); for (StructField field : FileType.INSTANCE.getFields()) { if (field.getName().equals("inline")) { @@ -503,7 +512,7 @@ public class FileTypeExpressionTest { Assertions.assertEquals(FileType.INSTANCE.publicStructType(), TypeCoercionUtils.processBoundFunction(explode).getDataType()); } - Cast struct = new Cast(file, FileType.INSTANCE.publicStructType(), true); + SlotReference struct = new SlotReference("s", FileType.INSTANCE.publicStructType()); Expression accessed = TypeCoercionUtils.processBoundFunction(new ElementAt(struct, new StringLiteral("INLINE"))); Assertions.assertTrue(accessed instanceof ElementAt); Assertions.assertSame(struct, accessed.child(0)); diff --git a/regression-test/data/datatype_p0/complex_types/test_file_type.out b/regression-test/data/datatype_p0/complex_types/test_file_type.out index 5ab79481012..e929f2f4c7a 100644 --- a/regression-test/data/datatype_p0/complex_types/test_file_type.out +++ b/regression-test/data/datatype_p0/complex_types/test_file_type.out @@ -23,11 +23,6 @@ 2 \N \N \N \N \N \N \N \N true true false 3 urn:example:three 2 8 \N \N urn:example:three false false true --- !struct -- -1 {"uri":"S3://Bucket/a%2fb?versionId=v1", "offset":null, "size":0, "content_type":"Image/PNG", "checksum":"CRC32:00000000", "inline":0x00FF80} -2 \N -3 {"uri":"urn:example:three", "offset":2, "size":8, "content_type":null, "checksum":null, "inline":0x} - -- !count -- 2 @@ -145,14 +140,9 @@ true true -- !nested_try_cast -- \N \N \N --- !explicit_reverse -- +-- !explicit_struct_to_file -- {"uri":"urn:cast","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} {"uri":"urn:cast","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} urn:cast {"uri":"urn:cast","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} urn:cast --- !inline_struct_roundtrip -- -1 00FF80 -2 \N -3 - -- !explicit_forward -- 2 urn:struct urn:struct 3 urn:reference urn:reference @@ -172,12 +162,6 @@ true true 3 [null, null] {"asset":null} {"asset":null} \N \N \N 4 [{"uri":"urn:reference","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}, null] {"asset":{"uri":"urn:reference","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}} {"asset":{"uri":"urn:reference","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}} \N \N \N --- !inline_struct -- -1 {"uri":null, "offset":null, "size":null, "content_type":null, "checksum":null, "inline":0x00FF} -2 {"uri":null, "offset":null, "size":null, "content_type":null, "checksum":null, "inline":0x} -3 \N -4 {"uri":"urn:reference", "offset":null, "size":null, "content_type":null, "checksum":null, "inline":null} - -- !inline_explode -- 1 \N \N \N \N \N 00FF 2 \N \N \N \N \N diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy index 9949360ea1e..92ef3652604 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy @@ -63,11 +63,6 @@ suite("test_file_type") { f IS NULL, f IS NOT NULL FROM test_file_type_values ORDER BY id """) - qt_struct fileQuery(""" - SELECT id, CAST(f AS STRUCT<uri:VARCHAR(65533),offset:BIGINT,size:BIGINT, - content_type:VARCHAR(1024),checksum:VARCHAR(1024),inline:VARBINARY>) - FROM test_file_type_values ORDER BY id - """) qt_count fileQuery("SELECT COUNT(f) FROM test_file_type_values") qt_ordinary_constant_element """ SELECT ELEMENT_AT(ARRAY(10, 20), 2), ELEMENT_AT(MAP('asset', 30), 'asset') @@ -296,13 +291,13 @@ suite("test_file_type") { for (def expression in [ "ARRAY_SORTBY(files, ARRAY(2, 1))", "ARRAY_MAP(x -> ELEMENT_AT(x, 'uri'), files)", "FIRST_VALUE(lookup) OVER (ORDER BY id)", "LAST_VALUE(files) OVER (ORDER BY id)", - "NTH_VALUE(holder, 1) OVER (ORDER BY id)", - "LAG(files) OVER (ORDER BY id)", "LEAD(holder) OVER (ORDER BY id)" + "NTH_VALUE(holder, 1) OVER (ORDER BY id)" ]) { sql fileQuery("SELECT ${expression} FROM test_file_type_nested") } for (def expression in [ - "ARRAY_SORT(files)", "ARRAY_DISTINCT(files)", "ARRAY_CONTAINS(files, ELEMENT_AT(files, 1))" + "ARRAY_SORT(files)", "ARRAY_DISTINCT(files)", "ARRAY_CONTAINS(files, ELEMENT_AT(files, 1))", + "LAG(files) OVER (ORDER BY id)", "LEAD(holder) OVER (ORDER BY id)" ]) { test { sql fileQuery("SELECT ${expression} FROM test_file_type_nested") @@ -324,6 +319,23 @@ suite("test_file_type") { for (def strict in [true, false]) { sql "SET enable_strict_cast = ${strict}" for (def castFunction in ["CAST", "TRY_CAST"]) { + for (def expression in [ + "${castFunction}(f AS ${publicStruct})", + "${castFunction}(f AS STRING)", + "${castFunction}(ARRAY(f) AS STRING)", + "${castFunction}(NAMED_STRUCT('f', f) AS STRING)", + "${castFunction}(MAP('f', f) AS STRING)", + "${castFunction}(CAST(NULL AS FILE) AS ${publicStruct})", + "${castFunction}(ARRAY(f) AS ARRAY<${publicStruct}>)", + "${castFunction}(NAMED_STRUCT('f', f) AS STRUCT<f:${publicStruct}>)", + "${castFunction}(MAP('f', f) AS MAP<STRING,${publicStruct}>)", + "${castFunction}(ARRAY(NAMED_STRUCT('f', f)) AS ARRAY<STRUCT<f:${publicStruct}>>)" + ]) { + test { + sql fileQuery("SELECT ${expression} FROM test_file_type_values") + exception "FILE" + } + } for (def externalType in ["JSON", "VARIANT"]) { for (def expression in [ "${castFunction}(f AS ${externalType})", @@ -431,13 +443,14 @@ suite("test_file_type") { } // Explicit conversions remain query expressions; no FILE assignment policy is imposed. - def fileValue = "CAST(NAMED_STRUCT('uri', 'urn:cast', 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS FILE)" - def fileArrayValue = "CAST(ARRAY(CAST(${fileValue} AS ${publicStruct})) AS ARRAY<FILE>)" - qt_explicit_reverse fileQuery(""" + def structValue = "NAMED_STRUCT('uri', 'urn:cast', 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL)" + def fileValue = "CAST(${structValue} AS FILE)" + def fileArrayValue = "CAST(ARRAY(${structValue}) AS ARRAY<FILE>)" + qt_explicit_struct_to_file fileQuery(""" SELECT CAST(TO_JSON(${fileValue}) AS STRING), TO_JSON(${fileValue}), - ELEMENT_AT(CAST(${fileValue} AS ${publicStruct}), 'uri'), + ELEMENT_AT(${fileValue}, 'uri'), CAST(TO_JSON(ELEMENT_AT(${fileArrayValue}, 1)) AS STRING), - ELEMENT_AT(ELEMENT_AT(CAST(${fileArrayValue} AS ARRAY<${publicStruct}>), 1), 'uri') + ELEMENT_AT(ELEMENT_AT(${fileArrayValue}, 1), 'uri') """) for (def query in [ "SELECT CAST(f AS STRING) FROM test_file_type_values", @@ -450,15 +463,9 @@ suite("test_file_type") { exception "FILE" } } - qt_inline_struct_roundtrip fileQuery(""" - SELECT id, FROM_BINARY(FL_GET_BINARY(CAST(CAST(f AS - STRUCT<inline:VARBINARY,checksum:STRING,content_type:STRING,size:BIGINT,offset:BIGINT,uri:STRING>) - AS FILE))) - FROM test_file_type_values ORDER BY id - """) qt_explicit_forward fileQuery(""" SELECT id, ELEMENT_AT(CAST(j AS FILE), 'uri'), - ELEMENT_AT(ELEMENT_AT(CAST(ARRAY(CAST(CAST(j AS FILE) AS ${publicStruct})) AS ARRAY<FILE>), 1), 'uri') + ELEMENT_AT(ELEMENT_AT(CAST(ARRAY(j) AS ARRAY<FILE>), 1), 'uri') FROM ( SELECT 2 AS id, NAMED_STRUCT('uri', 'urn:struct', 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS j @@ -512,7 +519,6 @@ def evaluate(value): FROM_BINARY(FL_GET_BINARY(ELEMENT_AT(MAP('asset', f), 'asset'))) FROM test_file_type_inline_values ORDER BY id """) - qt_inline_struct fileQuery("SELECT id, CAST(f AS ${publicStruct}) FROM test_file_type_inline_values ORDER BY id") qt_inline_explode fileQuery(""" SELECT id, uri, off, size, mime, checksum, FROM_BINARY(inline_bytes) FROM test_file_type_inline_values diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy index 17a8c0b3375..47573899b08 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy @@ -16,17 +16,14 @@ // under the License. suite("test_file_type_optimizer") { - def publicStruct = "STRUCT<uri:VARCHAR(65533),offset:BIGINT,size:BIGINT," + - "content_type:VARCHAR(1024),checksum:VARCHAR(1024),inline:VARBINARY>" def fileCtes = """ WITH file_values AS ( SELECT number AS id, CAST(NAMED_STRUCT('uri', CONCAT('urn:optimizer:', number), 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS FILE) AS f FROM numbers("number"="3") ), file_scan AS ( - SELECT id, f, CAST(ARRAY(CAST(f AS ${publicStruct})) AS ARRAY<FILE>) AS a, - CAST(NAMED_STRUCT('f', CAST(f AS ${publicStruct})) AS STRUCT<f:FILE>) AS s, - CAST(MAP('key', CAST(f AS ${publicStruct})) AS MAP<STRING,FILE>) AS m + SELECT id, f, ARRAY(f) AS a, + NAMED_STRUCT('f', f) AS s, MAP('key', f) AS m FROM file_values ) """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
