This is an automated email from the ASF dual-hosted git repository. HappenLee pushed a commit to branch paimon-blob-4.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit aa7cb4cc725959269e276601a05cdf16dd5a1a38 Author: happenlee <[email protected]> AuthorDate: Fri Oct 23 11:34:33 2026 +0800 [fix](file) Adapt updated FILE functions to the Paimon Blob branch ### What problem does this PR solve? Related PR: #68853, #69040 Problem Summary: Updated FILE functions restrict binary access to FL_GET_BINARY and expose LIST_FILES as path, size and nullable file columns. Adapt the existing native and JNI Paimon Blob regressions and listing fixtures to these interfaces. Preserve branch-4.1 admission of ordinary STRUCT IN predicates, retain the branch object-storage interfaces, and verify that binary consumers request the full FILE value alongside metadata projections. Cover native S3 and Azure directory prefixes [...] ### Release note Synchronize FILE function behavior with the updated upstream feature while preserving existing Paimon Blob readers and ordinary STRUCT IN behavior on branch-4.1. ### Check List (For Author) - Test: 75 distinct focused FE unit tests passed across seven suites; final rerun of FileTypeExpressionTest and FileAccessPathTest passed all 41 tests. FE Checkstyle, clang-format 16 scripts and git diff --check passed. BE build was blocked by unavailable local Paimon Rust dependencies; BE unit tests and end-to-end regressions were not run. - Behavior changed: Yes, FILE binary access uses FL_GET_BINARY; LIST_FILES exposes path, size and nullable file columns. - Does this need documentation: No, existing upstream FILE documentation covers the synchronized interfaces. --- .../common/data_gen_functions/vlist_file_tvf.cpp | 2 +- be/src/exprs/function/cast/function_cast_file.cpp | 1 - .../data_gen_functions/vlist_file_tvf_test.cpp | 58 ++++++++--- be/test/io/fs/azure_obj_storage_client_test.cpp | 5 + .../nereids/rules/rewrite/FileAccessPathTest.java | 12 ++- .../nereids/types/FileTypeExpressionTest.java | 5 +- gensrc/thrift/Exprs.thrift | 1 + .../complex_types/test_file_type.groovy | 2 +- .../paimon/test_paimon_blob_jni.groovy | 116 ++++++++++----------- .../paimon/test_paimon_blob_native.groovy | 116 ++++++++++----------- 10 files changed, 181 insertions(+), 137 deletions(-) diff --git a/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp b/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp index 21a0d992344..c0d1963903c 100644 --- a/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp +++ b/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp @@ -82,7 +82,7 @@ Status VListFileTVF::_fetch_next_page(RuntimeState* state) { } _objects = std::move(page.objects); for (auto& prefix : page.common_prefixes) { - _objects.emplace_back(ObjectMeta {.key = std::move(prefix)}); + _objects.emplace_back(io::ObjectMeta {.key = std::move(prefix)}); } _next_index = 0; _continuation_token = std::move(page.continuation_token); diff --git a/be/src/exprs/function/cast/function_cast_file.cpp b/be/src/exprs/function/cast/function_cast_file.cpp index c05f95ec9fe..9a859865e62 100644 --- a/be/src/exprs/function/cast/function_cast_file.cpp +++ b/be/src/exprs/function/cast/function_cast_file.cpp @@ -23,7 +23,6 @@ #include "core/data_type/data_type_file.h" #include "core/data_type/data_type_struct.h" #include "core/value/file_value.h" -#include "exprs/function/cast/cast_to_variant.h" #include "exprs/function/cast/cast_wrapper_decls.h" namespace doris::CastWrapper { diff --git a/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp b/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp index 4d46b2318a6..39eb96fc6eb 100644 --- a/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp +++ b/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp @@ -30,6 +30,7 @@ #include "core/column/column_file.h" #include "core/data_type/data_type_file.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/operator/datagen_operator.h" @@ -83,9 +84,10 @@ protected: void SetUp() override { descriptor = std::make_unique<MockRowDescriptor>( std::vector<DataTypePtr> {std::make_shared<DataTypeString>(), + std::make_shared<DataTypeInt64>(), make_nullable(std::make_shared<DataTypeFile>())}, &pool); - const std::array<std::string, 2> names {"path", "file"}; + const std::array<std::string, 3> names {"path", "size", "file"}; for (size_t i = 0; i < names.size(); ++i) { descriptor->tuple_desc_map[0]->slots()[i]->_col_name = names[i]; } @@ -101,7 +103,7 @@ protected: } File row(const Block& block, size_t index) { - return (*block.get_by_position(1).column)[index].get<TYPE_FILE>(); + return (*block.get_by_position(2).column)[index].get<TYPE_FILE>(); } void check_projection(const std::vector<size_t>& positions) { @@ -132,6 +134,9 @@ protected: EXPECT_EQ(value.get<TYPE_STRING>(), "s3://bucket/dir/a.txt"); break; case 1: + EXPECT_EQ(value.get<TYPE_BIGINT>(), 7); + break; + case 2: EXPECT_EQ(value.get<TYPE_FILE>()[0].get<TYPE_STRING>(), "s3://bucket/dir/a.txt"); EXPECT_EQ(value.get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 7); break; @@ -207,7 +212,7 @@ TEST_F(ListFileTVFTest, EmitsCompleteFilesAndSkipsDirectoryMarkers) { ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); ASSERT_EQ(block.rows(), 3); EXPECT_TRUE(eos); - EXPECT_TRUE(block.get_by_position(1).type->is_nullable()); + EXPECT_TRUE(block.get_by_position(2).type->is_nullable()); EXPECT_EQ((*block.get_by_position(0).column)[0].get<TYPE_STRING>(), "s3://bucket/dir/empty.txt"); const auto empty = row(block, 0); @@ -221,6 +226,27 @@ TEST_F(ListFileTVFTest, EmitsCompleteFilesAndSkipsDirectoryMarkers) { EXPECT_EQ(row(block, 2)[3].get<TYPE_STRING>(), "application/octet-stream"); } +TEST_F(ListFileTVFTest, NonRecursiveListingIncludesNativeDirectoryPrefixes) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetDelimiter(), "/"); + auto result = page({{"dir/a.txt", 7}}); + CommonPrefix prefix; + prefix.SetPrefix("dir/sub/"); + result.AddCommonPrefixes(prefix); + return ListObjectsV2Outcome(result); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(row(block, 0)[2].get<TYPE_BIGINT>(), 7); + EXPECT_EQ((*block.get_by_position(0).column)[1].get<TYPE_STRING>(), "s3://bucket/dir/sub/"); + EXPECT_EQ((*block.get_by_position(1).column)[1].get<TYPE_BIGINT>(), 0); + EXPECT_TRUE(block.get_by_position(2).column->is_null_at(1)); + EXPECT_TRUE(eos); +} + TEST_F(ListFileTVFTest, EmitsListingEtagsWithoutHeadRequests) { ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto&) { @@ -282,7 +308,7 @@ TEST_F(ListFileTVFTest, SplitsPagesIntoBatchesWithoutPrefetching) { ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); ASSERT_EQ(block.rows(), 2); EXPECT_FALSE(eos); - auto previous = block.get_by_position(1).column; + auto previous = block.get_by_position(2).column; block.clear_column_data(); ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); ASSERT_EQ(block.rows(), 1); @@ -329,11 +355,11 @@ TEST_F(ListFileTVFTest, ReusesOutputColumnsAcrossPages) { ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); ASSERT_FALSE(eos); const auto* path_column = block.get_by_position(0).column.get(); - const auto* file_column = block.get_by_position(1).column.get(); + const auto* file_column = block.get_by_position(2).column.get(); block.clear_column_data(); ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); EXPECT_EQ(block.get_by_position(0).column.get(), path_column); - EXPECT_EQ(block.get_by_position(1).column.get(), file_column); + EXPECT_EQ(block.get_by_position(2).column.get(), file_column); ASSERT_EQ(block.rows(), 1); EXPECT_EQ(row(block, 0)[0].get<TYPE_STRING>(), "s3://bucket/dir/b.txt"); EXPECT_TRUE(eos); @@ -352,13 +378,13 @@ TEST_F(ListFileTVFTest, RestoresReusedColumnsAfterListingError) { bool eos = false; ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); const auto* path_column = block.get_by_position(0).column.get(); - const auto* file_column = block.get_by_position(1).column.get(); + const auto* file_column = block.get_by_position(2).column.get(); block.clear_column_data(); EXPECT_FALSE(function->get_next(&state, &block, &eos).ok()); ASSERT_NE(block.get_by_position(0).column.get(), nullptr); - ASSERT_NE(block.get_by_position(1).column.get(), nullptr); + ASSERT_NE(block.get_by_position(2).column.get(), nullptr); EXPECT_EQ(block.get_by_position(0).column.get(), path_column); - EXPECT_EQ(block.get_by_position(1).column.get(), file_column); + EXPECT_EQ(block.get_by_position(2).column.get(), file_column); EXPECT_EQ(block.rows(), 0); } @@ -482,9 +508,9 @@ TEST_F(ListFileTVFTest, EmptyListingReturnsEmptyFileColumn) { bool eos = false; ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); EXPECT_EQ(block.rows(), 0); - ASSERT_EQ(block.columns(), 2); + ASSERT_EQ(block.columns(), 3); EXPECT_EQ(block.get_by_position(0).type->get_name(), "String"); - EXPECT_EQ(block.get_by_position(1).type->get_name(), "File"); + EXPECT_EQ(block.get_by_position(2).type->get_name(), "Nullable(File)"); EXPECT_TRUE(eos); } @@ -620,7 +646,7 @@ TEST_F(ListFileTVFTest, ZeroLimitDoesNotFetchAnyPage) { bool eos = false; ASSERT_TRUE(op.get_block(&context.state, &block, &eos).ok()); EXPECT_EQ(block.rows(), 0); - EXPECT_EQ(block.columns(), 2); + EXPECT_EQ(block.columns(), 3); EXPECT_TRUE(eos); } @@ -648,12 +674,16 @@ TEST_F(ListFileTVFTest, PrunedPathProjection) { check_projection({0}); } -TEST_F(ListFileTVFTest, PrunedFileProjection) { +TEST_F(ListFileTVFTest, PrunedSizeProjection) { check_projection({1}); } +TEST_F(ListFileTVFTest, PrunedFileProjection) { + check_projection({2}); +} + TEST_F(ListFileTVFTest, ReorderedPrunedProjection) { - check_projection({1, 0}); + check_projection({2, 1, 0}); } } // namespace diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 9f2fd1c8a5e..3ec2e578bc1 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -112,6 +112,11 @@ TEST(AzureObjStorageClientListTest, NativeHierarchyListingKeepsOnlyDirectBlobsAn auto page = client->list_objects_page({.prefix = "tree/", .delimiter = delimiter}, token); ASSERT_TRUE(page.resp.ok()) << page.resp.status.msg; + if (!delimiter.empty() && token.empty()) { + EXPECT_EQ(page.common_prefixes, std::vector<std::string> {"tree/sub/"}); + } else { + EXPECT_TRUE(page.common_prefixes.empty()); + } objects.insert(objects.end(), page.objects.begin(), page.objects.end()); token = page.has_more ? page.continuation_token : std::string(); } while (!token.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java index 9c7b3bb3159..a82a79f1a72 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; @@ -42,7 +43,6 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; -import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.thrift.TAccessPathType; import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TDataAccessPath; @@ -86,6 +86,16 @@ public class FileAccessPathTest { Assertions.assertTrue(FileType.INSTANCE.getFields().stream().allMatch(StructField::isNullable)); } + @Test + public void testBinaryReadKeepsFullFileAccessAlongsideMetadataProjection() { + SlotReference file = slot("f", FileType.INSTANCE, true); + AccessPathInfo info = prune(file, new FlGetBinary(file), field(file, "size")); + Assertions.assertEquals(ImmutableList.of(data("f")), info.getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); + Assertions.assertEquals(ImmutableList.of(data("f")), + prune(file, new IsNull(new FlGetBinary(file))).getAllAccessPaths()); + } + @Test public void testParentAndChildNullPaths() { SlotReference file = slot("f", FileType.INSTANCE, true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java index 9781fb512e1..15971818655 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java @@ -466,13 +466,12 @@ public class FileTypeExpressionTest { } @Test - public void testInPredicateRejectsOrdinaryStruct() { + public void testInPredicatePreservesOrdinaryStructAdmission() { for (String sqlType : Arrays.asList("STRUCT<n:BIGINT>", "STRUCT<m:MAP<STRING,INT>>", "STRUCT<j:JSON>", "STRUCT<a:ARRAY<JSON>>")) { Expression value = new SlotReference("s", type(sqlType)); InPredicate predicate = new InPredicate(value, Collections.singletonList(value)); - assertAnalysisError("in predicate could not contains complex type", - () -> TypeCoercionUtils.processInPredicate(predicate), sqlType); + Assertions.assertSame(predicate, TypeCoercionUtils.processInPredicate(predicate), sqlType); } } diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index 812a65740a7..d0a99a955be 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -92,6 +92,7 @@ enum TExprNodeType { PREDICATE = 43, // Normal literal LITERAL = 44, + // 46 is reserved for the removed FILE_LITERAL; never reuse it. } //enum TAggregationOp { diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy index 92ef3652604..13ac9e90c7a 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy @@ -95,7 +95,7 @@ suite("test_file_type") { sql "SET short_circuit_evaluation = ${shortCircuit}" def selected = sql """ SELECT number, ELEMENT_AT(chosen, 'uri'), - FROM_BINARY(ELEMENT_AT(chosen, 'inline')), CAST(chosen IS NULL AS INT) + FROM_BINARY(fl_get_binary(chosen)), CAST(chosen IS NULL AS INT) FROM ( SELECT number, CASE WHEN number % 3 = 0 THEN f diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy index e9c37a9cbda..d37cb27a744 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy @@ -125,25 +125,25 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker // Inline output: NULL, empty, binary and UTF-8 payloads, with metadata children left NULL. qt_basic_lengths """ select id, case_name, image is null, - length(element_at(image, 'inline')), - md5(cast(element_at(image, 'inline') as string)) + length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)) from blob_basic order by id """ qt_basic_bytes """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_basic where id in (3, 4, 5, 6, 7, 8, 16) order by id """ qt_text_payloads """ - select id, cast(element_at(image, 'inline') as string) + select id, cast(fl_get_binary(image) as string) from blob_basic where id in (9, 10, 14, 15, 17) order by id """ // Full-payload checks use independent Spark-written ordinary columns as the oracle. // Inline output must leave every metadata child NULL, including size. qt_basic_integrity """ select count(*), sum(image is null), - sum(image is not null and length(element_at(image, 'inline')) = 0), - sum(not (length(element_at(image, 'inline')) <=> expected_size)), - sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), + sum(image is not null and length(fl_get_binary(image)) = 0), + sum(not (length(fl_get_binary(image)) <=> expected_size)), + sum(not (hex(cast(fl_get_binary(image) as string)) <=> expected_hex)), sum(element_at(image, 'uri') is not null or element_at(image, 'offset') is not null or element_at(image, 'size') is not null @@ -153,76 +153,76 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker """ qt_blob_predicate """ select id from blob_basic - where cast(element_at(image, 'inline') as string) = 'hello world' order by id + where cast(fl_get_binary(image) as string) = 'hello world' order by id """ qt_and_predicate """ select id from blob_basic - where id > 10 and cast(element_at(image, 'inline') as string) = 'hello world' order by id + where id > 10 and cast(fl_get_binary(image) as string) = 'hello world' order by id """ qt_or_predicate """ - select id, hex(cast(element_at(image, 'inline') as string)) from blob_basic - where id = 1 or cast(element_at(image, 'inline') as string) = 'hello world' order by id + select id, hex(cast(fl_get_binary(image) as string)) from blob_basic + where id = 1 or cast(fl_get_binary(image) as string) = 'hello world' order by id """ qt_empty_count "select count(*) from blob_empty" qt_empty_projection """ - select id, length(element_at(image, 'inline')) from blob_empty order by id + select id, length(fl_get_binary(image)) from blob_empty order by id """ qt_all_null """ - select id, image is null, element_at(image, 'inline') is null + select id, image is null, fl_get_binary(image) is null from blob_all_null order by id """ // Two BLOB columns: independent values, projection order and filters on the other column. qt_multi_columns """ - select id, note, hex(cast(element_at(thumbnail, 'inline') as string)), - length(element_at(image, 'inline')), - md5(cast(element_at(image, 'inline') as string)) + select id, note, hex(cast(fl_get_binary(thumbnail) as string)), + length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)) from blob_multi_columns order by id """ qt_multi_integrity """ select count(*), - sum(not (length(element_at(image, 'inline')) <=> expected_size) - or not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), - sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size) - or not (hex(cast(element_at(thumbnail, 'inline') as string)) + sum(not (length(fl_get_binary(image)) <=> expected_size) + or not (hex(cast(fl_get_binary(image) as string)) <=> expected_hex)), + sum(not (length(fl_get_binary(thumbnail)) <=> expected_thumbnail_size) + or not (hex(cast(fl_get_binary(thumbnail) as string)) <=> expected_thumbnail_hex)) from blob_multi_columns """ // Filter on one BLOB but output only the other, including NULL and empty values. qt_only_thumbnail """ - select id, hex(cast(element_at(thumbnail, 'inline') as string)) + select id, hex(cast(fl_get_binary(thumbnail) as string)) from blob_multi_columns - where image is null or length(element_at(image, 'inline')) = 0 order by id + where image is null or length(fl_get_binary(image)) = 0 order by id """ // Partition pruning, later snapshots, compaction and schema changes must preserve row mapping. qt_partition_summary """ select region, count(*), sum(id), - sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)) + sum(not (hex(cast(fl_get_binary(image) as string)) <=> expected_hex)) from blob_partitioned group by region order by region nulls first """ qt_partition_filter """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_partitioned where region = 'b' and id in (7, 9, 12) order by id """ qt_null_partition """ - select id, length(element_at(image, 'inline')) + select id, length(fl_get_binary(image)) from blob_partitioned where region is null order by id """ qt_multi_snapshot """ - select id, case_name, hex(cast(element_at(image, 'inline') as string)) + select id, case_name, hex(cast(fl_get_binary(image) as string)) from blob_multi_snapshot order by id, case_name """ qt_compacted """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_compacted order by id """ qt_schema_evolution """ - select id, label, extra, length(element_at(image, 'inline')), - hex(cast(element_at(image, 'inline') as string)) + select id, label, extra, length(fl_get_binary(image)), + hex(cast(fl_get_binary(image) as string)) from blob_schema_evolution order by id """ qt_orc_fallback """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_orc_fallback order by id """ @@ -230,18 +230,18 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker // MD5/HEX samples below check bytes; descriptor cases check rollover across all files. qt_rollover_integrity """ select count(*) as row_count, sum(id) as id_sum, - sum(not (length(element_at(image, 'inline')) <=> expected_size)) + sum(not (length(fl_get_binary(image)) <=> expected_size)) as invalid_image_length_count, - sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size)) + sum(not (length(fl_get_binary(thumbnail)) <=> expected_thumbnail_size)) as invalid_thumbnail_length_count from blob_rollover where id between 0 and 1024 """ qt_rollover_sample """ - select id, length(element_at(image, 'inline')), - md5(cast(element_at(image, 'inline') as string)), - length(element_at(thumbnail, 'inline')), - hex(cast(element_at(thumbnail, 'inline') as string)) + select id, length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)), + length(fl_get_binary(thumbnail)), + hex(cast(fl_get_binary(thumbnail) as string)) from blob_rollover where id in (0, 1, 2, 16, 17, 18, 999, 1000, 1001, 1023, 1024, 1025, 1999, 2000, 2047, 2048, 2049, 2999, 3000, 3071, 3072, @@ -250,17 +250,17 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker """ // Explicit sparse IDs let Paimon apply the filter; keep descending order and LIMIT coverage. qt_rollover_sparse """ - select id, md5(cast(element_at(image, 'inline') as string)), - md5(cast(element_at(thumbnail, 'inline') as string)) + select id, md5(cast(fl_get_binary(image) as string)), + md5(cast(fl_get_binary(thumbnail) as string)) from blob_rollover where id in (0, 997, 1994, 2991, 3988) order by id desc limit 3 """ qt_rollover_cross_predicate """ - select id, hex(cast(element_at(thumbnail, 'inline') as string)) + select id, hex(cast(fl_get_binary(thumbnail) as string)) from blob_rollover where id in (16, 17, 18, 999, 1000, 1001) - and cast(element_at(image, 'inline') as string) like 'row-%' + and cast(fl_get_binary(image) as string) like 'row-%' order by id """ @@ -270,7 +270,7 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker case when image is null then null else right(element_at(image, 'uri'), 5) = '.blob' and element_at(image, 'offset') >= 4 end, - element_at(image, 'size'), element_at(image, 'inline') is null, + element_at(image, 'size'), fl_get_binary(image) is null, element_at(image, 'content_type') is null, element_at(image, 'checksum') is null from blob_basic@options('blob-as-descriptor'='true') where id in (1, 2, 3, 7, 8, 9, 10, 13, 33, 34, 35, 36, 37, 38, 39) @@ -281,7 +281,7 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker sum(not (element_at(image, 'size') <=> expected_size) or not ((element_at(image, 'uri') is null) <=> (expected_size is null)) or not ((element_at(image, 'offset') is null) <=> (expected_size is null)) - or element_at(image, 'inline') is not null + or fl_get_binary(image) is not null or element_at(image, 'content_type') is not null or element_at(image, 'checksum') is not null) from blob_basic@options('blob-as-descriptor'='true') @@ -295,7 +295,7 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker element_at(image, 'offset') as payload_offset, element_at(image, 'size') as payload_size, expected_size as expected_payload_size, - element_at(image, 'inline') is not null as has_inline, + fl_get_binary(image) is not null as has_inline, element_at(image, 'content_type') is not null as has_content_type, element_at(image, 'checksum') is not null as has_checksum from blob_rollover@options('blob-as-descriptor'='true') @@ -303,7 +303,7 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker select 'thumbnail', thumbnail is null, element_at(thumbnail, 'uri'), element_at(thumbnail, 'offset'), element_at(thumbnail, 'size'), expected_thumbnail_size, - element_at(thumbnail, 'inline') is not null, + fl_get_binary(thumbnail) is not null, element_at(thumbnail, 'content_type') is not null, element_at(thumbnail, 'checksum') is not null from blob_rollover@options('blob-as-descriptor'='true') @@ -362,62 +362,62 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker // The SDK reader's internal descriptor option must not change the user's output // mode or poison the cached table when queries alternate between representations. qt_table_default_descriptor """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_descriptor_default order by id """ qt_table_override_inline """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_descriptor_default@options('blob-as-descriptor'='false') order by id """ qt_table_default_again """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_descriptor_default order by id """ // Small payloads exercise the limit: do not allocate 10 MiB just to test an error. sql "set paimon_blob_max_size = 8" qt_limit_allowed """ - select id, length(element_at(image, 'inline')), - hex(cast(element_at(image, 'inline') as string)) + select id, length(fl_get_binary(image)), + hex(cast(fl_get_binary(image) as string)) from blob_size_limit where id in (1, 2, 11, 12) order by id """ test { - sql "select cast(element_at(image, 'inline') as string) from blob_size_limit where id = 13" + sql "select cast(fl_get_binary(image) as string) from blob_size_limit where id = 13" exception "paimon_blob_max_size=8" } test { // BLOB-dependent filters run after materialization, so they cannot bypass the limit. - sql "select id from blob_size_limit where length(element_at(image, 'inline')) <= 8" + sql "select id from blob_size_limit where length(fl_get_binary(image)) <= 8" exception "paimon_blob_max_size=8" } qt_limit_without_blob "select id from blob_size_limit where id = 13" qt_limit_count "select count(*) from blob_size_limit" qt_limit_only_thumbnail """ - select id, hex(cast(element_at(thumbnail, 'inline') as string)) + select id, hex(cast(fl_get_binary(thumbnail) as string)) from blob_multi_columns where id = 8 """ test { - sql "select cast(element_at(image, 'inline') as string) from blob_multi_columns where id = 8" + sql "select cast(fl_get_binary(image) as string) from blob_multi_columns where id = 8" exception "paimon_blob_max_size=8" } qt_limit_descriptor """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_size_limit@options('blob-as-descriptor'='true') where id = 13 """ sql "set paimon_blob_max_size = 9" qt_limit_raised """ - select id, length(element_at(image, 'inline')), - hex(cast(element_at(image, 'inline') as string)) + select id, length(fl_get_binary(image)), + hex(cast(fl_get_binary(image) as string)) from blob_size_limit where id = 13 """ sql "set paimon_blob_max_size = 1" qt_table_default_with_limit """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_descriptor_default order by id """ test { sql """ - select cast(element_at(image, 'inline') as string) + select cast(fl_get_binary(image) as string) from blob_descriptor_default@options('blob-as-descriptor'='false') where id = 10 """ exception "paimon_blob_max_size=1" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy index 258ec7f7aaf..d18fc48329d 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy @@ -125,25 +125,25 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc // Inline output: NULL, empty, binary and UTF-8 payloads, with metadata children left NULL. qt_basic_lengths """ select id, case_name, image is null, - length(element_at(image, 'inline')), - md5(cast(element_at(image, 'inline') as string)) + length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)) from blob_basic order by id """ qt_basic_bytes """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_basic where id in (3, 4, 5, 6, 7, 8, 16) order by id """ qt_text_payloads """ - select id, cast(element_at(image, 'inline') as string) + select id, cast(fl_get_binary(image) as string) from blob_basic where id in (9, 10, 14, 15, 17) order by id """ // Full-payload checks use independent Spark-written ordinary columns as the oracle. // Inline output must leave every metadata child NULL, including size. qt_basic_integrity """ select count(*), sum(image is null), - sum(image is not null and length(element_at(image, 'inline')) = 0), - sum(not (length(element_at(image, 'inline')) <=> expected_size)), - sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), + sum(image is not null and length(fl_get_binary(image)) = 0), + sum(not (length(fl_get_binary(image)) <=> expected_size)), + sum(not (hex(cast(fl_get_binary(image) as string)) <=> expected_hex)), sum(element_at(image, 'uri') is not null or element_at(image, 'offset') is not null or element_at(image, 'size') is not null @@ -153,76 +153,76 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc """ qt_blob_predicate """ select id from blob_basic - where cast(element_at(image, 'inline') as string) = 'hello world' order by id + where cast(fl_get_binary(image) as string) = 'hello world' order by id """ qt_and_predicate """ select id from blob_basic - where id > 10 and cast(element_at(image, 'inline') as string) = 'hello world' order by id + where id > 10 and cast(fl_get_binary(image) as string) = 'hello world' order by id """ qt_or_predicate """ - select id, hex(cast(element_at(image, 'inline') as string)) from blob_basic - where id = 1 or cast(element_at(image, 'inline') as string) = 'hello world' order by id + select id, hex(cast(fl_get_binary(image) as string)) from blob_basic + where id = 1 or cast(fl_get_binary(image) as string) = 'hello world' order by id """ qt_empty_count "select count(*) from blob_empty" qt_empty_projection """ - select id, length(element_at(image, 'inline')) from blob_empty order by id + select id, length(fl_get_binary(image)) from blob_empty order by id """ qt_all_null """ - select id, image is null, element_at(image, 'inline') is null + select id, image is null, fl_get_binary(image) is null from blob_all_null order by id """ // Two BLOB columns: independent values, projection order and filters on the other column. qt_multi_columns """ - select id, note, hex(cast(element_at(thumbnail, 'inline') as string)), - length(element_at(image, 'inline')), - md5(cast(element_at(image, 'inline') as string)) + select id, note, hex(cast(fl_get_binary(thumbnail) as string)), + length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)) from blob_multi_columns order by id """ qt_multi_integrity """ select count(*), - sum(not (length(element_at(image, 'inline')) <=> expected_size) - or not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), - sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size) - or not (hex(cast(element_at(thumbnail, 'inline') as string)) + sum(not (length(fl_get_binary(image)) <=> expected_size) + or not (hex(cast(fl_get_binary(image) as string)) <=> expected_hex)), + sum(not (length(fl_get_binary(thumbnail)) <=> expected_thumbnail_size) + or not (hex(cast(fl_get_binary(thumbnail) as string)) <=> expected_thumbnail_hex)) from blob_multi_columns """ // Filter on one BLOB but output only the other, including NULL and empty values. qt_only_thumbnail """ - select id, hex(cast(element_at(thumbnail, 'inline') as string)) + select id, hex(cast(fl_get_binary(thumbnail) as string)) from blob_multi_columns - where image is null or length(element_at(image, 'inline')) = 0 order by id + where image is null or length(fl_get_binary(image)) = 0 order by id """ // Partition pruning, later snapshots, compaction and schema changes must preserve row mapping. qt_partition_summary """ select region, count(*), sum(id), - sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)) + sum(not (hex(cast(fl_get_binary(image) as string)) <=> expected_hex)) from blob_partitioned group by region order by region nulls first """ qt_partition_filter """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_partitioned where region = 'b' and id in (7, 9, 12) order by id """ qt_null_partition """ - select id, length(element_at(image, 'inline')) + select id, length(fl_get_binary(image)) from blob_partitioned where region is null order by id """ qt_multi_snapshot """ - select id, case_name, hex(cast(element_at(image, 'inline') as string)) + select id, case_name, hex(cast(fl_get_binary(image) as string)) from blob_multi_snapshot order by id, case_name """ qt_compacted """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_compacted order by id """ qt_schema_evolution """ - select id, label, extra, length(element_at(image, 'inline')), - hex(cast(element_at(image, 'inline') as string)) + select id, label, extra, length(fl_get_binary(image)), + hex(cast(fl_get_binary(image) as string)) from blob_schema_evolution order by id """ qt_orc_fallback """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_orc_fallback order by id """ @@ -230,18 +230,18 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc // MD5/HEX samples below check bytes; descriptor cases check rollover across all files. qt_rollover_integrity """ select count(*) as row_count, sum(id) as id_sum, - sum(not (length(element_at(image, 'inline')) <=> expected_size)) + sum(not (length(fl_get_binary(image)) <=> expected_size)) as invalid_image_length_count, - sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size)) + sum(not (length(fl_get_binary(thumbnail)) <=> expected_thumbnail_size)) as invalid_thumbnail_length_count from blob_rollover where id between 0 and 1024 """ qt_rollover_sample """ - select id, length(element_at(image, 'inline')), - md5(cast(element_at(image, 'inline') as string)), - length(element_at(thumbnail, 'inline')), - hex(cast(element_at(thumbnail, 'inline') as string)) + select id, length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)), + length(fl_get_binary(thumbnail)), + hex(cast(fl_get_binary(thumbnail) as string)) from blob_rollover where id in (0, 1, 2, 16, 17, 18, 999, 1000, 1001, 1023, 1024, 1025, 1999, 2000, 2047, 2048, 2049, 2999, 3000, 3071, 3072, @@ -250,17 +250,17 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc """ // Explicit sparse IDs let Paimon apply the filter; keep descending order and LIMIT coverage. qt_rollover_sparse """ - select id, md5(cast(element_at(image, 'inline') as string)), - md5(cast(element_at(thumbnail, 'inline') as string)) + select id, md5(cast(fl_get_binary(image) as string)), + md5(cast(fl_get_binary(thumbnail) as string)) from blob_rollover where id in (0, 997, 1994, 2991, 3988) order by id desc limit 3 """ qt_rollover_cross_predicate """ - select id, hex(cast(element_at(thumbnail, 'inline') as string)) + select id, hex(cast(fl_get_binary(thumbnail) as string)) from blob_rollover where id in (16, 17, 18, 999, 1000, 1001) - and cast(element_at(image, 'inline') as string) like 'row-%' + and cast(fl_get_binary(image) as string) like 'row-%' order by id """ @@ -270,7 +270,7 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc case when image is null then null else right(element_at(image, 'uri'), 5) = '.blob' and element_at(image, 'offset') >= 4 end, - element_at(image, 'size'), element_at(image, 'inline') is null, + element_at(image, 'size'), fl_get_binary(image) is null, element_at(image, 'content_type') is null, element_at(image, 'checksum') is null from blob_basic@options('blob-as-descriptor'='true') where id in (1, 2, 3, 7, 8, 9, 10, 13, 33, 34, 35, 36, 37, 38, 39) @@ -281,7 +281,7 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc sum(not (element_at(image, 'size') <=> expected_size) or not ((element_at(image, 'uri') is null) <=> (expected_size is null)) or not ((element_at(image, 'offset') is null) <=> (expected_size is null)) - or element_at(image, 'inline') is not null + or fl_get_binary(image) is not null or element_at(image, 'content_type') is not null or element_at(image, 'checksum') is not null) from blob_basic@options('blob-as-descriptor'='true') @@ -295,7 +295,7 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc element_at(image, 'offset') as payload_offset, element_at(image, 'size') as payload_size, expected_size as expected_payload_size, - element_at(image, 'inline') is not null as has_inline, + fl_get_binary(image) is not null as has_inline, element_at(image, 'content_type') is not null as has_content_type, element_at(image, 'checksum') is not null as has_checksum from blob_rollover@options('blob-as-descriptor'='true') @@ -303,7 +303,7 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc select 'thumbnail', thumbnail is null, element_at(thumbnail, 'uri'), element_at(thumbnail, 'offset'), element_at(thumbnail, 'size'), expected_thumbnail_size, - element_at(thumbnail, 'inline') is not null, + fl_get_binary(thumbnail) is not null, element_at(thumbnail, 'content_type') is not null, element_at(thumbnail, 'checksum') is not null from blob_rollover@options('blob-as-descriptor'='true') @@ -362,62 +362,62 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc // Query-level overrides must not change the table's default descriptor output // when queries alternate between representations. qt_table_default_descriptor """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_descriptor_default order by id """ qt_table_override_inline """ - select id, hex(cast(element_at(image, 'inline') as string)) + select id, hex(cast(fl_get_binary(image) as string)) from blob_descriptor_default@options('blob-as-descriptor'='false') order by id """ qt_table_default_again """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_descriptor_default order by id """ // Small payloads exercise the limit: do not allocate 10 MiB just to test an error. sql "set paimon_blob_max_size = 8" qt_limit_allowed """ - select id, length(element_at(image, 'inline')), - hex(cast(element_at(image, 'inline') as string)) + select id, length(fl_get_binary(image)), + hex(cast(fl_get_binary(image) as string)) from blob_size_limit where id in (1, 2, 11, 12) order by id """ test { - sql "select cast(element_at(image, 'inline') as string) from blob_size_limit where id = 13" + sql "select cast(fl_get_binary(image) as string) from blob_size_limit where id = 13" exception "paimon_blob_max_size=8" } test { // BLOB-dependent filters run after materialization, so they cannot bypass the limit. - sql "select id from blob_size_limit where length(element_at(image, 'inline')) <= 8" + sql "select id from blob_size_limit where length(fl_get_binary(image)) <= 8" exception "paimon_blob_max_size=8" } qt_limit_without_blob "select id from blob_size_limit where id = 13" qt_limit_count "select count(*) from blob_size_limit" qt_limit_only_thumbnail """ - select id, hex(cast(element_at(thumbnail, 'inline') as string)) + select id, hex(cast(fl_get_binary(thumbnail) as string)) from blob_multi_columns where id = 8 """ test { - sql "select cast(element_at(image, 'inline') as string) from blob_multi_columns where id = 8" + sql "select cast(fl_get_binary(image) as string) from blob_multi_columns where id = 8" exception "paimon_blob_max_size=8" } qt_limit_descriptor """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_size_limit@options('blob-as-descriptor'='true') where id = 13 """ sql "set paimon_blob_max_size = 9" qt_limit_raised """ - select id, length(element_at(image, 'inline')), - hex(cast(element_at(image, 'inline') as string)) + select id, length(fl_get_binary(image)), + hex(cast(fl_get_binary(image) as string)) from blob_size_limit where id = 13 """ sql "set paimon_blob_max_size = 1" qt_table_default_with_limit """ - select id, element_at(image, 'size'), element_at(image, 'inline') is null + select id, element_at(image, 'size'), fl_get_binary(image) is null from blob_descriptor_default order by id """ test { sql """ - select cast(element_at(image, 'inline') as string) + select cast(fl_get_binary(image) as string) from blob_descriptor_default@options('blob-as-descriptor'='false') where id = 10 """ exception "paimon_blob_max_size=1" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
