This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch paimon-blob-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/paimon-blob-4.1 by this push:
new c7317f0cb17 paimon-blob-4.1: update blob large payload (#69517)
c7317f0cb17 is described below
commit c7317f0cb176559b280f6de9534088aa5ae3f761
Author: zhangstar333 <[email protected]>
AuthorDate: Fri Oct 23 16:21:35 2026 +0800
paimon-blob-4.1: update blob large payload (#69517)
update blob large payload
---
.../create_preinstalled_scripts/paimon/run15.sql | 22 +++++++
.../paimon/test_paimon_blob_jni.out | 27 +++++++++
.../paimon/test_paimon_blob_native.out | 27 +++++++++
.../paimon/test_paimon_blob_jni.groovy | 68 +++++++++++++++++++++-
.../paimon/test_paimon_blob_native.groovy | 68 +++++++++++++++++++++-
5 files changed, 208 insertions(+), 4 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql
index 2437a454024..b360e5fe13f 100644
---
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql
@@ -115,6 +115,28 @@ create table blob_size_limit (
insert into blob_size_limit
select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 11, 12,
13) order by id;
+-- Real payloads at and above Doris' default 10 MiB inline limit. Keep them
separate
+-- from the small fixtures so ordinary payload scans do not fail with the
default limit.
+-- Store an MD5 instead of a hex copy of each multi-megabyte payload.
+drop table if exists blob_large_payload;
+create table blob_large_payload (
+ id int, case_name string, image binary, expected_size int, expected_md5
string
+) using paimon tblproperties (
+ 'file.format' = 'parquet', 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' =
'true'
+);
+insert into blob_large_payload
+select /*+ COALESCE(1) */ id, case_name, image, expected_size, md5(image)
+from (
+ select id, case_name, encode(repeat('A', expected_size), 'UTF-8') as
image, expected_size
+ from values
+ (1, 'limit_10m_exact', 10485760),
+ (2, 'limit_10m_plus_one', 10485761),
+ (3, 'payload_11m', 11534336)
+ as seed(id, case_name, expected_size)
+) payloads
+order by id;
+
-- Two BLOB fields separated by an ordinary column. NULL maps and sizes are
independent.
create or replace temporary view paimon_blob_pair_seed as
select id, image, note, thumbnail,
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out
b/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out
index b631991409d..75d217106fb 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out
@@ -5,6 +5,7 @@ blob_basic
blob_compacted
blob_descriptor_default
blob_empty
+blob_large_payload
blob_multi_columns
blob_multi_snapshot
blob_orc_fallback
@@ -246,6 +247,32 @@ thumbnail 3781 0
2 0 true
10 11 true
+-- !large_default_exact --
+1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true
+
+-- !large_default_descriptor --
+1 10485760 true true true true
+2 10485761 true true true true
+3 11534336 true true true true
+
+-- !large_raised_content --
+1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true
+2 10485761 126c1549a4e98d4e0254c84619963624 true true
+3 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b true true
+
+-- !large_metadata_only --
+1 limit_10m_exact 10485760 8b8378787c0925f42ccb829f6cc2fb97
+2 limit_10m_plus_one 10485761 126c1549a4e98d4e0254c84619963624
+3 payload_11m 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b
+
+-- !large_count --
+3
+
+-- !large_descriptor_with_low_limit --
+1 10485760 true true true true
+2 10485761 true true true true
+3 11534336 true true true true
+
-- !limit_allowed --
1 \N \N
2 0
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out
b/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out
index b631991409d..75d217106fb 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out
@@ -5,6 +5,7 @@ blob_basic
blob_compacted
blob_descriptor_default
blob_empty
+blob_large_payload
blob_multi_columns
blob_multi_snapshot
blob_orc_fallback
@@ -246,6 +247,32 @@ thumbnail 3781 0
2 0 true
10 11 true
+-- !large_default_exact --
+1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true
+
+-- !large_default_descriptor --
+1 10485760 true true true true
+2 10485761 true true true true
+3 11534336 true true true true
+
+-- !large_raised_content --
+1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true
+2 10485761 126c1549a4e98d4e0254c84619963624 true true
+3 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b true true
+
+-- !large_metadata_only --
+1 limit_10m_exact 10485760 8b8378787c0925f42ccb829f6cc2fb97
+2 limit_10m_plus_one 10485761 126c1549a4e98d4e0254c84619963624
+3 payload_11m 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b
+
+-- !large_count --
+3
+
+-- !large_descriptor_with_low_limit --
+1 10485760 true true true true
+2 10485761 true true true true
+3 11534336 true true true true
+
-- !limit_allowed --
1 \N \N
2 0
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy
index d37cb27a744..60624e4d96f 100644
---
a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy
@@ -51,7 +51,7 @@ suite("test_paimon_blob_jni",
"p0,external,doris,external_docker,external_docker
sql "set force_jni_scanner = true"
sql "set paimon_blob_max_size = 10485760"
- ["blob_basic", "blob_all_null", "blob_size_limit",
"blob_multi_columns",
+ ["blob_basic", "blob_all_null", "blob_size_limit",
"blob_large_payload", "blob_multi_columns",
"blob_partitioned", "blob_multi_snapshot", "blob_rollover",
"blob_schema_evolution",
"blob_compacted", "blob_descriptor_default"].each { table ->
explain {
@@ -374,7 +374,71 @@ suite("test_paimon_blob_jni",
"p0,external,doris,external_docker,external_docker
from blob_descriptor_default order by id
"""
- // Small payloads exercise the limit: do not allocate 10 MiB just to
test an error.
+ // Real payloads exercise the default 10 MiB boundary without
returning large values.
+ qt_large_default_exact """
+ select id, length(fl_get_binary(image)),
+ md5(cast(fl_get_binary(image) as string)),
+ length(fl_get_binary(image)) <=> expected_size,
+ md5(cast(fl_get_binary(image) as string)) <=> expected_md5
+ from blob_large_payload where id = 1
+ """
+ test {
+ sql "select length(fl_get_binary(image)) from blob_large_payload
where id = 2"
+ exception "paimon_blob_max_size=10485760"
+ }
+ test {
+ sql "select length(fl_get_binary(image)) from blob_large_payload
where id = 3"
+ exception "paimon_blob_max_size=10485760"
+ }
+ test {
+ sql """
+ select id from blob_large_payload
+ where length(fl_get_binary(image)) <= 10485760
+ """
+ exception "paimon_blob_max_size=10485760"
+ }
+ qt_large_default_descriptor """
+ select id, element_at(image, 'size'),
+ element_at(image, 'size') <=> expected_size,
+ element_at(image, 'uri') is not null,
+ element_at(image, 'offset') >= 4,
+ fl_get_binary(image) is null
+ from blob_large_payload@options('blob-as-descriptor'='true') order
by id
+ """
+
+ sql "set paimon_blob_max_size = 12582912"
+ qt_large_raised_content """
+ select id, length(fl_get_binary(image)),
+ md5(cast(fl_get_binary(image) as string)),
+ length(fl_get_binary(image)) <=> expected_size,
+ md5(cast(fl_get_binary(image) as string)) <=> expected_md5
+ from blob_large_payload order by id
+ """
+
+ // Neither ordinary-column projection nor descriptor output needs
payload bytes.
+ sql "set paimon_blob_max_size = 1"
+ qt_large_metadata_only """
+ select id, case_name, expected_size, expected_md5
+ from blob_large_payload order by id
+ """
+ qt_large_count "select count(*) from blob_large_payload"
+ qt_large_descriptor_with_low_limit """
+ select id, element_at(image, 'size'),
+ element_at(image, 'size') <=> expected_size,
+ element_at(image, 'uri') is not null,
+ element_at(image, 'offset') >= 4,
+ fl_get_binary(image) is null
+ from blob_large_payload@options('blob-as-descriptor'='true') order
by id
+ """
+
+ // A successful inline read must not cache the larger limit for later
queries.
+ sql "set paimon_blob_max_size = 10485760"
+ test {
+ sql "select length(fl_get_binary(image)) from blob_large_payload
where id = 2"
+ exception "paimon_blob_max_size=10485760"
+ }
+
+ // Keep the smaller boundary cases for NULL, empty and multiple BLOB
columns.
sql "set paimon_blob_max_size = 8"
qt_limit_allowed """
select id, length(fl_get_binary(image)),
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy
index d18fc48329d..36606cc4ced 100644
---
a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy
@@ -51,7 +51,7 @@ suite("test_paimon_blob_native",
"p0,external,doris,external_docker,external_doc
sql "set force_jni_scanner = false"
sql "set paimon_blob_max_size = 10485760"
- ["blob_basic", "blob_all_null", "blob_size_limit",
"blob_multi_columns",
+ ["blob_basic", "blob_all_null", "blob_size_limit",
"blob_large_payload", "blob_multi_columns",
"blob_partitioned", "blob_multi_snapshot", "blob_rollover",
"blob_schema_evolution",
"blob_compacted", "blob_descriptor_default"].each { table ->
explain {
@@ -374,7 +374,71 @@ suite("test_paimon_blob_native",
"p0,external,doris,external_docker,external_doc
from blob_descriptor_default order by id
"""
- // Small payloads exercise the limit: do not allocate 10 MiB just to
test an error.
+ // Real payloads exercise the default 10 MiB boundary without
returning large values.
+ qt_large_default_exact """
+ select id, length(fl_get_binary(image)),
+ md5(cast(fl_get_binary(image) as string)),
+ length(fl_get_binary(image)) <=> expected_size,
+ md5(cast(fl_get_binary(image) as string)) <=> expected_md5
+ from blob_large_payload where id = 1
+ """
+ test {
+ sql "select length(fl_get_binary(image)) from blob_large_payload
where id = 2"
+ exception "paimon_blob_max_size=10485760"
+ }
+ test {
+ sql "select length(fl_get_binary(image)) from blob_large_payload
where id = 3"
+ exception "paimon_blob_max_size=10485760"
+ }
+ test {
+ sql """
+ select id from blob_large_payload
+ where length(fl_get_binary(image)) <= 10485760
+ """
+ exception "paimon_blob_max_size=10485760"
+ }
+ qt_large_default_descriptor """
+ select id, element_at(image, 'size'),
+ element_at(image, 'size') <=> expected_size,
+ element_at(image, 'uri') is not null,
+ element_at(image, 'offset') >= 4,
+ fl_get_binary(image) is null
+ from blob_large_payload@options('blob-as-descriptor'='true') order
by id
+ """
+
+ sql "set paimon_blob_max_size = 12582912"
+ qt_large_raised_content """
+ select id, length(fl_get_binary(image)),
+ md5(cast(fl_get_binary(image) as string)),
+ length(fl_get_binary(image)) <=> expected_size,
+ md5(cast(fl_get_binary(image) as string)) <=> expected_md5
+ from blob_large_payload order by id
+ """
+
+ // Neither ordinary-column projection nor descriptor output needs
payload bytes.
+ sql "set paimon_blob_max_size = 1"
+ qt_large_metadata_only """
+ select id, case_name, expected_size, expected_md5
+ from blob_large_payload order by id
+ """
+ qt_large_count "select count(*) from blob_large_payload"
+ qt_large_descriptor_with_low_limit """
+ select id, element_at(image, 'size'),
+ element_at(image, 'size') <=> expected_size,
+ element_at(image, 'uri') is not null,
+ element_at(image, 'offset') >= 4,
+ fl_get_binary(image) is null
+ from blob_large_payload@options('blob-as-descriptor'='true') order
by id
+ """
+
+ // A successful inline read must not cache the larger limit for later
queries.
+ sql "set paimon_blob_max_size = 10485760"
+ test {
+ sql "select length(fl_get_binary(image)) from blob_large_payload
where id = 2"
+ exception "paimon_blob_max_size=10485760"
+ }
+
+ // Keep the smaller boundary cases for NULL, empty and multiple BLOB
columns.
sql "set paimon_blob_max_size = 8"
qt_limit_allowed """
select id, length(fl_get_binary(image)),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
