This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 470a8b53974 [fix](be) exclude wal from variant format validation
(#68233) (#68242)
470a8b53974 is described below
commit 470a8b539747ead156c09c14923ffe1dea731124
Author: zhangstar333 <[email protected]>
AuthorDate: Sun Sep 20 19:28:56 2026 +0800
[fix](be) exclude wal from variant format validation (#68233) (#68242)
### What problem does this PR solve?
Problem Summary:
WAL stores doris blocks directly and supports variant columns. exclude
WAL from external variant file-format validation to avoid incorrectly
returning a not-supported error.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/format_v2/table_reader.cpp | 9 ++++++---
be/test/format_v2/table_reader_test.cpp | 11 ++++++++++-
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/be/src/format_v2/table_reader.cpp
b/be/src/format_v2/table_reader.cpp
index 4fc74740c6c..13dd36d5810 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -1199,14 +1199,17 @@ Status TableReader::init(TableReadOptions&& options) {
Status TableReader::validate_variant_file_mappings(FileFormat format,
const
std::vector<ColumnMapping>& mappings) {
- if (format == FileFormat::PARQUET || !std::ranges::any_of(mappings,
mapping_reads_variant)) {
+ // WAL stores Doris blocks directly and therefore supports Variant without
the external-file
+ // format restrictions applied here.
+ if (format == FileFormat::PARQUET || format == FileFormat::WAL ||
+ !std::ranges::any_of(mappings, mapping_reads_variant)) {
return Status::OK();
}
// Gate on a physical mapping, not the table schema: an older file may
legitimately omit a
// Variant field added by schema evolution, in which case the mapper
synthesizes NULL.
return Status::NotSupported(
- "External Variant is supported only for Parquet files in
FileScannerV2; file format "
- "{} is not supported",
+ "Variant is supported only for Parquet files and WAL in
FileScannerV2; file format {} "
+ "is not supported",
file_format_to_string(format));
}
diff --git a/be/test/format_v2/table_reader_test.cpp
b/be/test/format_v2/table_reader_test.cpp
index efa7386fc76..d2baf23fd2d 100644
--- a/be/test/format_v2/table_reader_test.cpp
+++ b/be/test/format_v2/table_reader_test.cpp
@@ -99,7 +99,8 @@ TEST(TableReaderTest,
VariantFormatGateUsesPhysicalFileMappings) {
const auto orc_status =
VariantValidationTableReader::validate(FileFormat::ORC,
{physical_variant});
EXPECT_TRUE(orc_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
orc_status;
- EXPECT_NE(orc_status.to_string().find("supported only for Parquet"),
std::string::npos);
+ EXPECT_NE(orc_status.to_string().find("supported only for Parquet files
and WAL"),
+ std::string::npos);
EXPECT_TRUE(
VariantValidationTableReader::validate(FileFormat::PARQUET,
{physical_variant}).ok());
@@ -121,6 +122,14 @@ TEST(TableReaderTest,
VariantFormatGateUsesPhysicalFileMappings) {
EXPECT_TRUE(nested_orc_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
nested_orc_status;
}
+TEST(TableReaderTest, VariantFormatGateAllowsWal) {
+ ColumnMapping physical_variant;
+ physical_variant.table_type =
make_nullable(std::make_shared<DataTypeVariantV2>());
+ physical_variant.file_local_id = 0;
+
+ EXPECT_TRUE(VariantValidationTableReader::validate(FileFormat::WAL,
{physical_variant}).ok());
+}
+
TEST(LocalColumnIndexTest,
MergeUnionsPartialChildrenAndFullProjectionDominates) {
LocalColumnIndex target {.index = 10, .project_all_children = false};
target.children.push_back({.index = 1});
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
