This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new f3fa485c feat(avro): apply column default values when reading missing
fields (#800)
f3fa485c is described below
commit f3fa485c0a35419ebdc61ad08446f5cfdfc33d4c
Author: Xin Huang <[email protected]>
AuthorDate: Sun Aug 2 20:49:26 2026 -0700
feat(avro): apply column default values when reading missing fields (#800)
## What
Part 3 of 4 of Iceberg v3 column default-value support (POC #731), built
on the
schema layer (#746) and the Parquet read path (#792).
When a column is present in the read (table) schema but absent from an
Avro data
file — because the column was added after those rows were written — fill
it with
the column's v3 `initial-default` instead of `null`.
## Changes
- **Avro projection** (`avro_schema_util.cc`): when a field is missing
from the
file and carries an `initial-default`, project it as
`FieldProjection::Kind::kDefault`, mirroring the generic / Parquet
paths.
- **Avro decode** (`avro_data_util.cc`, `avro_direct_decoder.cc`):
materialize
the `kDefault` branch through an Avro-local `AppendDefaultToBuilder`
helper.
It reuses the shared `ToArrowScalar` conversion, while keeping Avro's
row-by-row `ArrayBuilder` append behavior out of the shared Arrow
utility.
## Tests
- `avro_data_test`: `AppendDefaultToBuilder` appends a value and casts
to the
builder type; `AppendDatumToBuilder` fills missing required and optional
default fields.
- `avro_test`: end-to-end — write an Avro file with an old schema, then
read it
through `ReaderFactoryRegistry` with an evolved schema carrying defaults
(`ReadMissingFieldsWithDefaults`).
## Stack
1. #746 — schema: represent / serialize / validate (merged)
2. #792 — read path: Parquet (merged)
3. **this PR** — read path: Avro
4. schema evolution: `addColumn` / `updateColumnDefault` (#793)
---
src/iceberg/avro/avro_data_util.cc | 152 ++++++++++++++++++++
src/iceberg/avro/avro_data_util_internal.h | 37 +++++
src/iceberg/avro/avro_direct_decoder.cc | 3 +
src/iceberg/avro/avro_reader.cc | 2 +
src/iceberg/avro/avro_schema_util.cc | 4 +
src/iceberg/test/avro_data_test.cc | 219 +++++++++++++++++++++++++++++
src/iceberg/test/avro_test.cc | 29 ++++
7 files changed, 446 insertions(+)
diff --git a/src/iceberg/avro/avro_data_util.cc
b/src/iceberg/avro/avro_data_util.cc
index 7931b071..9be8907d 100644
--- a/src/iceberg/avro/avro_data_util.cc
+++ b/src/iceberg/avro/avro_data_util.cc
@@ -17,12 +17,15 @@
* under the License.
*/
+#include <span>
+
#include <arrow/array/builder_binary.h>
#include <arrow/array/builder_decimal.h>
#include <arrow/array/builder_nested.h>
#include <arrow/array/builder_primitive.h>
#include <arrow/extension_type.h>
#include <arrow/json/from_string.h>
+#include <arrow/scalar.h>
#include <arrow/type.h>
#include <arrow/util/decimal.h>
#include <avro/Generic.hh>
@@ -31,6 +34,7 @@
#include <avro/Types.hh>
#include "iceberg/arrow/arrow_status_internal.h"
+#include "iceberg/arrow/literal_util_internal.h"
#include "iceberg/avro/avro_data_util_internal.h"
#include "iceberg/avro/avro_schema_util_internal.h"
#include "iceberg/metadata_columns.h"
@@ -88,6 +92,8 @@ Status AppendStructToBuilder(const ::avro::NodePtr& avro_node,
metadata_context,
field_builder));
} else if (field_projection.kind == FieldProjection::Kind::kNull) {
ICEBERG_ARROW_RETURN_NOT_OK(field_builder->AppendNull());
+ } else if (field_projection.kind == FieldProjection::Kind::kDefault) {
+ ICEBERG_RETURN_UNEXPECTED(AppendDefaultToBuilder(field_projection,
field_builder));
} else if (field_projection.kind == FieldProjection::Kind::kMetadata) {
int32_t field_id = expected_field.field_id();
if (field_id == MetadataColumns::kFilePathColumnId) {
@@ -466,6 +472,10 @@ Status AppendFieldToBuilder(const ::avro::NodePtr&
avro_node,
return {};
}
+ if (projection.kind == FieldProjection::Kind::kDefault) {
+ return AppendDefaultToBuilder(projection, array_builder);
+ }
+
const bool is_row_lineage =
MetadataColumns::IsRowLineageColumn(projected_field.field_id());
@@ -497,6 +507,148 @@ Status AppendFieldToBuilder(const ::avro::NodePtr&
avro_node,
} // namespace
+namespace {
+
+Result<std::shared_ptr<::arrow::Scalar>> MakeDefaultScalar(
+ const Literal& literal, const std::shared_ptr<::arrow::DataType>&
builder_type) {
+ // The builder's own memory pool is not exposed, so the small scalar buffer
uses the
+ // default pool.
+ ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr<::arrow::Scalar> scalar,
+ arrow::ToArrowScalar(literal,
::arrow::default_memory_pool()));
+
+ // For an extension builder (e.g. `arrow.uuid`) target its storage type:
ToArrowScalar
+ // yields the storage scalar (fixed_size_binary(16) for uuid) and
Scalar::CastTo has no
+ // kernel that targets an extension type. This mirrors MakeDefaultArray's
extension
+ // handling.
+ std::shared_ptr<::arrow::DataType> target_type = builder_type;
+ if (target_type->id() == ::arrow::Type::EXTENSION) {
+ target_type = internal::checked_cast<const
::arrow::ExtensionType&>(*target_type)
+ .storage_type();
+ }
+
+ if (!scalar->type->Equals(*target_type)) {
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(scalar, scalar->CastTo(target_type));
+ }
+ return scalar;
+}
+
+Status PrepareStructDefaultScalars(std::span<FieldProjection> projections,
+ ::arrow::ArrayBuilder* builder);
+
+// Recurse into whatever nested builder this projection describes, so a
default cached for
+// a struct field is found at any nesting depth (e.g.
`list<list<struct<...>>>`) instead
+// of only when the collection's child is an immediate struct.
+Status PrepareNestedDefaultScalars(FieldProjection& projection,
+ ::arrow::ArrayBuilder* builder) {
+ if (projection.kind != FieldProjection::Kind::kProjected ||
+ projection.children.empty()) {
+ return {};
+ }
+
+ switch (builder->type()->id()) {
+ case ::arrow::Type::STRUCT:
+ return PrepareStructDefaultScalars(projection.children, builder);
+ case ::arrow::Type::LIST: {
+ // List projections store a single child for the element.
+ auto* list_builder =
internal::checked_cast<::arrow::ListBuilder*>(builder);
+ return PrepareNestedDefaultScalars(projection.children[0],
+ list_builder->value_builder());
+ }
+ case ::arrow::Type::LARGE_LIST: {
+ auto* list_builder =
internal::checked_cast<::arrow::LargeListBuilder*>(builder);
+ return PrepareNestedDefaultScalars(projection.children[0],
+ list_builder->value_builder());
+ }
+ case ::arrow::Type::MAP: {
+ auto* map_builder =
internal::checked_cast<::arrow::MapBuilder*>(builder);
+ if (projection.children.size() >= 1) {
+ ICEBERG_RETURN_UNEXPECTED(PrepareNestedDefaultScalars(
+ projection.children[0], map_builder->key_builder()));
+ }
+ if (projection.children.size() >= 2) {
+ ICEBERG_RETURN_UNEXPECTED(PrepareNestedDefaultScalars(
+ projection.children[1], map_builder->item_builder()));
+ }
+ return {};
+ }
+ default:
+ return {};
+ }
+}
+
+Status PrepareStructDefaultScalars(std::span<FieldProjection> projections,
+ ::arrow::ArrayBuilder* builder) {
+ auto* struct_builder =
internal::checked_cast<::arrow::StructBuilder*>(builder);
+ if (static_cast<size_t>(struct_builder->num_fields()) != projections.size())
{
+ return InvalidArgument(
+ "Inconsistent number of struct builder fields ({}) and projections
({})",
+ struct_builder->num_fields(), projections.size());
+ }
+
+ for (size_t i = 0; i < projections.size(); ++i) {
+ auto& field_projection = projections[i];
+ auto* field_builder = struct_builder->field_builder(static_cast<int>(i));
+
+ if (field_projection.kind == FieldProjection::Kind::kDefault) {
+ // Get-or-create the single Avro attributes container: another Avro
attribute may
+ // have created it already, so guard on `default_scalar` being unset
rather than on
+ // the container's presence (otherwise a pre-existing container would
skip
+ // preparation and silently fall back to per-row scalar rebuilding).
+ std::shared_ptr<AvroExtraAttributes> attrs;
+ if (field_projection.attributes == nullptr) {
+ attrs = std::make_shared<AvroExtraAttributes>();
+ field_projection.attributes = attrs;
+ } else {
+ // Avro attaches only AvroExtraAttributes; checked_pointer_cast
asserts that
+ // invariant in debug rather than silently reinterpreting a foreign
attributes
+ // type.
+ attrs = internal::checked_pointer_cast<AvroExtraAttributes>(
+ field_projection.attributes);
+ }
+ if (attrs->default_scalar == nullptr) {
+ ICEBERG_ASSIGN_OR_RAISE(
+ attrs->default_scalar,
+ MakeDefaultScalar(std::get<Literal>(field_projection.from),
+ field_builder->type()));
+ }
+ continue;
+ }
+
+ ICEBERG_RETURN_UNEXPECTED(
+ PrepareNestedDefaultScalars(field_projection, field_builder));
+ }
+ return {};
+}
+
+} // namespace
+
+Status PrepareDefaultScalars(SchemaProjection& projection,
+ ::arrow::ArrayBuilder* root_builder) {
+ return PrepareStructDefaultScalars(projection.fields, root_builder);
+}
+
+Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder*
builder) {
+ ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr<::arrow::Scalar> scalar,
+ MakeDefaultScalar(literal, builder->type()));
+ ICEBERG_ARROW_RETURN_NOT_OK(builder->AppendScalar(*scalar));
+ return {};
+}
+
+Status AppendDefaultToBuilder(const FieldProjection& projection,
+ ::arrow::ArrayBuilder* builder) {
+ // Avro projections carry a single attributes type, so once one is attached
it is an
+ // AvroExtraAttributes; use checked_cast instead of a per-row dynamic_cast.
+ if (projection.attributes != nullptr) {
+ const auto& attrs =
+ internal::checked_cast<const
AvroExtraAttributes&>(*projection.attributes);
+ if (attrs.default_scalar != nullptr) {
+
ICEBERG_ARROW_RETURN_NOT_OK(builder->AppendScalar(*attrs.default_scalar));
+ return {};
+ }
+ }
+ return AppendDefaultToBuilder(std::get<Literal>(projection.from), builder);
+}
+
Status AppendDatumToBuilder(const ::avro::NodePtr& avro_node,
const ::avro::GenericDatum& avro_datum,
const SchemaProjection& projection,
diff --git a/src/iceberg/avro/avro_data_util_internal.h
b/src/iceberg/avro/avro_data_util_internal.h
index 281187f1..9bb86c13 100644
--- a/src/iceberg/avro/avro_data_util_internal.h
+++ b/src/iceberg/avro/avro_data_util_internal.h
@@ -19,14 +19,51 @@
#pragma once
+#include <memory>
+
#include <arrow/array/builder_base.h>
+#include <arrow/scalar.h>
#include <avro/GenericDatum.hh>
#include "iceberg/arrow/metadata_column_util_internal.h"
+#include "iceberg/expression/literal.h"
#include "iceberg/schema_util.h"
namespace iceberg::avro {
+/// \brief Avro-specific per-field projection attributes.
+///
+/// `FieldProjection` has a single attributes slot, so all Avro-side
attributes live in
+/// one container (mirroring `ParquetExtraAttributes`) rather than separate
subclasses
+/// that could not coexist. `default_scalar` is the Arrow scalar for a
`kDefault` field,
+/// materialized once (see `PrepareDefaultScalars`) so row-by-row decode only
needs
+/// `AppendScalar` instead of repeating `ToArrowScalar` / `CastTo` per row.
+struct AvroExtraAttributes : FieldProjection::ExtraAttributes {
+ std::shared_ptr<::arrow::Scalar> default_scalar;
+};
+
+/// \brief Precompute cast Arrow scalars for every `kDefault` field under
`projection`.
+///
+/// Walks `root_builder` in lockstep with the projection so each default is
cast to the
+/// builder's Arrow type once per scan. Safe to call repeatedly; existing
+/// `AvroExtraAttributes` entries are left unchanged.
+Status PrepareDefaultScalars(SchemaProjection& projection,
+ ::arrow::ArrayBuilder* root_builder);
+
+/// \brief Append a literal once to `builder` while decoding Avro row-by-row.
+///
+/// Used to materialize `FieldProjection::Kind::kDefault`. Shares
`ToArrowScalar` with
+/// Parquet's batch path (`MakeDefaultArray`); the append shape stays
Avro-local because
+/// Avro builds Arrow arrays via per-row `ArrayBuilder`s rather than
whole-column arrays.
+/// Prefer the `FieldProjection` overload after `PrepareDefaultScalars` so the
scalar is
+/// reused across rows.
+Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder*
builder);
+
+/// \brief Append a `kDefault` projection, reusing a scalar cached on
+/// `projection.attributes` when present.
+Status AppendDefaultToBuilder(const FieldProjection& projection,
+ ::arrow::ArrayBuilder* builder);
+
/// \brief Append an Avro datum to an Arrow array builder.
///
/// This function handles schema evolution by using the provided projection to
map
diff --git a/src/iceberg/avro/avro_direct_decoder.cc
b/src/iceberg/avro/avro_direct_decoder.cc
index b66a9310..8f563b3b 100644
--- a/src/iceberg/avro/avro_direct_decoder.cc
+++ b/src/iceberg/avro/avro_direct_decoder.cc
@@ -29,6 +29,7 @@
#include <avro/Types.hh>
#include "iceberg/arrow/arrow_status_internal.h"
+#include "iceberg/avro/avro_data_util_internal.h"
#include "iceberg/avro/avro_direct_decoder_internal.h"
#include "iceberg/avro/avro_schema_util_internal.h"
#include "iceberg/metadata_columns.h"
@@ -209,6 +210,8 @@ Status DecodeStructToBuilder(const ::avro::NodePtr&
avro_node, ::avro::Decoder&
auto* field_builder =
struct_builder->field_builder(static_cast<int>(proj_idx));
if (field_projection.kind == FieldProjection::Kind::kNull) {
ICEBERG_ARROW_RETURN_NOT_OK(field_builder->AppendNull());
+ } else if (field_projection.kind == FieldProjection::Kind::kDefault) {
+ ICEBERG_RETURN_UNEXPECTED(AppendDefaultToBuilder(field_projection,
field_builder));
} else if (field_projection.kind == FieldProjection::Kind::kMetadata) {
int32_t field_id = expected_field.field_id();
if (field_id == MetadataColumns::kFilePathColumnId) {
diff --git a/src/iceberg/avro/avro_reader.cc b/src/iceberg/avro/avro_reader.cc
index b0977bcd..f333a38c 100644
--- a/src/iceberg/avro/avro_reader.cc
+++ b/src/iceberg/avro/avro_reader.cc
@@ -434,6 +434,8 @@ class AvroReader::Impl {
builder_result.status().message());
}
context_->builder_ = builder_result.MoveValueUnsafe();
+ ICEBERG_RETURN_UNEXPECTED(
+ PrepareDefaultScalars(projection_, context_->builder_.get()));
backend_->InitReadContext(backend_->GetReaderSchema());
return {};
diff --git a/src/iceberg/avro/avro_schema_util.cc
b/src/iceberg/avro/avro_schema_util.cc
index 14b464ce..5e6bee95 100644
--- a/src/iceberg/avro/avro_schema_util.cc
+++ b/src/iceberg/avro/avro_schema_util.cc
@@ -752,6 +752,10 @@ Result<FieldProjection> ProjectStruct(const StructType&
struct_type,
iter->second.local_index,
prune_source));
} else if (MetadataColumns::IsMetadataColumn(field_id)) {
child_projection.kind = FieldProjection::Kind::kMetadata;
+ } else if (expected_field.initial_default() != nullptr) {
+ // Rows written before the field existed assume its `initial-default`
value.
+ child_projection.kind = FieldProjection::Kind::kDefault;
+ child_projection.from = *expected_field.initial_default();
} else if (expected_field.optional()) {
child_projection.kind = FieldProjection::Kind::kNull;
} else {
diff --git a/src/iceberg/test/avro_data_test.cc
b/src/iceberg/test/avro_data_test.cc
index 7731f58d..4ea7855c 100644
--- a/src/iceberg/test/avro_data_test.cc
+++ b/src/iceberg/test/avro_data_test.cc
@@ -17,10 +17,16 @@
* under the License.
*/
+#include <memory>
#include <ranges>
+#include <vector>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
#include <arrow/c/bridge.h>
#include <arrow/json/from_string.h>
+#include <arrow/memory_pool.h>
+#include <arrow/type.h>
#include <arrow/util/decimal.h>
#include <avro/Compiler.hh>
#include <avro/Generic.hh>
@@ -31,6 +37,7 @@
#include "iceberg/avro/avro_data_util_internal.h"
#include "iceberg/avro/avro_schema_util_internal.h"
+#include "iceberg/expression/literal.h"
#include "iceberg/schema.h"
#include "iceberg/schema_internal.h"
#include "iceberg/schema_util.h"
@@ -662,6 +669,218 @@ TEST(AppendDatumToBuilderTest,
StructWithMissingOptionalField) {
avro_data,
expected_json));
}
+TEST(AppendDatumToBuilderTest, StructWithMissingDefaultFields) {
+ Schema iceberg_schema({
+ SchemaField::MakeRequired(1, "id", iceberg::int32()),
+ // Missing required field with an initial-default: filled with the
default.
+ SchemaField(2, "score", iceberg::int64(), /*optional=*/false, /*doc=*/{},
+ std::make_shared<const Literal>(Literal::Long(100))),
+ // Missing optional field with an initial-default: also filled, not null.
+ SchemaField(3, "grade", iceberg::string(), /*optional=*/true, /*doc=*/{},
+ std::make_shared<const Literal>(Literal::String("A"))),
+ });
+
+ // Create Avro schema that only has the id field (missing score and grade).
+ std::string avro_schema_json = R"({
+ "type": "record",
+ "name": "person",
+ "fields": [
+ {"name": "id", "type": "int", "field-id": 1}
+ ]
+ })";
+ auto avro_schema = ::avro::compileJsonSchemaFromString(avro_schema_json);
+
+ std::vector<::avro::GenericDatum> avro_data;
+ for (int i = 0; i < 2; ++i) {
+ ::avro::GenericDatum avro_datum(avro_schema.root());
+ auto& record = avro_datum.value<::avro::GenericRecord>();
+ record.fieldAt(0).value<int32_t>() = i + 1;
+ avro_data.push_back(avro_datum);
+ }
+
+ const std::string expected_json = R"([
+ {"id": 1, "score": 100, "grade": "A"},
+ {"id": 2, "score": 100, "grade": "A"}
+ ])";
+ ASSERT_NO_FATAL_FAILURE(VerifyAppendDatumToBuilder(iceberg_schema,
avro_schema.root(),
+ avro_data,
expected_json));
+}
+
+TEST(AppendDatumToBuilderTest, NestedListOfListStructWithMissingDefaultField) {
+ // A default on a struct nested two collection levels deep must still be
filled, which
+ // requires recursing through both list builders rather than only an
immediate struct.
+ auto inner_struct = std::make_shared<StructType>(std::vector<SchemaField>{
+ SchemaField::MakeRequired(5, "x", iceberg::int32()),
+ SchemaField(6, "y", iceberg::int64(), /*optional=*/false, /*doc=*/{},
+ std::make_shared<const Literal>(Literal::Long(7))),
+ });
+ auto inner_list =
+ std::make_shared<ListType>(SchemaField::MakeRequired(4, "element",
inner_struct));
+ auto outer_list =
+ std::make_shared<ListType>(SchemaField::MakeRequired(3, "element",
inner_list));
+ Schema iceberg_schema({
+ SchemaField::MakeRequired(1, "id", iceberg::int32()),
+ SchemaField::MakeRequired(2, "matrix", outer_list),
+ });
+
+ // The Avro schema's innermost record only has `x`, so `y` comes from its
default.
+ std::string avro_schema_json = R"({
+ "type": "record",
+ "name": "outer",
+ "fields": [
+ {"name": "id", "type": "int", "field-id": 1},
+ {"name": "matrix", "field-id": 2, "type": {
+ "type": "array", "element-id": 3, "items": {
+ "type": "array", "element-id": 4, "items": {
+ "type": "record", "name": "point",
+ "fields": [{"name": "x", "type": "int", "field-id": 5}]
+ }
+ }
+ }}
+ ]
+ })";
+ auto avro_schema = ::avro::compileJsonSchemaFromString(avro_schema_json);
+
+ std::vector<::avro::GenericDatum> avro_data;
+ ::avro::GenericDatum avro_datum(avro_schema.root());
+ auto& record = avro_datum.value<::avro::GenericRecord>();
+ record.fieldAt(0).value<int32_t>() = 1;
+ auto& outer_array = record.fieldAt(1).value<::avro::GenericArray>();
+ ::avro::GenericDatum inner_datum(avro_schema.root()->leafAt(1)->leafAt(0));
+ auto& inner_array = inner_datum.value<::avro::GenericArray>();
+ for (int32_t x : {10, 20}) {
+ ::avro::GenericDatum
point_datum(avro_schema.root()->leafAt(1)->leafAt(0)->leafAt(0));
+ point_datum.value<::avro::GenericRecord>().fieldAt(0).value<int32_t>() = x;
+ inner_array.value().push_back(point_datum);
+ }
+ outer_array.value().push_back(inner_datum);
+ avro_data.push_back(avro_datum);
+
+ const std::string expected_json = R"([
+ {"id": 1, "matrix": [[{"x": 10, "y": 7}, {"x": 20, "y": 7}]]}
+ ])";
+ ASSERT_NO_FATAL_FAILURE(VerifyAppendDatumToBuilder(iceberg_schema,
avro_schema.root(),
+ avro_data,
expected_json));
+}
+
+TEST(AppendDefaultToBuilderTest, AppendsValue) {
+ ::arrow::Int64Builder builder;
+ ASSERT_THAT(AppendDefaultToBuilder(Literal::Long(42), &builder), IsOk());
+ ASSERT_THAT(AppendDefaultToBuilder(Literal::Long(42), &builder), IsOk());
+
+ std::shared_ptr<::arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+ ASSERT_EQ(array->length(), 2);
+ const auto& long_array = static_cast<const ::arrow::Int64Array&>(*array);
+ ASSERT_EQ(long_array.Value(0), 42);
+ ASSERT_EQ(long_array.Value(1), 42);
+}
+
+TEST(AppendDefaultToBuilderTest, CastsToBuilderType) {
+ // The literal's natural type (int32) differs from the builder type (int64);
the value
+ // is cast to the builder type.
+ ::arrow::Int64Builder builder;
+ ASSERT_THAT(AppendDefaultToBuilder(Literal::Int(7), &builder), IsOk());
+
+ std::shared_ptr<::arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+ ASSERT_EQ(array->length(), 1);
+ ASSERT_EQ(static_cast<const ::arrow::Int64Array&>(*array).Value(0), 7);
+}
+
+TEST(AppendDefaultToBuilderTest, ReusesPreparedScalar) {
+ auto pool = ::arrow::default_memory_pool();
+ auto child = std::make_shared<::arrow::Int64Builder>(pool);
+ ::arrow::StructBuilder struct_builder(
+ ::arrow::struct_({::arrow::field("d", ::arrow::int64())}), pool,
{child});
+
+ FieldProjection projection;
+ projection.kind = FieldProjection::Kind::kDefault;
+ projection.from = Literal::Long(42);
+
+ SchemaProjection schema_projection;
+ schema_projection.fields.push_back(projection);
+ ASSERT_THAT(PrepareDefaultScalars(schema_projection, &struct_builder),
IsOk());
+ ASSERT_NE(dynamic_cast<const AvroExtraAttributes*>(
+ schema_projection.fields[0].attributes.get()),
+ nullptr);
+
+ auto* field_builder = struct_builder.field_builder(0);
+ ASSERT_THAT(AppendDefaultToBuilder(schema_projection.fields[0],
field_builder), IsOk());
+ ASSERT_THAT(AppendDefaultToBuilder(schema_projection.fields[0],
field_builder), IsOk());
+
+ std::shared_ptr<::arrow::Array> array;
+ ASSERT_TRUE(field_builder->Finish(&array).ok());
+ ASSERT_EQ(array->length(), 2);
+ const auto& long_array = static_cast<const ::arrow::Int64Array&>(*array);
+ ASSERT_EQ(long_array.Value(0), 42);
+ ASSERT_EQ(long_array.Value(1), 42);
+}
+
+TEST(AppendDefaultToBuilderTest, PreparesScalarWhenContainerAlreadyExists) {
+ // The Avro attributes container may already exist (e.g. attached by another
Avro
+ // attribute) with default_scalar unset. Preparation must still fill
default_scalar
+ // rather than skip on the container's mere presence, otherwise the append
path falls
+ // back to rebuilding the scalar per row.
+ auto pool = ::arrow::default_memory_pool();
+ auto child = std::make_shared<::arrow::Int64Builder>(pool);
+ ::arrow::StructBuilder struct_builder(
+ ::arrow::struct_({::arrow::field("d", ::arrow::int64())}), pool,
{child});
+
+ FieldProjection projection;
+ projection.kind = FieldProjection::Kind::kDefault;
+ projection.from = Literal::Long(42);
+ // Pre-attach an empty container, simulating another attribute having
created it.
+ projection.attributes = std::make_shared<AvroExtraAttributes>();
+
+ SchemaProjection schema_projection;
+ schema_projection.fields.push_back(projection);
+ ASSERT_THAT(PrepareDefaultScalars(schema_projection, &struct_builder),
IsOk());
+
+ const auto* attrs = dynamic_cast<const AvroExtraAttributes*>(
+ schema_projection.fields[0].attributes.get());
+ ASSERT_NE(attrs, nullptr);
+ ASSERT_NE(attrs->default_scalar, nullptr);
+}
+
+TEST(AppendDefaultToBuilderTest, PreparesScalarUnderNestedCollections) {
+ // A default under `list<list<struct<...>>>` must be prepared too, so
decoding reuses
+ // the cached scalar instead of rebuilding it for every element.
+ auto pool = ::arrow::default_memory_pool();
+ auto leaf = std::make_shared<::arrow::Int64Builder>(pool);
+ auto point_builder = std::make_shared<::arrow::StructBuilder>(
+ ::arrow::struct_({::arrow::field("y", ::arrow::int64())}), pool,
+ std::vector<std::shared_ptr<::arrow::ArrayBuilder>>{leaf});
+ auto inner_list = std::make_shared<::arrow::ListBuilder>(pool,
point_builder);
+ auto outer_list = std::make_shared<::arrow::ListBuilder>(pool, inner_list);
+ ::arrow::StructBuilder root_builder(
+ ::arrow::struct_({::arrow::field("matrix", outer_list->type())}), pool,
+ std::vector<std::shared_ptr<::arrow::ArrayBuilder>>{outer_list});
+
+ FieldProjection leaf_default;
+ leaf_default.kind = FieldProjection::Kind::kDefault;
+ leaf_default.from = Literal::Long(7);
+
+ FieldProjection point_projection;
+ point_projection.kind = FieldProjection::Kind::kProjected;
+ point_projection.children.push_back(leaf_default);
+
+ FieldProjection inner_element;
+ inner_element.kind = FieldProjection::Kind::kProjected;
+ inner_element.children.push_back(point_projection);
+
+ FieldProjection outer_element;
+ outer_element.kind = FieldProjection::Kind::kProjected;
+ outer_element.children.push_back(inner_element);
+
+ SchemaProjection schema_projection;
+ schema_projection.fields.push_back(outer_element);
+ ASSERT_THAT(PrepareDefaultScalars(schema_projection, &root_builder), IsOk());
+
+ const auto& prepared =
schema_projection.fields[0].children[0].children[0].children[0];
+ ASSERT_NE(dynamic_cast<const
AvroExtraAttributes*>(prepared.attributes.get()), nullptr);
+}
+
TEST(AppendDatumToBuilderTest, NestedStructWithMissingOptionalFields) {
Schema iceberg_schema({
SchemaField::MakeRequired(1, "id", iceberg::int32()),
diff --git a/src/iceberg/test/avro_test.cc b/src/iceberg/test/avro_test.cc
index 604cf5b0..20156a57 100644
--- a/src/iceberg/test/avro_test.cc
+++ b/src/iceberg/test/avro_test.cc
@@ -42,6 +42,7 @@
#include "iceberg/avro/avro_register.h"
#include "iceberg/avro/avro_stream_internal.h"
#include "iceberg/avro/avro_writer.h"
+#include "iceberg/expression/literal.h"
#include "iceberg/file_reader.h"
#include "iceberg/metadata_columns.h"
#include "iceberg/schema.h"
@@ -318,6 +319,34 @@ TEST_F(AvroReaderTest, ReadTwoFields) {
ASSERT_NO_FATAL_FAILURE(VerifyExhausted(*reader));
}
+TEST_F(AvroReaderTest, ReadMissingFieldsWithDefaults) {
+ // The file contains only fields 1 and 2; the projected schema adds fields 3
and 4
+ // with initial-defaults, which are filled for all rows written before the
columns
+ // existed.
+ CreateSimpleAvroFile();
+
+ auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
+ SchemaField::MakeRequired(1, "id", std::make_shared<IntType>()),
+ SchemaField::MakeOptional(2, "name", std::make_shared<StringType>()),
+ SchemaField(3, "score", std::make_shared<LongType>(), /*optional=*/false,
+ /*doc=*/{}, std::make_shared<const
Literal>(Literal::Long(100))),
+ SchemaField(4, "status", std::make_shared<StringType>(),
/*optional=*/true,
+ /*doc=*/{}, std::make_shared<const
Literal>(Literal::String("active"))),
+ });
+
+ auto reader_result = ReaderFactoryRegistry::Open(
+ FileFormatType::kAvro,
+ {.path = temp_avro_file_, .io = file_io_, .projection = schema});
+ ASSERT_THAT(reader_result, IsOk());
+ auto reader = std::move(reader_result.value());
+
+ ASSERT_NO_FATAL_FAILURE(VerifyNextBatch(*reader,
+ R"([[1, "Alice", 100, "active"],
+ [2, "Bob", 100, "active"],
+ [3, "Charlie", 100,
"active"]])"));
+ ASSERT_NO_FATAL_FAILURE(VerifyExhausted(*reader));
+}
+
TEST_F(AvroReaderTest, RoundTripWithGenericFileIO) {
file_io_ = std::make_shared<iceberg::test::StdFileIO>();
temp_avro_file_ = CreateNewTempFilePathWithSuffix(".avro");
