This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new ee3c9592 fix(orc): support CHAR/VARCHAR compatibility and extend
format compatibility tests (#362)
ee3c9592 is described below
commit ee3c9592c311b43f4afffaafb5636ecf609f1bf3
Author: lxy <[email protected]>
AuthorDate: Fri Sep 18 18:27:39 2026 +0800
fix(orc): support CHAR/VARCHAR compatibility and extend format
compatibility tests (#362)
---
src/paimon/format/orc/orc_file_batch_reader.cpp | 34 ++-
.../format/orc/orc_file_batch_reader_test.cpp | 14 +
test/inte/paimon_read_compat_inte_test.cpp | 305 +++++++++++++--------
.../java_array_blob_types/README.md | 17 ++
...ata-e450b386-992c-4934-a77c-428b16f00082-0.avro | Bin 0 -> 217 bytes
...ata-e450b386-992c-4934-a77c-428b16f00082-1.blob | Bin 0 -> 85 bytes
...manifest-1d608455-d871-46a3-ad5c-c6359f77c02c-0 | Bin 0 -> 2221 bytes
...est-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-0 | Bin 0 -> 1158 bytes
...est-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-1 | Bin 0 -> 1261 bytes
.../java_array_blob_types/schema/schema-0 | 7 +-
.../java_array_blob_types/snapshot/EARLIEST | 1 +
.../java_array_blob_types/snapshot/LATEST | 1 +
.../java_array_blob_types/snapshot/snapshot-1 | 18 ++
.../java_map_blob_types/README.md | 17 ++
...ata-67ea4504-2d8c-449a-b66d-e51f7866489a-0.avro | Bin 0 -> 217 bytes
...ata-67ea4504-2d8c-449a-b66d-e51f7866489a-1.blob | Bin 0 -> 100 bytes
...manifest-d901117d-19e9-45d7-9627-7b385635001e-0 | Bin 0 -> 2216 bytes
...est-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-0 | Bin 0 -> 1158 bytes
...est-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-1 | Bin 0 -> 1261 bytes
.../java_map_blob_types/schema/schema-0 | 7 +-
.../java_map_blob_types/snapshot/EARLIEST | 1 +
.../java_map_blob_types/snapshot/LATEST | 1 +
.../java_map_blob_types/snapshot/snapshot-1 | 18 ++
.../java_time_types/README.md | 12 +
...ata-552bc110-1000-47fd-91f8-d8c988911595-0.avro | Bin 0 -> 407 bytes
...manifest-3d3ad23a-e678-485a-aeae-fb733e7d970a-0 | Bin 0 -> 2189 bytes
...est-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-0 | Bin 0 -> 1158 bytes
...est-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-1 | Bin 0 -> 1261 bytes
.../java_time_types/schema/schema-0 | 17 +-
.../java_time_types/snapshot/EARLIEST | 1 +
.../java_time_types/snapshot/LATEST | 1 +
.../java_time_types/snapshot/snapshot-1 | 18 ++
.../java_types/README.md | 92 +++++++
...ata-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-0.avro | Bin 0 -> 4485 bytes
...ata-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-1.blob | Bin 0 -> 81 bytes
...manifest-ad942b8a-bdb3-4a5d-a6ab-637f8d2a9ba5-0 | Bin 0 -> 2747 bytes
...est-list-ecb389f6-229e-4460-ab9e-d739db56e794-0 | Bin 0 -> 1158 bytes
...est-list-ecb389f6-229e-4460-ab9e-d739db56e794-1 | Bin 0 -> 1261 bytes
.../java_types/schema/schema-0 | 112 ++------
.../java_types/snapshot/EARLIEST | 1 +
.../java_types/snapshot/LATEST | 1 +
.../java_types/snapshot/snapshot-1 | 18 ++
.../java_vector_types/README.md | 16 ++
...ata-3e8ef942-de78-4a10-acb8-911560a123bd-0.avro | Bin 0 -> 1052 bytes
...manifest-ef4060ce-0bd8-4eab-ba5f-4f4a90603b8f-0 | Bin 0 -> 2189 bytes
...est-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-0 | Bin 0 -> 1158 bytes
...est-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-1 | Bin 0 -> 1258 bytes
.../java_vector_types/schema/schema-0 | 7 +-
.../java_vector_types/snapshot/EARLIEST | 1 +
.../java_vector_types/snapshot/LATEST | 1 +
.../java_vector_types/snapshot/snapshot-1 | 18 ++
.../java_array_blob_types/README.md | 18 ++
...data-51e35438-5081-45b2-a42b-01a7146ef8dd-0.orc | Bin 0 -> 219 bytes
...ata-51e35438-5081-45b2-a42b-01a7146ef8dd-1.blob | Bin 0 -> 100 bytes
...manifest-fc560a6b-5521-465f-9523-72234acb5ee8-0 | Bin 0 -> 2241 bytes
...est-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-0 | Bin 0 -> 1158 bytes
...est-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-1 | Bin 0 -> 1258 bytes
.../java_array_blob_types/schema/schema-0 | 8 +-
.../java_array_blob_types/snapshot/EARLIEST | 1 +
.../java_array_blob_types/snapshot/LATEST | 1 +
.../java_array_blob_types/snapshot/snapshot-1 | 18 ++
.../java_map_blob_types/README.md | 18 ++
...data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-0.orc | Bin 0 -> 219 bytes
...ata-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-1.blob | Bin 0 -> 100 bytes
...manifest-be1d7959-ae2c-4e46-84d0-3f42732e6560-0 | Bin 0 -> 2237 bytes
...est-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-0 | Bin 0 -> 1158 bytes
...est-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-1 | Bin 0 -> 1261 bytes
.../java_map_blob_types/schema/schema-0 | 8 +-
.../java_map_blob_types/snapshot/EARLIEST | 1 +
.../java_map_blob_types/snapshot/LATEST | 1 +
.../java_map_blob_types/snapshot/snapshot-1 | 18 ++
.../java_time_types/README.md | 14 +
...data-c3c43658-047f-42b1-bfae-26f67cd77c5f-0.orc | Bin 0 -> 626 bytes
...manifest-306250ad-71a2-42a3-9d47-a7146425bc2b-0 | Bin 0 -> 2198 bytes
...est-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-0 | Bin 0 -> 1158 bytes
...est-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-1 | Bin 0 -> 1264 bytes
.../java_time_types/schema/schema-0 | 8 +-
.../java_time_types/snapshot/EARLIEST | 1 +
.../java_time_types/snapshot/LATEST | 1 +
.../java_time_types/snapshot/snapshot-1 | 18 ++
.../java_types/README.md | 100 +++++++
...data-4135518c-e709-4232-ad15-508e494bdcf8-0.orc | Bin 0 -> 6058 bytes
...ata-4135518c-e709-4232-ad15-508e494bdcf8-1.blob | Bin 0 -> 81 bytes
...manifest-6b6f74b3-783e-4138-84a2-343b997c2f53-0 | Bin 0 -> 2727 bytes
...est-list-76873624-e06e-4313-b7ae-ec0e45180cc6-0 | Bin 0 -> 1158 bytes
...est-list-76873624-e06e-4313-b7ae-ec0e45180cc6-1 | Bin 0 -> 1261 bytes
.../java_types/schema/schema-0 | 63 +----
.../java_types/snapshot/EARLIEST | 1 +
.../java_types/snapshot/LATEST | 1 +
.../java_types/snapshot/snapshot-1 | 18 ++
.../append_types_compatibility.db/README.md | 55 ----
.../java_array_blob_types/README.md | 17 ++
.../java_array_blob_types/schema/schema-0 | 3 +-
.../java_map_blob_types/README.md | 17 ++
.../java_map_blob_types/schema/schema-0 | 3 +-
.../java_time_types/README.md | 13 +
.../java_time_types/schema/schema-0 | 3 +-
.../java_types/README.md | 103 +++++++
.../java_types/schema/schema-0 | 3 +-
.../java_vector_types/README.md | 16 ++
.../java_vector_types/schema/schema-0 | 3 +-
.../python_array_blob_types/README.md | 17 ++
.../python_array_blob_types/schema/schema-0 | 3 +-
.../python_map_blob_types/README.md | 17 ++
.../python_map_blob_types/schema/schema-0 | 3 +-
.../python_time_types/README.md | 13 +
.../python_time_types/schema/schema-0 | 5 +-
.../python_types/README.md | 103 +++++++
.../python_types/schema/schema-0 | 3 +-
.../python_vector_types/README.md | 16 ++
.../python_vector_types/schema/schema-0 | 5 +-
.../rust_array_blob_types/README.md | 17 ++
.../rust_array_blob_types/schema/schema-0 | 2 +-
.../rust_map_blob_types/README.md | 17 ++
.../rust_map_blob_types/schema/schema-0 | 2 +-
.../rust_time_types/README.md | 13 +
.../rust_time_types/schema/schema-0 | 2 +-
.../rust_types/README.md | 103 +++++++
.../rust_types/schema/schema-0 | 2 +-
.../rust_vector_types/README.md | 16 ++
.../rust_vector_types/schema/schema-0 | 2 +-
121 files changed, 1280 insertions(+), 391 deletions(-)
diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp
b/src/paimon/format/orc/orc_file_batch_reader.cpp
index d45cf4d9..526ff7cd 100644
--- a/src/paimon/format/orc/orc_file_batch_reader.cpp
+++ b/src/paimon/format/orc/orc_file_batch_reader.cpp
@@ -47,6 +47,36 @@
namespace paimon::orc {
namespace {
+bool IsStringTypeCompatible(const ::orc::Type* src_type, const ::orc::Type*
target_type) {
+ ::orc::TypeKind src_kind = src_type->getKind();
+ return (src_kind == ::orc::TypeKind::CHAR || src_kind ==
::orc::TypeKind::VARCHAR) &&
+ target_type->getKind() == ::orc::TypeKind::STRING;
+}
+
+bool IsCompleteTypeCompatible(const ::orc::Type* src_type, const ::orc::Type*
target_type) {
+ ::orc::TypeKind src_kind = src_type->getKind();
+ ::orc::TypeKind target_kind = target_type->getKind();
+ if (IsStringTypeCompatible(src_type, target_type)) {
+ return true;
+ }
+ if (src_kind != target_kind || src_type->getSubtypeCount() !=
target_type->getSubtypeCount()) {
+ return false;
+ }
+ if (src_type->getSubtypeCount() == 0) {
+ return src_type->toString() == target_type->toString();
+ }
+ for (uint64_t i = 0; i < src_type->getSubtypeCount(); i++) {
+ if (src_kind == ::orc::TypeKind::STRUCT &&
+ src_type->getFieldName(i) != target_type->getFieldName(i)) {
+ return false;
+ }
+ if (!IsCompleteTypeCompatible(src_type->getSubtype(i),
target_type->getSubtype(i))) {
+ return false;
+ }
+ }
+ return true;
+}
+
void CollectAllColumnIds(const ::orc::Type* type, std::vector<uint64_t>*
column_ids) {
column_ids->push_back(type->getColumnId());
for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
@@ -208,7 +238,7 @@ Status OrcFileBatchReader::CollectTargetColumnIds(const
::orc::Type* src_type,
std::vector<uint64_t>*
target_column_ids) {
auto src_kind = src_type->getKind();
auto target_kind = target_type->getKind();
- if (src_kind != target_kind) {
+ if (src_kind != target_kind && !IsStringTypeCompatible(src_type,
target_type)) {
return Status::Invalid(fmt::format("type kind mismatch: src {} vs
target {}",
src_type->toString(),
target_type->toString()));
}
@@ -235,7 +265,7 @@ Status OrcFileBatchReader::CollectTargetColumnIds(const
::orc::Type* src_type,
// Partial field recall inside list/map types is unsupported, so
the target must match
// the complete source subtree. Include the container and every
descendant because all
// of their streams are recalled by the ORC reader.
- if (src_type->toString() != target_type->toString()) {
+ if (!IsCompleteTypeCompatible(src_type, target_type)) {
return Status::Invalid(fmt::format("type mismatch: src {} vs
target {}",
src_type->toString(),
target_type->toString()));
}
diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp
b/src/paimon/format/orc/orc_file_batch_reader_test.cpp
index 3f57f1fb..e73e1c7e 100644
--- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp
+++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp
@@ -533,6 +533,20 @@ TEST_F(OrcFileBatchReaderTest,
TestCollectTargetColumnIdsPrimitiveMap) {
ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{1, 2, 3}));
}
+TEST_F(OrcFileBatchReaderTest,
TestCollectTargetColumnIdsCompatibleStringKinds) {
+ std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(
+ "struct<char_value:char(3),varchar_value:varchar(64),"
+ "nested:array<map<char(4),varchar(16)>>>");
+ std::unique_ptr<::orc::Type> target_type =
::orc::Type::buildTypeFromString(
+
"struct<char_value:string,varchar_value:string,nested:array<map<string,string>>>");
+ std::vector<uint64_t> target_column_ids;
+
+ ASSERT_OK(OrcFileBatchReader::CollectTargetColumnIds(src_type.get(),
target_type.get(),
+ &target_column_ids));
+ // char_value(1), varchar_value(2), nested-list(3), map(4), key(5),
value(6)
+ ASSERT_EQ(target_column_ids, (std::vector<uint64_t>{1, 2, 3, 4, 5, 6}));
+}
+
TEST_F(OrcFileBatchReaderTest, TestCollectTargetColumnIdsDeeplyNestedMap) {
std::string schema =
"struct<attributes:map<string,array<struct<score:double,tags:array<string>>>>>";
diff --git a/test/inte/paimon_read_compat_inte_test.cpp
b/test/inte/paimon_read_compat_inte_test.cpp
index a5595150..5948f0ef 100644
--- a/test/inte/paimon_read_compat_inte_test.cpp
+++ b/test/inte/paimon_read_compat_inte_test.cpp
@@ -17,6 +17,7 @@
* under the License.
*/
+#include <algorithm>
#include <cstdint>
#include <map>
#include <memory>
@@ -47,14 +48,28 @@ namespace {
const char kDatabaseName[] = "append_types_compatibility";
-std::vector<std::string> WriterPrefixes() {
- return {"python", "rust", "java"};
+struct CompatibilityParam {
+ std::string file_format;
+ std::string writer_prefix;
+ bool supports_variant;
+ bool supports_vector;
+ bool supports_nanosecond_timestamps;
+};
+
+std::vector<CompatibilityParam> CompatibilityParams() {
+ return {
+ {"parquet", "python", true, true, true}, {"parquet", "rust", true,
true, true},
+ {"parquet", "java", true, true, true}, {"orc", "java", false, false,
true},
+ {"avro", "java", false, true, false},
+ };
}
Result<std::shared_ptr<arrow::ChunkedArray>> ReadTable(
- const std::string& table_name, const std::vector<std::string>& field_names,
+ const std::string& file_format, const std::string& table_name,
+ const std::vector<std::string>& field_names,
const std::map<std::string, std::string>& options = {}) {
- std::string table_path = GetDataDir() + "/parquet/" + kDatabaseName +
".db/" + table_name;
+ std::string table_path =
+ GetDataDir() + "/" + file_format + "/" + kDatabaseName + ".db/" +
table_name;
ScanContextBuilder scan_context_builder(table_path);
scan_context_builder.SetOptions(options);
@@ -75,6 +90,13 @@ Result<std::shared_ptr<arrow::ChunkedArray>> ReadTable(
return ReadResultCollector::CollectResult(std::move(batch_reader));
}
+Result<std::shared_ptr<arrow::ChunkedArray>> ReadTable(
+ const CompatibilityParam& param, const std::string& table_name,
+ const std::vector<std::string>& field_names,
+ const std::map<std::string, std::string>& options = {}) {
+ return ReadTable(param.file_format, table_name, field_names, options);
+}
+
std::shared_ptr<arrow::StructArray> GetOnlyStructChunk(
const std::shared_ptr<arrow::ChunkedArray>& result) {
if (!result || result->num_chunks() != 1) {
@@ -134,10 +156,10 @@ void AssertVariantJsonAt(const
std::shared_ptr<arrow::StructArray>& variants, in
} // namespace
-class PaimonReadCompatInteTest : public ::testing::TestWithParam<std::string>
{};
+class PaimonReadCompatInteTest : public
::testing::TestWithParam<CompatibilityParam> {};
TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) {
- const std::string& writer_prefix = GetParam();
+ const CompatibilityParam& param = GetParam();
TimezoneGuard timezone_guard("Asia/Shanghai");
// Project every non-BLOB field from the main compatibility table in
schema order.
@@ -175,8 +197,7 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues)
{
"f_timestamp_ltz_3",
"f_timestamp_ltz_6",
"f_timestamp_ltz_9",
- // VARIANT and nested container types.
- "f_variant",
+ // Nested container types supported by every fixture.
"f_array_int",
"f_map_string_bigint",
"f_row",
@@ -185,12 +206,18 @@ TEST_P(PaimonReadCompatInteTest,
ReadsCompatibleTypeValues) {
"f_map_array",
"f_array_row",
"f_map_row",
- "f_deep_row",
- "f_array_variant",
- "f_map_variant",
};
+ if (!param.supports_nanosecond_timestamps) {
+ fields.erase(std::remove(fields.begin(), fields.end(),
"f_timestamp_9"), fields.end());
+ fields.erase(std::remove(fields.begin(), fields.end(),
"f_timestamp_ltz_9"), fields.end());
+ }
+ if (param.supports_variant) {
+ auto nested_types = std::find(fields.begin(), fields.end(),
"f_array_int");
+ fields.insert(nested_types, "f_variant");
+ fields.insert(fields.end(), {"f_deep_row", "f_array_variant",
"f_map_variant"});
+ }
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
- ReadTable(writer_prefix + "_types", fields));
+ ReadTable(param, param.writer_prefix + "_types",
fields));
std::shared_ptr<arrow::StructArray> rows = GetOnlyStructChunk(result);
ASSERT_TRUE(rows);
ASSERT_EQ(rows->length(), 3);
@@ -208,7 +235,9 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues)
{
AssertFieldEqualsJson(rows, "f_double", arrow::float64(), "[-12345.6789,
null, null]");
// Character and binary strings, including empty and embedded-null values.
- AssertFieldEqualsJson(rows, "f_char", arrow::utf8(), R"(["char", null,
""])");
+ AssertFieldEqualsJson(
+ rows, "f_char", arrow::utf8(),
+ param.file_format == "orc" ? R"(["char ", null, " "])" :
R"(["char", null, ""])");
AssertFieldEqualsJson(rows, "f_varchar", arrow::utf8(), R"(["varchar-中文",
null, ""])");
AssertFieldEqualsJson(rows, "f_string", arrow::utf8(), R"(["pypaimon
2.0.0", null, ""])");
AssertFieldEqualsJson(rows, "f_binary", arrow::binary(), R"(["12345678",
null, "ABCDEFGH"])");
@@ -237,8 +266,10 @@ TEST_P(PaimonReadCompatInteTest,
ReadsCompatibleTypeValues) {
R"(["2024-02-29 12:34:56.123", null, null])");
AssertFieldEqualsJson(rows, "f_timestamp_6",
arrow::timestamp(arrow::TimeUnit::MICRO),
R"(["2024-02-29 12:34:56.123456", null, null])");
- AssertFieldEqualsJson(rows, "f_timestamp_9",
arrow::timestamp(arrow::TimeUnit::NANO),
- R"(["2024-02-29 12:34:56.123456000", null, null])");
+ if (param.supports_nanosecond_timestamps) {
+ AssertFieldEqualsJson(rows, "f_timestamp_9",
arrow::timestamp(arrow::TimeUnit::NANO),
+ R"(["2024-02-29 12:34:56.123456000", null,
null])");
+ }
AssertFieldEqualsJson(rows, "f_timestamp_ltz_0",
arrow::timestamp(arrow::TimeUnit::SECOND,
"Asia/Shanghai"),
R"(["2024-02-29 12:34:56", null, null])");
@@ -248,26 +279,39 @@ TEST_P(PaimonReadCompatInteTest,
ReadsCompatibleTypeValues) {
AssertFieldEqualsJson(rows, "f_timestamp_ltz_6",
arrow::timestamp(arrow::TimeUnit::MICRO,
"Asia/Shanghai"),
R"(["2024-02-29 12:34:56.123456", null, null])");
- AssertFieldEqualsJson(rows, "f_timestamp_ltz_9",
- arrow::timestamp(arrow::TimeUnit::NANO,
"Asia/Shanghai"),
- R"(["2024-02-29 12:34:56.123456000", null, null])");
+ if (param.supports_nanosecond_timestamps) {
+ AssertFieldEqualsJson(rows, "f_timestamp_ltz_9",
+ arrow::timestamp(arrow::TimeUnit::NANO,
"Asia/Shanghai"),
+ R"(["2024-02-29 12:34:56.123456000", null,
null])");
+ }
// First-level ARRAY, MAP, and ROW values with null and empty containers.
AssertFieldEqualsJson(rows, "f_array_int", arrow::list(arrow::int32()),
"[[1, null, 3], null, []]");
AssertFieldEqualsJson(rows, "f_map_string_bigint",
arrow::map(arrow::utf8(), arrow::int64()),
R"([[ ["one", 1], ["null", null] ], null, []])");
- std::shared_ptr<arrow::DataType> row_type = arrow::struct_({
- arrow::field("nested_int", arrow::int32()),
- arrow::field("nested_string", arrow::utf8()),
- arrow::field("nested_decimal", arrow::decimal128(19, 4)),
- arrow::field("nested_timestamp",
arrow::timestamp(arrow::TimeUnit::NANO)),
- arrow::field("nested_timestamp_ltz",
- arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai")),
- });
- AssertFieldEqualsJson(
- rows, "f_row", row_type,
- R"([[7, "nested", "123456789012345.6789", "2024-02-29
12:34:56.123456000", "2024-02-29 12:34:56.123456000"], null, null])");
+ std::shared_ptr<arrow::DataType> row_type;
+ std::string row_json;
+ if (param.supports_nanosecond_timestamps) {
+ row_type = arrow::struct_({
+ arrow::field("nested_int", arrow::int32()),
+ arrow::field("nested_string", arrow::utf8()),
+ arrow::field("nested_decimal", arrow::decimal128(19, 4)),
+ arrow::field("nested_timestamp",
arrow::timestamp(arrow::TimeUnit::NANO)),
+ arrow::field("nested_timestamp_ltz",
+ arrow::timestamp(arrow::TimeUnit::NANO,
"Asia/Shanghai")),
+ });
+ row_json =
+ R"([[7, "nested", "123456789012345.6789", "2024-02-29
12:34:56.123456000", "2024-02-29 12:34:56.123456000"], null, null])";
+ } else {
+ row_type = arrow::struct_({
+ arrow::field("nested_int", arrow::int32()),
+ arrow::field("nested_string", arrow::utf8()),
+ arrow::field("nested_decimal", arrow::decimal128(19, 4)),
+ });
+ row_json = R"([[7, "nested", "123456789012345.6789"], null, null])";
+ }
+ AssertFieldEqualsJson(rows, "f_row", row_type, row_json);
// Nested ARRAY/MAP combinations and containers whose values are ROWs.
AssertFieldEqualsJson(rows, "f_array_array_int",
arrow::list(arrow::list(arrow::int32())),
@@ -292,81 +336,88 @@ TEST_P(PaimonReadCompatInteTest,
ReadsCompatibleTypeValues) {
rows, "f_map_row", map_row_type,
R"([[ ["first", [true, "2024-02-29 12:34:56.123456"]], ["second",
null] ], null, []])");
- // Top-level VARIANT values are compared as normalized JSON.
- const std::string object_variant_json =
-
R"({"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}})";
- const std::string array_variant_json = R"([1,"two",false,{"k":"v"}])";
- std::shared_ptr<arrow::StructArray> variants =
-
std::dynamic_pointer_cast<arrow::StructArray>(rows->GetFieldByName("f_variant"));
- ASSERT_TRUE(variants);
- AssertVariantJsonAt(variants, 0, object_variant_json);
- ASSERT_TRUE(variants->IsNull(1));
- ASSERT_TRUE(variants->IsNull(2));
-
- // Deeply nested ROW -> ARRAY<ROW<VARIANT>> and ROW -> MAP values.
- std::shared_ptr<arrow::StructArray> deep_rows =
-
std::dynamic_pointer_cast<arrow::StructArray>(rows->GetFieldByName("f_deep_row"));
- ASSERT_TRUE(deep_rows);
- ASSERT_FALSE(deep_rows->IsNull(0));
- ASSERT_TRUE(deep_rows->IsNull(1));
- ASSERT_TRUE(deep_rows->IsNull(2));
- std::shared_ptr<arrow::ListArray> children =
-
std::dynamic_pointer_cast<arrow::ListArray>(deep_rows->GetFieldByName("children"));
- ASSERT_TRUE(children);
- ASSERT_EQ(children->value_length(0), 2);
- std::shared_ptr<arrow::StructArray> child_rows =
- std::dynamic_pointer_cast<arrow::StructArray>(children->values());
- ASSERT_TRUE(child_rows);
- AssertArrayEqualsJson(child_rows->GetFieldByName("leaf_id"),
"f_deep_row.children.leaf_id",
- "[10, 11]");
- std::shared_ptr<arrow::StructArray> leaf_variants =
-
std::dynamic_pointer_cast<arrow::StructArray>(child_rows->GetFieldByName("leaf_variant"));
- ASSERT_TRUE(leaf_variants);
- AssertVariantJsonAt(leaf_variants, 0, array_variant_json);
- ASSERT_TRUE(leaf_variants->IsNull(1));
- std::shared_ptr<arrow::MapArray> labels =
-
std::dynamic_pointer_cast<arrow::MapArray>(deep_rows->GetFieldByName("labels"));
- ASSERT_TRUE(labels);
- AssertArrayEqualsJson(labels->Slice(0, 1), "f_deep_row.labels",
- R"([[ ["language", "python"], ["format", "parquet"]
]])");
-
- // ARRAY<VARIANT> values, including null elements and empty arrays.
- std::shared_ptr<arrow::ListArray> array_variants =
-
std::dynamic_pointer_cast<arrow::ListArray>(rows->GetFieldByName("f_array_variant"));
- ASSERT_TRUE(array_variants);
- ASSERT_EQ(array_variants->value_length(0), 3);
- ASSERT_TRUE(array_variants->IsNull(1));
- ASSERT_EQ(array_variants->value_length(2), 0);
- std::shared_ptr<arrow::StructArray> array_variant_values =
-
std::dynamic_pointer_cast<arrow::StructArray>(array_variants->values());
- ASSERT_TRUE(array_variant_values);
- AssertVariantJsonAt(array_variant_values, 0, object_variant_json);
- ASSERT_TRUE(array_variant_values->IsNull(1));
- AssertVariantJsonAt(array_variant_values, 2, array_variant_json);
-
- // MAP<STRING, VARIANT> values and their key/value alignment.
- std::shared_ptr<arrow::MapArray> map_variants =
-
std::dynamic_pointer_cast<arrow::MapArray>(rows->GetFieldByName("f_map_variant"));
- ASSERT_TRUE(map_variants);
- ASSERT_EQ(map_variants->value_length(0), 2);
- ASSERT_TRUE(map_variants->IsNull(1));
- ASSERT_EQ(map_variants->value_length(2), 0);
- AssertArrayEqualsJson(map_variants->keys(), "f_map_variant.keys",
R"(["object", "array"])");
- std::shared_ptr<arrow::StructArray> map_variant_values =
- std::dynamic_pointer_cast<arrow::StructArray>(map_variants->items());
- ASSERT_TRUE(map_variant_values);
- AssertVariantJsonAt(map_variant_values, 0, object_variant_json);
- AssertVariantJsonAt(map_variant_values, 1, array_variant_json);
+ if (param.supports_variant) {
+ // Top-level VARIANT values are compared as normalized JSON.
+ const std::string object_variant_json =
+
R"({"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}})";
+ const std::string array_variant_json = R"([1,"two",false,{"k":"v"}])";
+ std::shared_ptr<arrow::StructArray> variants =
+
std::dynamic_pointer_cast<arrow::StructArray>(rows->GetFieldByName("f_variant"));
+ ASSERT_TRUE(variants);
+ AssertVariantJsonAt(variants, 0, object_variant_json);
+ ASSERT_TRUE(variants->IsNull(1));
+ ASSERT_TRUE(variants->IsNull(2));
+
+ // Deeply nested ROW -> ARRAY<ROW<VARIANT>> and ROW -> MAP values.
+ std::shared_ptr<arrow::StructArray> deep_rows =
+
std::dynamic_pointer_cast<arrow::StructArray>(rows->GetFieldByName("f_deep_row"));
+ ASSERT_TRUE(deep_rows);
+ ASSERT_FALSE(deep_rows->IsNull(0));
+ ASSERT_TRUE(deep_rows->IsNull(1));
+ ASSERT_TRUE(deep_rows->IsNull(2));
+ std::shared_ptr<arrow::ListArray> children =
+
std::dynamic_pointer_cast<arrow::ListArray>(deep_rows->GetFieldByName("children"));
+ ASSERT_TRUE(children);
+ ASSERT_EQ(children->value_length(0), 2);
+ std::shared_ptr<arrow::StructArray> child_rows =
+ std::dynamic_pointer_cast<arrow::StructArray>(children->values());
+ ASSERT_TRUE(child_rows);
+ AssertArrayEqualsJson(child_rows->GetFieldByName("leaf_id"),
"f_deep_row.children.leaf_id",
+ "[10, 11]");
+ std::shared_ptr<arrow::StructArray> leaf_variants =
+ std::dynamic_pointer_cast<arrow::StructArray>(
+ child_rows->GetFieldByName("leaf_variant"));
+ ASSERT_TRUE(leaf_variants);
+ AssertVariantJsonAt(leaf_variants, 0, array_variant_json);
+ ASSERT_TRUE(leaf_variants->IsNull(1));
+ std::shared_ptr<arrow::MapArray> labels =
+
std::dynamic_pointer_cast<arrow::MapArray>(deep_rows->GetFieldByName("labels"));
+ ASSERT_TRUE(labels);
+ AssertArrayEqualsJson(labels->Slice(0, 1), "f_deep_row.labels",
+ R"([[ ["language", "python"], ["format",
"parquet"] ]])");
+
+ // ARRAY<VARIANT> values, including null elements and empty arrays.
+ std::shared_ptr<arrow::ListArray> array_variants =
+
std::dynamic_pointer_cast<arrow::ListArray>(rows->GetFieldByName("f_array_variant"));
+ ASSERT_TRUE(array_variants);
+ ASSERT_EQ(array_variants->value_length(0), 3);
+ ASSERT_TRUE(array_variants->IsNull(1));
+ ASSERT_EQ(array_variants->value_length(2), 0);
+ std::shared_ptr<arrow::StructArray> array_variant_values =
+
std::dynamic_pointer_cast<arrow::StructArray>(array_variants->values());
+ ASSERT_TRUE(array_variant_values);
+ AssertVariantJsonAt(array_variant_values, 0, object_variant_json);
+ ASSERT_TRUE(array_variant_values->IsNull(1));
+ AssertVariantJsonAt(array_variant_values, 2, array_variant_json);
+
+ // MAP<STRING, VARIANT> values and their key/value alignment.
+ std::shared_ptr<arrow::MapArray> map_variants =
+
std::dynamic_pointer_cast<arrow::MapArray>(rows->GetFieldByName("f_map_variant"));
+ ASSERT_TRUE(map_variants);
+ ASSERT_EQ(map_variants->value_length(0), 2);
+ ASSERT_TRUE(map_variants->IsNull(1));
+ ASSERT_EQ(map_variants->value_length(2), 0);
+ AssertArrayEqualsJson(map_variants->keys(), "f_map_variant.keys",
R"(["object", "array"])");
+ std::shared_ptr<arrow::StructArray> map_variant_values =
+
std::dynamic_pointer_cast<arrow::StructArray>(map_variants->items());
+ ASSERT_TRUE(map_variant_values);
+ AssertVariantJsonAt(map_variant_values, 0, object_variant_json);
+ AssertVariantJsonAt(map_variant_values, 1, array_variant_json);
+ }
}
TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) {
- const std::string& writer_prefix = GetParam();
+ const CompatibilityParam& param = GetParam();
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
- ReadTable(writer_prefix + "_types", {"f_blob",
"f_blob_descriptor"},
- {{Options::FILE_SYSTEM, "local"},
- {Options::BLOB_AS_DESCRIPTOR, "true"},
- {Options::BLOB_VIEW_RESOLVE_ENABLED,
"false"}}));
+ std::vector<std::string> blob_fields = {"f_blob", "f_blob_descriptor"};
+ std::map<std::string, std::string> blob_descriptor_options;
+ blob_descriptor_options[Options::FILE_SYSTEM] = "local";
+ blob_descriptor_options[Options::BLOB_AS_DESCRIPTOR] = "true";
+ blob_descriptor_options[Options::BLOB_VIEW_RESOLVE_ENABLED] = "false";
+
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<arrow::ChunkedArray> result,
+ ReadTable(param, param.writer_prefix + "_types", blob_fields,
blob_descriptor_options));
std::shared_ptr<arrow::StructArray> rows = GetOnlyStructChunk(result);
ASSERT_TRUE(rows);
ASSERT_EQ(rows->length(), 3);
@@ -392,18 +443,18 @@ TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) {
ASSERT_EQ(empty_descriptor->Length(), 0);
// Resolve the external BLOB and validate its payload instead of its
descriptor.
- std::map<std::string, std::string> blob_value_options = {
- {Options::FILE_SYSTEM, "local"},
- {Options::BLOB_AS_DESCRIPTOR, "false"},
- };
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray>
blob_value_result,
- ReadTable(writer_prefix + "_types", {"f_blob"},
blob_value_options));
+ std::map<std::string, std::string> blob_value_options;
+ blob_value_options[Options::FILE_SYSTEM] = "local";
+ blob_value_options[Options::BLOB_AS_DESCRIPTOR] = "false";
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<arrow::ChunkedArray> blob_value_result,
+ ReadTable(param, param.writer_prefix + "_types", {"f_blob"},
blob_value_options));
std::shared_ptr<arrow::StructArray> blob_value_rows =
GetOnlyStructChunk(blob_value_result);
ASSERT_TRUE(blob_value_rows);
AssertFieldEqualsJson(blob_value_rows, "f_blob", arrow::large_binary(),
R"(["ordinary blob payload from pypaimon 2.0.0",
null, ""])");
- // Inline BLOB descriptors exercise compatible binary types restored from
ARROW:schema.
+ // Inline BLOB descriptors exercise each file format's compatible binary
representation.
std::shared_ptr<arrow::LargeBinaryArray> inline_blobs =
std::dynamic_pointer_cast<arrow::LargeBinaryArray>(
rows->GetFieldByName("f_blob_descriptor"));
@@ -423,12 +474,12 @@ TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) {
// Python and Java store MAP<STRING, BLOB> values in standard separate
BLOB files. Validate
// resolved payloads, null values, and an empty map. Rust stores raw
values inline in Parquet,
// which is not a compatible Paimon BLOB representation and is asserted
separately below.
- if (writer_prefix == "rust") {
+ if (param.writer_prefix == "rust") {
return;
}
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<arrow::ChunkedArray> map_blob_result,
- ReadTable(writer_prefix + "_map_blob_types", {"id", "f_map_blob"},
blob_value_options));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> map_blob_result,
+ ReadTable(param, param.writer_prefix +
"_map_blob_types",
+ {"id", "f_map_blob"}, blob_value_options));
std::shared_ptr<arrow::StructArray> map_blob_rows =
GetOnlyStructChunk(map_blob_result);
ASSERT_TRUE(map_blob_rows);
ASSERT_EQ(map_blob_rows->length(), 3);
@@ -440,12 +491,15 @@ TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) {
TEST(PaimonReadCompatInteStandaloneTest, RejectsNonStandardRustMapBlob) {
ASSERT_NOK_WITH_MSG(
- ReadTable("rust_map_blob_types", {"f_map_blob"}),
+ ReadTable("parquet", "rust_map_blob_types", {"f_map_blob"}),
"Parquet does not support partial projection inside list/map: src
map<string, binary");
}
TEST_P(PaimonReadCompatInteTest, ReadsVectorValues) {
- const std::string& writer_prefix = GetParam();
+ const CompatibilityParam& param = GetParam();
+ if (!param.supports_vector) {
+ return;
+ }
// Validate every supported VECTOR element type.
std::vector<std::string> vector_fields = {
@@ -453,7 +507,7 @@ TEST_P(PaimonReadCompatInteTest, ReadsVectorValues) {
"f_vector_int", "f_vector_bigint", "f_vector_float",
"f_vector_double",
};
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> vector_result,
- ReadTable(writer_prefix + "_vector_types",
vector_fields));
+ ReadTable(param, param.writer_prefix +
"_vector_types", vector_fields));
std::shared_ptr<arrow::StructArray> vector_rows =
GetOnlyStructChunk(vector_result);
ASSERT_TRUE(vector_rows);
ASSERT_EQ(vector_rows->length(), 2);
@@ -479,9 +533,10 @@ TEST_P(PaimonReadCompatInteTest, ReadsVectorValues) {
"[[1.125, -2.25, 3.5], [4.125, 5.25, 6.5]]");
}
-INSTANTIATE_TEST_SUITE_P(Writers, PaimonReadCompatInteTest,
::testing::ValuesIn(WriterPrefixes()),
- [](const ::testing::TestParamInfo<std::string>& info)
{
- return info.param;
+INSTANTIATE_TEST_SUITE_P(FormatsAndWriters, PaimonReadCompatInteTest,
+ ::testing::ValuesIn(CompatibilityParams()),
+ [](const
::testing::TestParamInfo<CompatibilityParam>& info) {
+ return info.param.file_format + "_" +
info.param.writer_prefix;
});
struct UnsupportedReadCase {
@@ -492,6 +547,7 @@ struct UnsupportedReadCase {
};
struct UnsupportedReadParam {
+ std::string file_format;
std::string writer_prefix;
UnsupportedReadCase read_case;
};
@@ -500,7 +556,8 @@ class PaimonUnsupportedTypeInteTest : public
::testing::TestWithParam<Unsupporte
TEST_P(PaimonUnsupportedTypeInteTest, ReportsExpectedError) {
const UnsupportedReadParam& test_case = GetParam();
- ASSERT_NOK_WITH_MSG(ReadTable(test_case.writer_prefix + "_" +
test_case.read_case.table_suffix,
+ ASSERT_NOK_WITH_MSG(ReadTable(test_case.file_format,
+ test_case.writer_prefix + "_" +
test_case.read_case.table_suffix,
{test_case.read_case.field_name}),
test_case.read_case.expected_error);
}
@@ -515,9 +572,14 @@ std::vector<UnsupportedReadParam> UnsupportedReadParams() {
{"TimePrecision9", "time_types", "f_time_9", "Unsupported type: TIME"},
};
std::vector<UnsupportedReadParam> result;
- for (const std::string& writer_prefix : WriterPrefixes()) {
+ for (const CompatibilityParam& param : CompatibilityParams()) {
for (const UnsupportedReadCase& read_case : read_cases) {
- result.push_back({writer_prefix, read_case});
+ // Java Avro cannot create TIME(6/9), so its negative table
contains TIME(0/3) only.
+ if (param.file_format == "avro" &&
+ (read_case.name == "TimePrecision6" || read_case.name ==
"TimePrecision9")) {
+ continue;
+ }
+ result.push_back({param.file_format, param.writer_prefix,
read_case});
}
}
return result;
@@ -526,7 +588,8 @@ std::vector<UnsupportedReadParam> UnsupportedReadParams() {
INSTANTIATE_TEST_SUITE_P(UnsupportedTypes, PaimonUnsupportedTypeInteTest,
::testing::ValuesIn(UnsupportedReadParams()),
[](const
::testing::TestParamInfo<UnsupportedReadParam>& info) {
- return info.param.writer_prefix +
info.param.read_case.name;
+ return info.param.file_format + "_" +
info.param.writer_prefix + "_" +
+ info.param.read_case.name;
});
} // namespace paimon::test
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/README.md
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/README.md
new file mode 100644
index 00000000..d9cd7bf0
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_array_blob ARRAY<BLOB>.
+
+Options:
+
+bucket = -1
+file.format = avro
+blob-field = f_array_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, ["array-blob-value", null, ""])
+Add: (2, null)
+Add: (3, [])
+
+Paimon C++ is expected to reject this table while ARRAY<BLOB> is unsupported.
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/bucket-0/data-e450b386-992c-4934-a77c-428b16f00082-0.avro
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/bucket-0/data-e450b386-992c-4934-a77c-428b16f00082-0.avro
new file mode 100644
index 00000000..0186940f
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/bucket-0/data-e450b386-992c-4934-a77c-428b16f00082-0.avro
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/bucket-0/data-e450b386-992c-4934-a77c-428b16f00082-1.blob
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/bucket-0/data-e450b386-992c-4934-a77c-428b16f00082-1.blob
new file mode 100644
index 00000000..a5b24e12
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/bucket-0/data-e450b386-992c-4934-a77c-428b16f00082-1.blob
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-1d608455-d871-46a3-ad5c-c6359f77c02c-0
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-1d608455-d871-46a3-ad5c-c6359f77c02c-0
new file mode 100644
index 00000000..76718ec7
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-1d608455-d871-46a3-ad5c-c6359f77c02c-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-0
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-0
new file mode 100644
index 00000000..24a7b07a
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-1
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-1
new file mode 100644
index 00000000..012cb960
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/schema/schema-0
similarity index 83%
copy from
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
copy to
test/test_data/avro/append_types_compatibility.db/java_array_blob_types/schema/schema-0
index 0bce94c9..0ec58c84 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
+++
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/schema/schema-0
@@ -21,8 +21,7 @@
"blob-field" : "f_array_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "avro"
},
- "timeMillis" : 1788859707631
-}
\ No newline at end of file
+ "timeMillis" : 1789549988349
+}
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..0abfb546
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "5a95869a-09be-4cd5-b769-faa937e03b11",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-d1d4bb47-ac3a-43a1-b20a-16384869a4b6-1",
+ "deltaManifestListSize" : 1261,
+ "commitUser" : "88af33b1-2dd5-4c11-9edd-7d13cc81d1cb",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789549988372,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/README.md
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/README.md
new file mode 100644
index 00000000..43810f80
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_map_blob MAP<STRING NOT NULL, BLOB>.
+
+Options:
+
+bucket = -1
+file.format = avro
+blob-field = f_map_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, {"left":"blob-map-left","right":null})
+Add: (2, null)
+Add: (3, {})
+
+This writer stores map BLOB values in Paimon's external BLOB representation.
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/bucket-0/data-67ea4504-2d8c-449a-b66d-e51f7866489a-0.avro
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/bucket-0/data-67ea4504-2d8c-449a-b66d-e51f7866489a-0.avro
new file mode 100644
index 00000000..4f67788c
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/bucket-0/data-67ea4504-2d8c-449a-b66d-e51f7866489a-0.avro
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/bucket-0/data-67ea4504-2d8c-449a-b66d-e51f7866489a-1.blob
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/bucket-0/data-67ea4504-2d8c-449a-b66d-e51f7866489a-1.blob
new file mode 100644
index 00000000..bf82127d
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/bucket-0/data-67ea4504-2d8c-449a-b66d-e51f7866489a-1.blob
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-d901117d-19e9-45d7-9627-7b385635001e-0
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-d901117d-19e9-45d7-9627-7b385635001e-0
new file mode 100644
index 00000000..0838f067
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-d901117d-19e9-45d7-9627-7b385635001e-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-0
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-0
new file mode 100644
index 00000000..3482cd16
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-1
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-1
new file mode 100644
index 00000000..7832b4f0
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/schema/schema-0
similarity index 83%
copy from
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
copy to
test/test_data/avro/append_types_compatibility.db/java_map_blob_types/schema/schema-0
index 340c0ade..084bbcdd 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
+++
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/schema/schema-0
@@ -22,8 +22,7 @@
"blob-field" : "f_map_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "avro"
},
- "timeMillis" : 1788859708882
-}
\ No newline at end of file
+ "timeMillis" : 1789549988375
+}
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..f7dc836f
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "c3c36bdf-5a2e-4be2-b4b5-12a047bd98bb",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-d38e0d87-461f-4cd4-aabf-d47141e679b7-1",
+ "deltaManifestListSize" : 1261,
+ "commitUser" : "c24edb81-3117-4136-8f93-65311bbf6b79",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789549988393,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/README.md
b/test/test_data/avro/append_types_compatibility.db/java_time_types/README.md
new file mode 100644
index 00000000..c62c1438
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_time_types/README.md
@@ -0,0 +1,12 @@
+Schema:
+
+id INT NOT NULL
+f_time_0 TIME(0)
+f_time_3 TIME(3)
+
+Options: bucket = -1, file.format = avro. The table is unpartitioned and has
no primary
+key.
+
+Add: (1, 12:34:56, 12:34:56.123)
+Add: (2, null, null)
+Add: (3, null, null)
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/bucket-0/data-552bc110-1000-47fd-91f8-d8c988911595-0.avro
b/test/test_data/avro/append_types_compatibility.db/java_time_types/bucket-0/data-552bc110-1000-47fd-91f8-d8c988911595-0.avro
new file mode 100644
index 00000000..0d926d19
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_time_types/bucket-0/data-552bc110-1000-47fd-91f8-d8c988911595-0.avro
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-3d3ad23a-e678-485a-aeae-fb733e7d970a-0
b/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-3d3ad23a-e678-485a-aeae-fb733e7d970a-0
new file mode 100644
index 00000000..9e4316ba
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-3d3ad23a-e678-485a-aeae-fb733e7d970a-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-0
b/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-0
new file mode 100644
index 00000000..8a42d2fc
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-1
b/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-1
new file mode 100644
index 00000000..ae30edbe
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_time_types/manifest/manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
b/test/test_data/avro/append_types_compatibility.db/java_time_types/schema/schema-0
similarity index 57%
copy from
test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
copy to
test/test_data/avro/append_types_compatibility.db/java_time_types/schema/schema-0
index 02f0fbff..2f50e251 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
+++
b/test/test_data/avro/append_types_compatibility.db/java_time_types/schema/schema-0
@@ -13,22 +13,13 @@
"id" : 2,
"name" : "f_time_3",
"type" : "TIME(3)"
- }, {
- "id" : 3,
- "name" : "f_time_6",
- "type" : "TIME(6)"
- }, {
- "id" : 4,
- "name" : "f_time_9",
- "type" : "TIME(9)"
} ],
- "highestFieldId" : 4,
+ "highestFieldId" : 2,
"partitionKeys" : [ ],
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "avro"
},
- "timeMillis" : 1788853436258
-}
\ No newline at end of file
+ "timeMillis" : 1789549988395
+}
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/EARLIEST
b/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/LATEST
b/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/snapshot-1
b/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/snapshot-1
new file mode 100644
index 00000000..deac9189
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_time_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "7a749721-b43a-479c-8267-5cf7202ae260",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-7e7bdeb2-7827-4a05-aed8-e1aea50f9144-1",
+ "deltaManifestListSize" : 1261,
+ "commitUser" : "53b53c25-6a25-4df2-a0cc-10f050fb32c6",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789549988409,
+ "totalRecordCount" : 3,
+ "deltaRecordCount" : 3,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/README.md
b/test/test_data/avro/append_types_compatibility.db/java_types/README.md
new file mode 100644
index 00000000..8e437015
--- /dev/null
+++ b/test/test_data/avro/append_types_compatibility.db/java_types/README.md
@@ -0,0 +1,92 @@
+Schema:
+
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_int INT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_string STRING
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_bytes BYTES
+f_blob BLOB
+f_blob_descriptor BLOB
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_9_2 DECIMAL(9, 2)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_decimal_38_38 DECIMAL(38, 38)
+f_date DATE
+f_timestamp_0 TIMESTAMP(0)
+f_timestamp_3 TIMESTAMP(3)
+f_timestamp_6 TIMESTAMP(6)
+f_timestamp_ltz_0 TIMESTAMP_LTZ(0)
+f_timestamp_ltz_3 TIMESTAMP_LTZ(3)
+f_timestamp_ltz_6 TIMESTAMP_LTZ(6)
+f_array_int ARRAY<INT>
+f_map_string_bigint MAP<STRING NOT NULL, BIGINT>
+f_row ROW<nested_int INT, nested_string STRING, nested_decimal DECIMAL(19, 4)>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING NOT NULL, INT>>
+f_map_array MAP<STRING NOT NULL, ARRAY<INT>>
+f_array_row ARRAY<ROW<name STRING, score DECIMAL(9, 2)>>
+f_map_row MAP<STRING NOT NULL, ROW<enabled BOOLEAN, event_time TIMESTAMP(6)>>
+
+Options:
+
+bucket = -1
+file.format = avro
+blob-field = f_blob,f_blob_descriptor
+blob-descriptor-field = f_blob_descriptor
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table has no partition key or primary key. Values below are listed as
[id=1, id=2, id=3].
+Timestamps and local-zoned timestamps are shown in Asia/Shanghai; binary
values are hexadecimal.
+
+id: [1, 2, 3]
+f_boolean: [true, null, null]
+f_tinyint: [-8, null, null]
+f_smallint: [1234, null, null]
+f_int: [-123456, null, null]
+f_bigint: [9223372036854770000, null, null]
+f_float: [1.25, null, null]
+f_double: [-12345.6789, null, null]
+f_char: ["char", null, ""]
+f_varchar: ["varchar-中文", null, ""]
+f_string: ["pypaimon 2.0.0", null, ""]
+f_binary: [0x3132333435363738, null, 0x4142434445464748]
+f_varbinary: [0x76617262696e6172790076616c7565, null, 0x]
+f_bytes: [0x62797465730076616c7565, null, 0x]
+f_blob: ["ordinary blob payload from pypaimon 2.0.0", null, ""]
+f_blob_descriptor:
[Blob("file:///nonexistent/pypaimon-all-types-external-blob.bin", 7, 11), null,
null]
+f_decimal_1_0: [9, null, null]
+f_decimal_9_2: [1234567.89, null, null]
+f_decimal_18_2: [1234567890123456.78, null, null]
+f_decimal_19_2: [12345678901234567.89, null, null]
+f_decimal_38_18: [12345678901234567890.123456789012345678, null, null]
+f_decimal_38_38: [0.12345678901234567890123456789012345678, null, null]
+f_date: [2024-02-29, null, null]
+f_timestamp_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_ltz_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_ltz_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_ltz_6: [2024-02-29 12:34:56.123456, null, null]
+f_array_int: [[1, null, 3], null, []]
+f_map_string_bigint: [{"one":1,"null":null}, null, {}]
+f_row: [(7,"nested",123456789012345.6789), null, null]
+f_array_array_int: [[[1,2],null,[]], null, []]
+f_array_map: [[{"a":1,"b":null},null,{}], null, []]
+f_map_array: [{"numbers":[1,null,3],"empty":[]}, null, {}]
+f_array_row: [[("alice",99.50),null,("bob",null)], null, []]
+f_map_row: [{"first":(true,2024-02-29 12:34:56.123456),"second":null}, null,
{}]
+
+Java Avro supports timestamp precision only through 6. VARIANT and its nested
forms are omitted
+because Java cannot derive an Avro schema for them.
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/bucket-0/data-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-0.avro
b/test/test_data/avro/append_types_compatibility.db/java_types/bucket-0/data-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-0.avro
new file mode 100644
index 00000000..d6c02f2c
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_types/bucket-0/data-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-0.avro
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/bucket-0/data-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-1.blob
b/test/test_data/avro/append_types_compatibility.db/java_types/bucket-0/data-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-1.blob
new file mode 100644
index 00000000..6561d11d
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_types/bucket-0/data-6fb2ed11-0ae4-4fe3-9bf8-8e1a54ecf8b7-1.blob
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-ad942b8a-bdb3-4a5d-a6ab-637f8d2a9ba5-0
b/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-ad942b8a-bdb3-4a5d-a6ab-637f8d2a9ba5-0
new file mode 100644
index 00000000..710e3aea
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-ad942b8a-bdb3-4a5d-a6ab-637f8d2a9ba5-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-0
b/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-0
new file mode 100644
index 00000000..49628b0a
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-1
b/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-1
new file mode 100644
index 00000000..ca3b4108
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_types/manifest/manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
b/test/test_data/avro/append_types_compatibility.db/java_types/schema/schema-0
similarity index 69%
copy from
test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
copy to
test/test_data/avro/append_types_compatibility.db/java_types/schema/schema-0
index 554c947a..5d099b9b 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
+++
b/test/test_data/avro/append_types_compatibility.db/java_types/schema/schema-0
@@ -107,37 +107,25 @@
"type" : "TIMESTAMP(6)"
}, {
"id" : 26,
- "name" : "f_timestamp_9",
- "type" : "TIMESTAMP(9)"
- }, {
- "id" : 27,
"name" : "f_timestamp_ltz_0",
"type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE"
}, {
- "id" : 28,
+ "id" : 27,
"name" : "f_timestamp_ltz_3",
"type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE"
}, {
- "id" : 29,
+ "id" : 28,
"name" : "f_timestamp_ltz_6",
"type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE"
}, {
- "id" : 30,
- "name" : "f_timestamp_ltz_9",
- "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE"
- }, {
- "id" : 31,
- "name" : "f_variant",
- "type" : "VARIANT"
- }, {
- "id" : 32,
+ "id" : 29,
"name" : "f_array_int",
"type" : {
"type" : "ARRAY",
"element" : "INT"
}
}, {
- "id" : 33,
+ "id" : 30,
"name" : "f_map_string_bigint",
"type" : {
"type" : "MAP",
@@ -145,34 +133,26 @@
"value" : "BIGINT"
}
}, {
- "id" : 34,
+ "id" : 31,
"name" : "f_row",
"type" : {
"type" : "ROW",
"fields" : [ {
- "id" : 35,
+ "id" : 32,
"name" : "nested_int",
"type" : "INT"
}, {
- "id" : 36,
+ "id" : 33,
"name" : "nested_string",
"type" : "STRING"
}, {
- "id" : 37,
+ "id" : 34,
"name" : "nested_decimal",
"type" : "DECIMAL(19, 4)"
- }, {
- "id" : 38,
- "name" : "nested_timestamp",
- "type" : "TIMESTAMP(9)"
- }, {
- "id" : 39,
- "name" : "nested_timestamp_ltz",
- "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE"
} ]
}
}, {
- "id" : 40,
+ "id" : 35,
"name" : "f_array_array_int",
"type" : {
"type" : "ARRAY",
@@ -182,7 +162,7 @@
}
}
}, {
- "id" : 41,
+ "id" : 36,
"name" : "f_array_map",
"type" : {
"type" : "ARRAY",
@@ -193,7 +173,7 @@
}
}
}, {
- "id" : 42,
+ "id" : 37,
"name" : "f_map_array",
"type" : {
"type" : "MAP",
@@ -204,25 +184,25 @@
}
}
}, {
- "id" : 43,
+ "id" : 38,
"name" : "f_array_row",
"type" : {
"type" : "ARRAY",
"element" : {
"type" : "ROW",
"fields" : [ {
- "id" : 44,
+ "id" : 39,
"name" : "name",
"type" : "STRING"
}, {
- "id" : 45,
+ "id" : 40,
"name" : "score",
"type" : "DECIMAL(9, 2)"
} ]
}
}
}, {
- "id" : 46,
+ "id" : 41,
"name" : "f_map_row",
"type" : {
"type" : "MAP",
@@ -230,66 +210,18 @@
"value" : {
"type" : "ROW",
"fields" : [ {
- "id" : 47,
+ "id" : 42,
"name" : "enabled",
"type" : "BOOLEAN"
}, {
- "id" : 48,
+ "id" : 43,
"name" : "event_time",
"type" : "TIMESTAMP(6)"
} ]
}
}
- }, {
- "id" : 49,
- "name" : "f_deep_row",
- "type" : {
- "type" : "ROW",
- "fields" : [ {
- "id" : 52,
- "name" : "children",
- "type" : {
- "type" : "ARRAY",
- "element" : {
- "type" : "ROW",
- "fields" : [ {
- "id" : 50,
- "name" : "leaf_id",
- "type" : "BIGINT"
- }, {
- "id" : 51,
- "name" : "leaf_variant",
- "type" : "VARIANT"
- } ]
- }
- }
- }, {
- "id" : 53,
- "name" : "labels",
- "type" : {
- "type" : "MAP",
- "key" : "STRING NOT NULL",
- "value" : "STRING"
- }
- } ]
- }
- }, {
- "id" : 54,
- "name" : "f_array_variant",
- "type" : {
- "type" : "ARRAY",
- "element" : "VARIANT"
- }
- }, {
- "id" : 55,
- "name" : "f_map_variant",
- "type" : {
- "type" : "MAP",
- "key" : "STRING NOT NULL",
- "value" : "VARIANT"
- }
} ],
- "highestFieldId" : 55,
+ "highestFieldId" : 43,
"partitionKeys" : [ ],
"primaryKeys" : [ ],
"options" : {
@@ -297,10 +229,8 @@
"blob-field" : "f_blob,f_blob_descriptor",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
- "file.format" : "parquet",
+ "file.format" : "avro",
"blob-descriptor-field" : "f_blob_descriptor"
},
- "comment" : "All Paimon data types written by Java Paimon",
- "timeMillis" : 1788853434699
-}
\ No newline at end of file
+ "timeMillis" : 1789549987393
+}
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/EARLIEST
b/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/LATEST
b/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/snapshot-1
b/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/snapshot-1
new file mode 100644
index 00000000..6158a056
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "09d9c09d-a5da-4263-a9f6-19f5abb8dd4e",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-ecb389f6-229e-4460-ab9e-d739db56e794-1",
+ "deltaManifestListSize" : 1261,
+ "commitUser" : "660bd68f-e39e-4fec-80b9-d965a3bc3245",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789549988252,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/README.md
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/README.md
new file mode 100644
index 00000000..0420a933
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/README.md
@@ -0,0 +1,16 @@
+Schema:
+
+id INT NOT NULL
+f_vector_boolean VECTOR<BOOLEAN, 3>
+f_vector_tinyint VECTOR<TINYINT, 3>
+f_vector_smallint VECTOR<SMALLINT, 3>
+f_vector_int VECTOR<INT, 3>
+f_vector_bigint VECTOR<BIGINT, 3>
+f_vector_float VECTOR<FLOAT, 3>
+f_vector_double VECTOR<DOUBLE, 3>
+
+Options: bucket = -1, file.format = avro. The table is unpartitioned and has
no primary
+key.
+
+Add: (1, [true,false,true], [-1,0,1], [-1000,0,1000], [-100000,0,100000],
[-10000000000,0,10000000000], [1.25,-2.5,3.75], [1.125,-2.25,3.5])
+Add: (2, [false,true,false], [2,3,4], [2000,3000,4000],
[200000,300000,400000], [20000000000,30000000000,40000000000], [4.25,5.5,6.75],
[4.125,5.25,6.5])
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/bucket-0/data-3e8ef942-de78-4a10-acb8-911560a123bd-0.avro
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/bucket-0/data-3e8ef942-de78-4a10-acb8-911560a123bd-0.avro
new file mode 100644
index 00000000..a5fc44e8
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/bucket-0/data-3e8ef942-de78-4a10-acb8-911560a123bd-0.avro
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-ef4060ce-0bd8-4eab-ba5f-4f4a90603b8f-0
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-ef4060ce-0bd8-4eab-ba5f-4f4a90603b8f-0
new file mode 100644
index 00000000..d51087ba
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-ef4060ce-0bd8-4eab-ba5f-4f4a90603b8f-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-0
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-0
new file mode 100644
index 00000000..b46217d5
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-0
differ
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-1
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-1
new file mode 100644
index 00000000..00ff734f
Binary files /dev/null and
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/manifest/manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/schema/schema-0
similarity index 93%
copy from
test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
copy to
test/test_data/avro/append_types_compatibility.db/java_vector_types/schema/schema-0
index 6e93d2b4..7254ef19 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
+++
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/schema/schema-0
@@ -67,8 +67,7 @@
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "avro"
},
- "timeMillis" : 1788853436194
-}
\ No newline at end of file
+ "timeMillis" : 1789549988314
+}
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/LATEST
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1
new file mode 100644
index 00000000..f11bed9c
--- /dev/null
+++
b/test/test_data/avro/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "681acc56-27b0-4a55-bbc6-476aa5111a20",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-4d5f3302-46cf-4658-b90a-e05d3cd39b76-1",
+ "deltaManifestListSize" : 1258,
+ "commitUser" : "4705de16-97c8-4367-9713-2322f84c24c2",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789549988346,
+ "totalRecordCount" : 2,
+ "deltaRecordCount" : 2,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/README.md
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/README.md
new file mode 100644
index 00000000..039de56b
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/README.md
@@ -0,0 +1,18 @@
+Schema: id INT NOT NULL, f_array_blob ARRAY<BLOB>.
+
+Options:
+
+bucket = -1
+file.format = orc
+orc.timestamp-ltz.legacy.type = false
+blob-field = f_array_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, ["blob-array-left", null, "blob-array-right"])
+Add: (2, null)
+Add: (3, [])
+
+Paimon C++ is expected to reject this table while ARRAY<BLOB> is unsupported.
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/bucket-0/data-51e35438-5081-45b2-a42b-01a7146ef8dd-0.orc
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/bucket-0/data-51e35438-5081-45b2-a42b-01a7146ef8dd-0.orc
new file mode 100644
index 00000000..63a72400
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/bucket-0/data-51e35438-5081-45b2-a42b-01a7146ef8dd-0.orc
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/bucket-0/data-51e35438-5081-45b2-a42b-01a7146ef8dd-1.blob
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/bucket-0/data-51e35438-5081-45b2-a42b-01a7146ef8dd-1.blob
new file mode 100644
index 00000000..91fc2fc4
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/bucket-0/data-51e35438-5081-45b2-a42b-01a7146ef8dd-1.blob
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-fc560a6b-5521-465f-9523-72234acb5ee8-0
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-fc560a6b-5521-465f-9523-72234acb5ee8-0
new file mode 100644
index 00000000..d65b66c0
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-fc560a6b-5521-465f-9523-72234acb5ee8-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-0
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-0
new file mode 100644
index 00000000..2a64dcdc
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-1
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-1
new file mode 100644
index 00000000..1e2c2e53
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/schema/schema-0
similarity index 81%
copy from
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
copy to
test/test_data/orc/append_types_compatibility.db/java_array_blob_types/schema/schema-0
index 0bce94c9..3e5015e6 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
+++
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/schema/schema-0
@@ -21,8 +21,8 @@
"blob-field" : "f_array_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "orc",
+ "orc.timestamp-ltz.legacy.type" : "false"
},
- "timeMillis" : 1788859707631
-}
\ No newline at end of file
+ "timeMillis" : 1789541075356
+}
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..a2baf51f
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "8b1c9fcd-1d43-4800-86ad-d9a3f43e1a9d",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-8ebc8260-da53-49fd-a42b-0ddb8a3e0269-1",
+ "deltaManifestListSize" : 1258,
+ "commitUser" : "3da29e72-d559-458a-95f5-2b1354bb0693",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789541075423,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/README.md
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/README.md
new file mode 100644
index 00000000..5cb98751
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/README.md
@@ -0,0 +1,18 @@
+Schema: id INT NOT NULL, f_map_blob MAP<STRING NOT NULL, BLOB>.
+
+Options:
+
+bucket = -1
+file.format = orc
+orc.timestamp-ltz.legacy.type = false
+blob-field = f_map_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, {"left":"blob-map-left","right":null})
+Add: (2, null)
+Add: (3, {})
+
+This writer stores map BLOB values in Paimon's external BLOB representation.
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/bucket-0/data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-0.orc
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/bucket-0/data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-0.orc
new file mode 100644
index 00000000..63a72400
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/bucket-0/data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-0.orc
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/bucket-0/data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-1.blob
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/bucket-0/data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-1.blob
new file mode 100644
index 00000000..bf82127d
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/bucket-0/data-56bef3f1-cd23-4fe9-9411-d52f5bf1db9d-1.blob
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-be1d7959-ae2c-4e46-84d0-3f42732e6560-0
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-be1d7959-ae2c-4e46-84d0-3f42732e6560-0
new file mode 100644
index 00000000..b460734c
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-be1d7959-ae2c-4e46-84d0-3f42732e6560-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-0
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-0
new file mode 100644
index 00000000..9c0e72fb
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-1
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-1
new file mode 100644
index 00000000..011f1232
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/schema/schema-0
similarity index 82%
copy from
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
copy to
test/test_data/orc/append_types_compatibility.db/java_map_blob_types/schema/schema-0
index 340c0ade..c1211fb7 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
+++
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/schema/schema-0
@@ -22,8 +22,8 @@
"blob-field" : "f_map_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "orc",
+ "orc.timestamp-ltz.legacy.type" : "false"
},
- "timeMillis" : 1788859708882
-}
\ No newline at end of file
+ "timeMillis" : 1789541075427
+}
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..81b4dfb3
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "86711cf1-74e2-478b-b5e0-bc9cc076cd7c",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-9fe2cdf2-7bce-48cc-8111-decf7e1e0232-1",
+ "deltaManifestListSize" : 1261,
+ "commitUser" : "07518da5-5e79-4d4c-872a-ec3c542a56ae",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789541075479,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/README.md
b/test/test_data/orc/append_types_compatibility.db/java_time_types/README.md
new file mode 100644
index 00000000..c3a0992c
--- /dev/null
+++ b/test/test_data/orc/append_types_compatibility.db/java_time_types/README.md
@@ -0,0 +1,14 @@
+Schema:
+
+id INT NOT NULL
+f_time_0 TIME(0)
+f_time_3 TIME(3)
+f_time_6 TIME(6)
+f_time_9 TIME(9)
+
+Options: bucket = -1, file.format = orc, and orc.timestamp-ltz.legacy.type =
false. The table
+is unpartitioned and has no primary key.
+
+Add: (1, 12:34:56, 12:34:56.123, 12:34:56.123000, 12:34:56.123000000)
+Add: (2, null, null, null, null)
+Add: (3, 00:00:00, 00:00:00.000, 00:00:00.000000, 00:00:00.000000000)
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/bucket-0/data-c3c43658-047f-42b1-bfae-26f67cd77c5f-0.orc
b/test/test_data/orc/append_types_compatibility.db/java_time_types/bucket-0/data-c3c43658-047f-42b1-bfae-26f67cd77c5f-0.orc
new file mode 100644
index 00000000..e1c840ea
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_time_types/bucket-0/data-c3c43658-047f-42b1-bfae-26f67cd77c5f-0.orc
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-306250ad-71a2-42a3-9d47-a7146425bc2b-0
b/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-306250ad-71a2-42a3-9d47-a7146425bc2b-0
new file mode 100644
index 00000000..61efacdc
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-306250ad-71a2-42a3-9d47-a7146425bc2b-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-0
b/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-0
new file mode 100644
index 00000000..ae076f33
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-1
b/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-1
new file mode 100644
index 00000000..23fa8207
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_time_types/manifest/manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
b/test/test_data/orc/append_types_compatibility.db/java_time_types/schema/schema-0
similarity index 82%
copy from
test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
copy to
test/test_data/orc/append_types_compatibility.db/java_time_types/schema/schema-0
index 02f0fbff..62822ad6 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
+++
b/test/test_data/orc/append_types_compatibility.db/java_time_types/schema/schema-0
@@ -27,8 +27,8 @@
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "manifest.format" : "avro",
- "file.format" : "parquet"
+ "file.format" : "orc",
+ "orc.timestamp-ltz.legacy.type" : "false"
},
- "timeMillis" : 1788853436258
-}
\ No newline at end of file
+ "timeMillis" : 1789541075484
+}
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/EARLIEST
b/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/LATEST
b/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/snapshot-1
b/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/snapshot-1
new file mode 100644
index 00000000..178610c8
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_time_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "f3e8a686-c553-447c-a644-a4bc4462a1e6",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-e0a1ab29-c5e3-438c-b82f-753a4006e310-1",
+ "deltaManifestListSize" : 1264,
+ "commitUser" : "93535f72-946f-4916-b851-45b816d009cf",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789541075550,
+ "totalRecordCount" : 3,
+ "deltaRecordCount" : 3,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/README.md
b/test/test_data/orc/append_types_compatibility.db/java_types/README.md
new file mode 100644
index 00000000..1d04825b
--- /dev/null
+++ b/test/test_data/orc/append_types_compatibility.db/java_types/README.md
@@ -0,0 +1,100 @@
+Schema:
+
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_int INT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_string STRING
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_bytes BYTES
+f_blob BLOB
+f_blob_descriptor BLOB
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_9_2 DECIMAL(9, 2)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_decimal_38_38 DECIMAL(38, 38)
+f_date DATE
+f_timestamp_0 TIMESTAMP(0)
+f_timestamp_3 TIMESTAMP(3)
+f_timestamp_6 TIMESTAMP(6)
+f_timestamp_9 TIMESTAMP(9)
+f_timestamp_ltz_0 TIMESTAMP_LTZ(0)
+f_timestamp_ltz_3 TIMESTAMP_LTZ(3)
+f_timestamp_ltz_6 TIMESTAMP_LTZ(6)
+f_timestamp_ltz_9 TIMESTAMP_LTZ(9)
+f_array_int ARRAY<INT>
+f_map_string_bigint MAP<STRING NOT NULL, BIGINT>
+f_row ROW<nested_int INT, nested_string STRING, nested_decimal DECIMAL(19, 4),
+ nested_timestamp TIMESTAMP(9), nested_timestamp_ltz TIMESTAMP_LTZ(9)>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING NOT NULL, INT>>
+f_map_array MAP<STRING NOT NULL, ARRAY<INT>>
+f_array_row ARRAY<ROW<name STRING, score DECIMAL(9, 2)>>
+f_map_row MAP<STRING NOT NULL, ROW<enabled BOOLEAN, event_time TIMESTAMP(6)>>
+
+Options:
+
+bucket = -1
+file.format = orc
+orc.timestamp-ltz.legacy.type = false
+blob-field = f_blob,f_blob_descriptor
+blob-descriptor-field = f_blob_descriptor
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table has no partition key or primary key. Values below are listed as
[id=1, id=2, id=3].
+Timestamps and local-zoned timestamps are shown in Asia/Shanghai; binary
values are hexadecimal.
+
+id: [1, 2, 3]
+f_boolean: [true, null, null]
+f_tinyint: [-8, null, null]
+f_smallint: [1234, null, null]
+f_int: [-123456, null, null]
+f_bigint: [9223372036854770000, null, null]
+f_float: [1.25, null, null]
+f_double: [-12345.6789, null, null]
+f_char: ["char ", null, " "]
+f_varchar: ["varchar-中文", null, ""]
+f_string: ["pypaimon 2.0.0", null, ""]
+f_binary: [0x3132333435363738, null, 0x4142434445464748]
+f_varbinary: [0x76617262696e6172790076616c7565, null, 0x]
+f_bytes: [0x62797465730076616c7565, null, 0x]
+f_blob: ["ordinary blob payload from pypaimon 2.0.0", null, ""]
+f_blob_descriptor:
[Blob("file:///nonexistent/pypaimon-all-types-external-blob.bin", 7, 11), null,
null]
+f_decimal_1_0: [9, null, null]
+f_decimal_9_2: [1234567.89, null, null]
+f_decimal_18_2: [1234567890123456.78, null, null]
+f_decimal_19_2: [12345678901234567.89, null, null]
+f_decimal_38_18: [12345678901234567890.123456789012345678, null, null]
+f_decimal_38_38: [0.12345678901234567890123456789012345678, null, null]
+f_date: [2024-02-29, null, null]
+f_timestamp_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_9: [2024-02-29 12:34:56.123456000, null, null]
+f_timestamp_ltz_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_ltz_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_ltz_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_ltz_9: [2024-02-29 12:34:56.123456000, null, null]
+f_array_int: [[1, null, 3], null, []]
+f_map_string_bigint: [{"one":1,"null":null}, null, {}]
+f_row: [(7,"nested",123456789012345.6789,2024-02-29
12:34:56.123456000,2024-02-29 12:34:56.123456000), null, null]
+f_array_array_int: [[[1,2],null,[]], null, []]
+f_array_map: [[{"a":1,"b":null},null,{}], null, []]
+f_map_array: [{"numbers":[1,null,3],"empty":[]}, null, {}]
+f_array_row: [[("alice",99.50),null,("bob",null)], null, []]
+f_map_row: [{"first":(true,2024-02-29 12:34:56.123456),"second":null}, null,
{}]
+
+The two top-level precision-9 timestamp fields and the two precision-9
timestamp children of
+f_row are written with microsecond data. Their complete nanosecond
representation is shown with
+the trailing three zeros (.123456000) instead of abbreviating it to .123456.
VARIANT and VECTOR
+are omitted because Java's ORC path does not support them.
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/bucket-0/data-4135518c-e709-4232-ad15-508e494bdcf8-0.orc
b/test/test_data/orc/append_types_compatibility.db/java_types/bucket-0/data-4135518c-e709-4232-ad15-508e494bdcf8-0.orc
new file mode 100644
index 00000000..64e7be78
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_types/bucket-0/data-4135518c-e709-4232-ad15-508e494bdcf8-0.orc
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/bucket-0/data-4135518c-e709-4232-ad15-508e494bdcf8-1.blob
b/test/test_data/orc/append_types_compatibility.db/java_types/bucket-0/data-4135518c-e709-4232-ad15-508e494bdcf8-1.blob
new file mode 100644
index 00000000..6561d11d
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_types/bucket-0/data-4135518c-e709-4232-ad15-508e494bdcf8-1.blob
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-6b6f74b3-783e-4138-84a2-343b997c2f53-0
b/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-6b6f74b3-783e-4138-84a2-343b997c2f53-0
new file mode 100644
index 00000000..d1f6a696
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-6b6f74b3-783e-4138-84a2-343b997c2f53-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-0
b/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-0
new file mode 100644
index 00000000..e8b4feaa
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-0
differ
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-1
b/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-1
new file mode 100644
index 00000000..1edb5e34
Binary files /dev/null and
b/test/test_data/orc/append_types_compatibility.db/java_types/manifest/manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
b/test/test_data/orc/append_types_compatibility.db/java_types/schema/schema-0
similarity index 79%
copy from
test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
copy to
test/test_data/orc/append_types_compatibility.db/java_types/schema/schema-0
index 554c947a..ea80ddb6 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
+++
b/test/test_data/orc/append_types_compatibility.db/java_types/schema/schema-0
@@ -125,10 +125,6 @@
"id" : 30,
"name" : "f_timestamp_ltz_9",
"type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE"
- }, {
- "id" : 31,
- "name" : "f_variant",
- "type" : "VARIANT"
}, {
"id" : 32,
"name" : "f_array_int",
@@ -240,56 +236,8 @@
} ]
}
}
- }, {
- "id" : 49,
- "name" : "f_deep_row",
- "type" : {
- "type" : "ROW",
- "fields" : [ {
- "id" : 52,
- "name" : "children",
- "type" : {
- "type" : "ARRAY",
- "element" : {
- "type" : "ROW",
- "fields" : [ {
- "id" : 50,
- "name" : "leaf_id",
- "type" : "BIGINT"
- }, {
- "id" : 51,
- "name" : "leaf_variant",
- "type" : "VARIANT"
- } ]
- }
- }
- }, {
- "id" : 53,
- "name" : "labels",
- "type" : {
- "type" : "MAP",
- "key" : "STRING NOT NULL",
- "value" : "STRING"
- }
- } ]
- }
- }, {
- "id" : 54,
- "name" : "f_array_variant",
- "type" : {
- "type" : "ARRAY",
- "element" : "VARIANT"
- }
- }, {
- "id" : 55,
- "name" : "f_map_variant",
- "type" : {
- "type" : "MAP",
- "key" : "STRING NOT NULL",
- "value" : "VARIANT"
- }
} ],
- "highestFieldId" : 55,
+ "highestFieldId" : 48,
"partitionKeys" : [ ],
"primaryKeys" : [ ],
"options" : {
@@ -297,10 +245,9 @@
"blob-field" : "f_blob,f_blob_descriptor",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
- "file.format" : "parquet",
+ "file.format" : "orc",
+ "orc.timestamp-ltz.legacy.type" : "false",
"blob-descriptor-field" : "f_blob_descriptor"
},
- "comment" : "All Paimon data types written by Java Paimon",
- "timeMillis" : 1788853434699
-}
\ No newline at end of file
+ "timeMillis" : 1789541073351
+}
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/EARLIEST
b/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/LATEST
b/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/snapshot-1
b/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/snapshot-1
new file mode 100644
index 00000000..3444f7fe
--- /dev/null
+++
b/test/test_data/orc/append_types_compatibility.db/java_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "e86ee647-90a5-4fc0-b88e-10c2b1641d89",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-76873624-e06e-4313-b7ae-ec0e45180cc6-1",
+ "deltaManifestListSize" : 1261,
+ "commitUser" : "eaeb0288-350f-4fab-a4ec-dabf1fb99899",
+ "writerVersion" :
"java-2.2-SNAPSHOT-bbae07d37545cf2cf65b397007fa68c387bb521f",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789541075249,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md
b/test/test_data/parquet/append_types_compatibility.db/README.md
deleted file mode 100644
index a1ac373a..00000000
--- a/test/test_data/parquet/append_types_compatibility.db/README.md
+++ /dev/null
@@ -1,55 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Paimon writer compatibility fixtures
-
-The `append_types_compatibility.db` database contains append-only
compatibility tables written by
-Python, Rust, and Java Paimon. Table names use the writer as a prefix and
describe the group of
-types in that table; no individual table is claimed to contain every Paimon
type.
-
-The writers and Parquet implementations are:
-
-- `python_*`: `pypaimon==2.0.0` with `pyarrow==19.0.1`. The Parquet files
contain `ARROW:schema` metadata.
-- `rust_*`: Paimon Rust 0.4.0 at commit
`ced0c86b4db76265b5b2aedcbc9909a208f2c130`, using `parquet-rs version 58.4.0`.
The Parquet files contain `ARROW:schema` metadata.
-- `java_*`: the Java batch-write API from `paimon-bundle-2-ali-2.6.jar`, using
`parquet-mr version 1.16.0`. The Parquet files do not contain `ARROW:schema`
metadata.
-
-## Table groups
-
-Each writer prefix has five tables with the same declared schema and intended
logical values:
-
-- `<writer>_types`: 43 columns and 3 rows covering integral and floating
types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE,
TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions,
VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in
every nullable column and row 3 exercises empty values.
-- `<writer>_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT,
INT, BIGINT, FLOAT, and DOUBLE.
-- `<writer>_array_blob_types`: Paimon C++ does not currently support
ARRAY<BLOB>.
-- `<writer>_map_blob_types`: MAP<STRING, BLOB>, including non-null,
null, and empty map
- values. Python and Java store these values in standard separate BLOB files,
which Paimon C++
- can read. Rust stores the raw values inline as Parquet `binary`; this is not
the Paimon BLOB
- descriptor representation, cannot be read as BLOB by Java, and is
intentionally rejected by
- Paimon C++.
-- `<writer>_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++
currently rejects `TIME` while parsing the table schema.
-
-The tables are separated because Paimon C++ does not allow VECTOR in a
data-evolution table, BLOB requires data evolution, and a schema-level
incompatibility must not prevent compatible columns from being tested.
-
-`MULTISET` is absent because the Python batch writer cannot produce it through
its PyArrow conversion and Paimon C++ rejects it while parsing the table
schema. Paimon Rust 0.4.0 supports `MULTISET` and represents it as an Arrow map
from each element to its `INT` count, but it cannot be included in an
equivalent three-writer fixture. Consequently, this fixture is a compatibility
matrix, not an assertion that every type supported by every Paimon
implementation can be represented by one table.
-
-The `f_blob_descriptor` value intentionally points at a nonexistent external
URI. Read it with `blob-as-descriptor=true`; it exercises the inline descriptor
and Parquet `ARROW:schema` path rather than external blob fetching.
-
-Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`,
but the stored types
-differ for `f_blob_descriptor`. The Python fixture records the physical
Parquet `binary` column as
-Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The
compatibility test
-verifies that Paimon C++ can read both representations and preserve the
descriptor value.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/README.md
new file mode 100644
index 00000000..c4f2883b
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_array_blob ARRAY<BLOB>.
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_array_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, ["blob-array-0", null, "blob-array-2"])
+Add: (2, null)
+Add: (3, [])
+
+Paimon C++ is expected to reject this table while ARRAY<BLOB> is unsupported.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
index 0bce94c9..dbaf8d3d 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
@@ -21,8 +21,7 @@
"blob-field" : "f_array_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
"file.format" : "parquet"
},
"timeMillis" : 1788859707631
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/README.md
new file mode 100644
index 00000000..92bbda8d
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_map_blob MAP<STRING NOT NULL, BLOB>.
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_map_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, {"left":"blob-map-left","right":null})
+Add: (2, null)
+Add: (3, {})
+
+This writer stores map BLOB values in Paimon's external BLOB representation.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
index 340c0ade..51695bbc 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
@@ -22,8 +22,7 @@
"blob-field" : "f_map_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
"file.format" : "parquet"
},
"timeMillis" : 1788859708882
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/java_time_types/README.md
new file mode 100644
index 00000000..500634f1
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/java_time_types/README.md
@@ -0,0 +1,13 @@
+Schema:
+
+id INT NOT NULL
+f_time_0 TIME(0)
+f_time_3 TIME(3)
+f_time_6 TIME(6)
+f_time_9 TIME(9)
+
+Options: bucket = -1, file.format = parquet. The table is unpartitioned and
has no primary
+key.
+
+Add: (1, 12:34:56, 12:34:56.123, 12:34:56.123000, 12:34:56.123000000)
+Add: (2, null, null, null, null)
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
index 02f0fbff..b9ef0aae 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0
@@ -27,8 +27,7 @@
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "manifest.format" : "avro",
"file.format" : "parquet"
},
"timeMillis" : 1788853436258
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/java_types/README.md
new file mode 100644
index 00000000..bdd1e737
--- /dev/null
+++ b/test/test_data/parquet/append_types_compatibility.db/java_types/README.md
@@ -0,0 +1,103 @@
+Schema:
+
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_int INT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_string STRING
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_bytes BYTES
+f_blob BLOB
+f_blob_descriptor BLOB
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_9_2 DECIMAL(9, 2)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_decimal_38_38 DECIMAL(38, 38)
+f_date DATE
+f_timestamp_0 TIMESTAMP(0)
+f_timestamp_3 TIMESTAMP(3)
+f_timestamp_6 TIMESTAMP(6)
+f_timestamp_9 TIMESTAMP(9)
+f_timestamp_ltz_0 TIMESTAMP_LTZ(0)
+f_timestamp_ltz_3 TIMESTAMP_LTZ(3)
+f_timestamp_ltz_6 TIMESTAMP_LTZ(6)
+f_timestamp_ltz_9 TIMESTAMP_LTZ(9)
+f_variant VARIANT
+f_array_int ARRAY<INT>
+f_map_string_bigint MAP<STRING NOT NULL, BIGINT>
+f_row ROW<nested_int INT, nested_string STRING, nested_decimal DECIMAL(19, 4),
+ nested_timestamp TIMESTAMP(9), nested_timestamp_ltz TIMESTAMP_LTZ(9)>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING NOT NULL, INT>>
+f_map_array MAP<STRING NOT NULL, ARRAY<INT>>
+f_array_row ARRAY<ROW<name STRING, score DECIMAL(9, 2)>>
+f_map_row MAP<STRING NOT NULL, ROW<enabled BOOLEAN, event_time TIMESTAMP(6)>>
+f_deep_row ROW<children ARRAY<ROW<leaf_id BIGINT, leaf_variant VARIANT>>,
+ labels MAP<STRING NOT NULL, STRING>>
+f_array_variant ARRAY<VARIANT>
+f_map_variant MAP<STRING NOT NULL, VARIANT>
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_blob,f_blob_descriptor
+blob-descriptor-field = f_blob_descriptor
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table has no partition key or primary key. Values below are listed as
[id=1, id=2, id=3].
+Timestamps and local-zoned timestamps are shown in Asia/Shanghai; binary
values are hexadecimal.
+
+id: [1, 2, 3]
+f_boolean: [true, null, null]
+f_tinyint: [-8, null, null]
+f_smallint: [1234, null, null]
+f_int: [-123456, null, null]
+f_bigint: [9223372036854770000, null, null]
+f_float: [1.25, null, null]
+f_double: [-12345.6789, null, null]
+f_char: ["char", null, ""]
+f_varchar: ["varchar-中文", null, ""]
+f_string: ["pypaimon 2.0.0", null, ""]
+f_binary: [0x3132333435363738, null, 0x4142434445464748]
+f_varbinary: [0x76617262696e6172790076616c7565, null, 0x]
+f_bytes: [0x62797465730076616c7565, null, 0x]
+f_blob: ["ordinary blob payload from pypaimon 2.0.0", null, ""]
+f_blob_descriptor:
[Blob("file:///nonexistent/pypaimon-all-types-external-blob.bin", 7, 11), null,
null]
+f_decimal_1_0: [9, null, null]
+f_decimal_9_2: [1234567.89, null, null]
+f_decimal_18_2: [1234567890123456.78, null, null]
+f_decimal_19_2: [12345678901234567.89, null, null]
+f_decimal_38_18: [12345678901234567890.123456789012345678, null, null]
+f_decimal_38_38: [0.12345678901234567890123456789012345678, null, null]
+f_date: [2024-02-29, null, null]
+f_timestamp_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_9: [2024-02-29 12:34:56.123456000, null, null]
+f_timestamp_ltz_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_ltz_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_ltz_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_ltz_9: [2024-02-29 12:34:56.123456000, null, null]
+f_variant:
[{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},
null, null]
+f_array_int: [[1, null, 3], null, []]
+f_map_string_bigint: [{"one":1,"null":null}, null, {}]
+f_row: [(7,"nested",123456789012345.6789,2024-02-29
12:34:56.123456000,2024-02-29 12:34:56.123456000), null, null]
+f_array_array_int: [[[1,2],null,[]], null, []]
+f_array_map: [[{"a":1,"b":null},null,{}], null, []]
+f_map_array: [{"numbers":[1,null,3],"empty":[]}, null, {}]
+f_array_row: [[("alice",99.50),null,("bob",null)], null, []]
+f_map_row: [{"first":(true,2024-02-29 12:34:56.123456),"second":null}, null,
{}]
+f_deep_row:
[([{"leaf_id":10,"leaf_variant":[1,"two",false,{"k":"v"}]},{"leaf_id":11,"leaf_variant":null}],{"language":"python","format":"parquet"}),
null, null]
+f_array_variant:
[[{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},null,[1,"two",false,{"k":"v"}]],
null, []]
+f_map_variant:
[{"object":{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},"array":[1,"two",false,{"k":"v"}]},
null, {}]
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
index 554c947a..5f07dc57 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0
@@ -297,10 +297,9 @@
"blob-field" : "f_blob,f_blob_descriptor",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
- "manifest.format" : "avro",
"file.format" : "parquet",
"blob-descriptor-field" : "f_blob_descriptor"
},
"comment" : "All Paimon data types written by Java Paimon",
"timeMillis" : 1788853434699
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/README.md
new file mode 100644
index 00000000..e9c06524
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/README.md
@@ -0,0 +1,16 @@
+Schema:
+
+id INT NOT NULL
+f_vector_boolean VECTOR<BOOLEAN, 3>
+f_vector_tinyint VECTOR<TINYINT, 3>
+f_vector_smallint VECTOR<SMALLINT, 3>
+f_vector_int VECTOR<INT, 3>
+f_vector_bigint VECTOR<BIGINT, 3>
+f_vector_float VECTOR<FLOAT, 3>
+f_vector_double VECTOR<DOUBLE, 3>
+
+Options: bucket = -1, file.format = parquet. The table is unpartitioned and
has no primary
+key.
+
+Add: (1, [true,false,true], [-1,0,1], [-1000,0,1000], [-100000,0,100000],
[-10000000000,0,10000000000], [1.25,-2.5,3.75], [1.125,-2.25,3.5])
+Add: (2, [false,true,false], [2,3,4], [2000,3000,4000],
[200000,300000,400000], [20000000000,30000000000,40000000000], [4.25,5.5,6.75],
[4.125,5.25,6.5])
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
index 6e93d2b4..6d841b85 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0
@@ -67,8 +67,7 @@
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "manifest.format" : "avro",
"file.format" : "parquet"
},
"timeMillis" : 1788853436194
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/README.md
new file mode 100644
index 00000000..c4f2883b
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_array_blob ARRAY<BLOB>.
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_array_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, ["blob-array-0", null, "blob-array-2"])
+Add: (2, null)
+Add: (3, [])
+
+Paimon C++ is expected to reject this table while ARRAY<BLOB> is unsupported.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
index 9eff531e..01009c4e 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
@@ -23,11 +23,10 @@
"options": {
"bucket": "-1",
"file.format": "parquet",
- "manifest.format": "avro",
"row-tracking.enabled": "true",
"data-evolution.enabled": "true",
"blob-field": "f_array_blob"
},
"comment": "Generated by pypaimon 2.0.0 with ARRAY<BLOB>.",
"timeMillis": 1788859561835
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/README.md
new file mode 100644
index 00000000..92bbda8d
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_map_blob MAP<STRING NOT NULL, BLOB>.
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_map_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, {"left":"blob-map-left","right":null})
+Add: (2, null)
+Add: (3, {})
+
+This writer stores map BLOB values in Paimon's external BLOB representation.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
index 08ed496f..f4571d59 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
@@ -24,11 +24,10 @@
"options": {
"bucket": "-1",
"file.format": "parquet",
- "manifest.format": "avro",
"row-tracking.enabled": "true",
"data-evolution.enabled": "true",
"blob-field": "f_map_blob"
},
"comment": "Generated by pypaimon 2.0.0 with MAP<STRING, BLOB>.",
"timeMillis": 1788859561850
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_time_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/python_time_types/README.md
new file mode 100644
index 00000000..500634f1
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_time_types/README.md
@@ -0,0 +1,13 @@
+Schema:
+
+id INT NOT NULL
+f_time_0 TIME(0)
+f_time_3 TIME(3)
+f_time_6 TIME(6)
+f_time_9 TIME(9)
+
+Options: bucket = -1, file.format = parquet. The table is unpartitioned and
has no primary
+key.
+
+Add: (1, 12:34:56, 12:34:56.123, 12:34:56.123000, 12:34:56.123000000)
+Add: (2, null, null, null, null)
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0
index 99509c46..3f8ae8bb 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0
@@ -33,9 +33,8 @@
"primaryKeys": [],
"options": {
"bucket": "-1",
- "file.format": "parquet",
- "manifest.format": "avro"
+ "file.format": "parquet"
},
"comment": "Generated by pypaimon 2.0.0 with Python-supported TIME
declarations.",
"timeMillis": 1788513128460
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/python_types/README.md
new file mode 100644
index 00000000..bdd1e737
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_types/README.md
@@ -0,0 +1,103 @@
+Schema:
+
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_int INT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_string STRING
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_bytes BYTES
+f_blob BLOB
+f_blob_descriptor BLOB
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_9_2 DECIMAL(9, 2)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_decimal_38_38 DECIMAL(38, 38)
+f_date DATE
+f_timestamp_0 TIMESTAMP(0)
+f_timestamp_3 TIMESTAMP(3)
+f_timestamp_6 TIMESTAMP(6)
+f_timestamp_9 TIMESTAMP(9)
+f_timestamp_ltz_0 TIMESTAMP_LTZ(0)
+f_timestamp_ltz_3 TIMESTAMP_LTZ(3)
+f_timestamp_ltz_6 TIMESTAMP_LTZ(6)
+f_timestamp_ltz_9 TIMESTAMP_LTZ(9)
+f_variant VARIANT
+f_array_int ARRAY<INT>
+f_map_string_bigint MAP<STRING NOT NULL, BIGINT>
+f_row ROW<nested_int INT, nested_string STRING, nested_decimal DECIMAL(19, 4),
+ nested_timestamp TIMESTAMP(9), nested_timestamp_ltz TIMESTAMP_LTZ(9)>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING NOT NULL, INT>>
+f_map_array MAP<STRING NOT NULL, ARRAY<INT>>
+f_array_row ARRAY<ROW<name STRING, score DECIMAL(9, 2)>>
+f_map_row MAP<STRING NOT NULL, ROW<enabled BOOLEAN, event_time TIMESTAMP(6)>>
+f_deep_row ROW<children ARRAY<ROW<leaf_id BIGINT, leaf_variant VARIANT>>,
+ labels MAP<STRING NOT NULL, STRING>>
+f_array_variant ARRAY<VARIANT>
+f_map_variant MAP<STRING NOT NULL, VARIANT>
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_blob,f_blob_descriptor
+blob-descriptor-field = f_blob_descriptor
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table has no partition key or primary key. Values below are listed as
[id=1, id=2, id=3].
+Timestamps and local-zoned timestamps are shown in Asia/Shanghai; binary
values are hexadecimal.
+
+id: [1, 2, 3]
+f_boolean: [true, null, null]
+f_tinyint: [-8, null, null]
+f_smallint: [1234, null, null]
+f_int: [-123456, null, null]
+f_bigint: [9223372036854770000, null, null]
+f_float: [1.25, null, null]
+f_double: [-12345.6789, null, null]
+f_char: ["char", null, ""]
+f_varchar: ["varchar-中文", null, ""]
+f_string: ["pypaimon 2.0.0", null, ""]
+f_binary: [0x3132333435363738, null, 0x4142434445464748]
+f_varbinary: [0x76617262696e6172790076616c7565, null, 0x]
+f_bytes: [0x62797465730076616c7565, null, 0x]
+f_blob: ["ordinary blob payload from pypaimon 2.0.0", null, ""]
+f_blob_descriptor:
[Blob("file:///nonexistent/pypaimon-all-types-external-blob.bin", 7, 11), null,
null]
+f_decimal_1_0: [9, null, null]
+f_decimal_9_2: [1234567.89, null, null]
+f_decimal_18_2: [1234567890123456.78, null, null]
+f_decimal_19_2: [12345678901234567.89, null, null]
+f_decimal_38_18: [12345678901234567890.123456789012345678, null, null]
+f_decimal_38_38: [0.12345678901234567890123456789012345678, null, null]
+f_date: [2024-02-29, null, null]
+f_timestamp_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_9: [2024-02-29 12:34:56.123456000, null, null]
+f_timestamp_ltz_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_ltz_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_ltz_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_ltz_9: [2024-02-29 12:34:56.123456000, null, null]
+f_variant:
[{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},
null, null]
+f_array_int: [[1, null, 3], null, []]
+f_map_string_bigint: [{"one":1,"null":null}, null, {}]
+f_row: [(7,"nested",123456789012345.6789,2024-02-29
12:34:56.123456000,2024-02-29 12:34:56.123456000), null, null]
+f_array_array_int: [[[1,2],null,[]], null, []]
+f_array_map: [[{"a":1,"b":null},null,{}], null, []]
+f_map_array: [{"numbers":[1,null,3],"empty":[]}, null, {}]
+f_array_row: [[("alice",99.50),null,("bob",null)], null, []]
+f_map_row: [{"first":(true,2024-02-29 12:34:56.123456),"second":null}, null,
{}]
+f_deep_row:
[([{"leaf_id":10,"leaf_variant":[1,"two",false,{"k":"v"}]},{"leaf_id":11,"leaf_variant":null}],{"language":"python","format":"parquet"}),
null, null]
+f_array_variant:
[[{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},null,[1,"two",false,{"k":"v"}]],
null, []]
+f_map_variant:
[{"object":{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},"array":[1,"two",false,{"k":"v"}]},
null, {}]
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0
index ccffbb67..bf4633ad 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0
@@ -376,7 +376,6 @@
"options": {
"bucket": "-1",
"file.format": "parquet",
- "manifest.format": "avro",
"row-tracking.enabled": "true",
"data-evolution.enabled": "true",
"blob-field": "f_blob,f_blob_descriptor",
@@ -384,4 +383,4 @@
},
"comment": "Generated by pypaimon 2.0.0 with scalar, blob, variant, and
nested types.",
"timeMillis": 1788513128436
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/README.md
new file mode 100644
index 00000000..e9c06524
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/README.md
@@ -0,0 +1,16 @@
+Schema:
+
+id INT NOT NULL
+f_vector_boolean VECTOR<BOOLEAN, 3>
+f_vector_tinyint VECTOR<TINYINT, 3>
+f_vector_smallint VECTOR<SMALLINT, 3>
+f_vector_int VECTOR<INT, 3>
+f_vector_bigint VECTOR<BIGINT, 3>
+f_vector_float VECTOR<FLOAT, 3>
+f_vector_double VECTOR<DOUBLE, 3>
+
+Options: bucket = -1, file.format = parquet. The table is unpartitioned and
has no primary
+key.
+
+Add: (1, [true,false,true], [-1,0,1], [-1000,0,1000], [-100000,0,100000],
[-10000000000,0,10000000000], [1.25,-2.5,3.75], [1.125,-2.25,3.5])
+Add: (2, [false,true,false], [2,3,4], [2000,3000,4000],
[200000,300000,400000], [20000000000,30000000000,40000000000], [4.25,5.5,6.75],
[4.125,5.25,6.5])
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0
index f7cf4b49..3fd92c86 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0
@@ -83,9 +83,8 @@
"primaryKeys": [],
"options": {
"bucket": "-1",
- "file.format": "parquet",
- "manifest.format": "avro"
+ "file.format": "parquet"
},
"comment": "Generated by pypaimon 2.0.0 with every supported VECTOR element
type.",
"timeMillis": 1788513128451
-}
\ No newline at end of file
+}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/README.md
new file mode 100644
index 00000000..c4f2883b
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_array_blob ARRAY<BLOB>.
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_array_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, ["blob-array-0", null, "blob-array-2"])
+Add: (2, null)
+Add: (3, [])
+
+Paimon C++ is expected to reject this table while ARRAY<BLOB> is unsupported.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
index c090dea0..343c501c 100644
---
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
@@ -1 +1 @@
-{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","row-tracking.enabled":"true","bucket":"-1","blob-field":"f_array_blob"},"comment":null,"timeMillis":1788860192967}
\ No newline at end of file
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","data-evolution.enabled":"true","row-tracking.enabled":"true","bucket":"-1","blob-field":"f_array_blob"},"comment":null,"timeMillis":1788860192967}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/README.md
new file mode 100644
index 00000000..7c15b6ae
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/README.md
@@ -0,0 +1,17 @@
+Schema: id INT NOT NULL, f_map_blob MAP<STRING NOT NULL, BLOB>.
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_map_blob
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table is unpartitioned and has no primary key.
+
+Add: (1, {"left":"blob-map-left","right":null})
+Add: (2, null)
+Add: (3, {})
+
+Paimon Rust stores these values inline as Parquet binary; Java and C++
intentionally reject that representation as a Paimon BLOB map.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
index b6cd8894..484b8f32 100644
---
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
@@ -1 +1 @@
-{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_map_blob","type":{"type":"MAP","key":"STRING NOT
NULL","value":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"blob-field":"f_map_blob","row-tracking.enabled":"true","file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","bucket":"-1"},"comment":null,"timeMillis":1788860192979}
\ No newline at end of file
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_map_blob","type":{"type":"MAP","key":"STRING NOT
NULL","value":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"blob-field":"f_map_blob","row-tracking.enabled":"true","file.format":"parquet","data-evolution.enabled":"true","bucket":"-1"},"comment":null,"timeMillis":1788860192979}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/README.md
new file mode 100644
index 00000000..500634f1
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/README.md
@@ -0,0 +1,13 @@
+Schema:
+
+id INT NOT NULL
+f_time_0 TIME(0)
+f_time_3 TIME(3)
+f_time_6 TIME(6)
+f_time_9 TIME(9)
+
+Options: bucket = -1, file.format = parquet. The table is unpartitioned and
has no primary
+key.
+
+Add: (1, 12:34:56, 12:34:56.123, 12:34:56.123000, 12:34:56.123000000)
+Add: (2, null, null, null, null)
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0
index dccabf94..2a343a75 100644
---
a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0
@@ -1 +1 @@
-{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_time_0","type":"TIME(0)"},{"id":2,"name":"f_time_3","type":"TIME(3)"},{"id":3,"name":"f_time_6","type":"TIME(6)"},{"id":4,"name":"f_time_9","type":"TIME(9)"}],"highestFieldId":4,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","bucket":"-1","manifest.format":"avro"},"comment":null,"timeMillis":1788858155034}
\ No newline at end of file
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_time_0","type":"TIME(0)"},{"id":2,"name":"f_time_3","type":"TIME(3)"},{"id":3,"name":"f_time_6","type":"TIME(6)"},{"id":4,"name":"f_time_9","type":"TIME(9)"}],"highestFieldId":4,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","bucket":"-1"},"comment":null,"timeMillis":1788858155034}
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/rust_types/README.md
new file mode 100644
index 00000000..bdd1e737
--- /dev/null
+++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/README.md
@@ -0,0 +1,103 @@
+Schema:
+
+id INT NOT NULL
+f_boolean BOOLEAN
+f_tinyint TINYINT
+f_smallint SMALLINT
+f_int INT
+f_bigint BIGINT
+f_float FLOAT
+f_double DOUBLE
+f_char CHAR(8)
+f_varchar VARCHAR(64)
+f_string STRING
+f_binary BINARY(8)
+f_varbinary VARBINARY(64)
+f_bytes BYTES
+f_blob BLOB
+f_blob_descriptor BLOB
+f_decimal_1_0 DECIMAL(1, 0)
+f_decimal_9_2 DECIMAL(9, 2)
+f_decimal_18_2 DECIMAL(18, 2)
+f_decimal_19_2 DECIMAL(19, 2)
+f_decimal_38_18 DECIMAL(38, 18)
+f_decimal_38_38 DECIMAL(38, 38)
+f_date DATE
+f_timestamp_0 TIMESTAMP(0)
+f_timestamp_3 TIMESTAMP(3)
+f_timestamp_6 TIMESTAMP(6)
+f_timestamp_9 TIMESTAMP(9)
+f_timestamp_ltz_0 TIMESTAMP_LTZ(0)
+f_timestamp_ltz_3 TIMESTAMP_LTZ(3)
+f_timestamp_ltz_6 TIMESTAMP_LTZ(6)
+f_timestamp_ltz_9 TIMESTAMP_LTZ(9)
+f_variant VARIANT
+f_array_int ARRAY<INT>
+f_map_string_bigint MAP<STRING NOT NULL, BIGINT>
+f_row ROW<nested_int INT, nested_string STRING, nested_decimal DECIMAL(19, 4),
+ nested_timestamp TIMESTAMP(9), nested_timestamp_ltz TIMESTAMP_LTZ(9)>
+f_array_array_int ARRAY<ARRAY<INT>>
+f_array_map ARRAY<MAP<STRING NOT NULL, INT>>
+f_map_array MAP<STRING NOT NULL, ARRAY<INT>>
+f_array_row ARRAY<ROW<name STRING, score DECIMAL(9, 2)>>
+f_map_row MAP<STRING NOT NULL, ROW<enabled BOOLEAN, event_time TIMESTAMP(6)>>
+f_deep_row ROW<children ARRAY<ROW<leaf_id BIGINT, leaf_variant VARIANT>>,
+ labels MAP<STRING NOT NULL, STRING>>
+f_array_variant ARRAY<VARIANT>
+f_map_variant MAP<STRING NOT NULL, VARIANT>
+
+Options:
+
+bucket = -1
+file.format = parquet
+blob-field = f_blob,f_blob_descriptor
+blob-descriptor-field = f_blob_descriptor
+data-evolution.enabled = true
+row-tracking.enabled = true
+
+The table has no partition key or primary key. Values below are listed as
[id=1, id=2, id=3].
+Timestamps and local-zoned timestamps are shown in Asia/Shanghai; binary
values are hexadecimal.
+
+id: [1, 2, 3]
+f_boolean: [true, null, null]
+f_tinyint: [-8, null, null]
+f_smallint: [1234, null, null]
+f_int: [-123456, null, null]
+f_bigint: [9223372036854770000, null, null]
+f_float: [1.25, null, null]
+f_double: [-12345.6789, null, null]
+f_char: ["char", null, ""]
+f_varchar: ["varchar-中文", null, ""]
+f_string: ["pypaimon 2.0.0", null, ""]
+f_binary: [0x3132333435363738, null, 0x4142434445464748]
+f_varbinary: [0x76617262696e6172790076616c7565, null, 0x]
+f_bytes: [0x62797465730076616c7565, null, 0x]
+f_blob: ["ordinary blob payload from pypaimon 2.0.0", null, ""]
+f_blob_descriptor:
[Blob("file:///nonexistent/pypaimon-all-types-external-blob.bin", 7, 11), null,
null]
+f_decimal_1_0: [9, null, null]
+f_decimal_9_2: [1234567.89, null, null]
+f_decimal_18_2: [1234567890123456.78, null, null]
+f_decimal_19_2: [12345678901234567.89, null, null]
+f_decimal_38_18: [12345678901234567890.123456789012345678, null, null]
+f_decimal_38_38: [0.12345678901234567890123456789012345678, null, null]
+f_date: [2024-02-29, null, null]
+f_timestamp_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_9: [2024-02-29 12:34:56.123456000, null, null]
+f_timestamp_ltz_0: [2024-02-29 12:34:56, null, null]
+f_timestamp_ltz_3: [2024-02-29 12:34:56.123, null, null]
+f_timestamp_ltz_6: [2024-02-29 12:34:56.123456, null, null]
+f_timestamp_ltz_9: [2024-02-29 12:34:56.123456000, null, null]
+f_variant:
[{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},
null, null]
+f_array_int: [[1, null, 3], null, []]
+f_map_string_bigint: [{"one":1,"null":null}, null, {}]
+f_row: [(7,"nested",123456789012345.6789,2024-02-29
12:34:56.123456000,2024-02-29 12:34:56.123456000), null, null]
+f_array_array_int: [[[1,2],null,[]], null, []]
+f_array_map: [[{"a":1,"b":null},null,{}], null, []]
+f_map_array: [{"numbers":[1,null,3],"empty":[]}, null, {}]
+f_array_row: [[("alice",99.50),null,("bob",null)], null, []]
+f_map_row: [{"first":(true,2024-02-29 12:34:56.123456),"second":null}, null,
{}]
+f_deep_row:
[([{"leaf_id":10,"leaf_variant":[1,"two",false,{"k":"v"}]},{"leaf_id":11,"leaf_variant":null}],{"language":"python","format":"parquet"}),
null, null]
+f_array_variant:
[[{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},null,[1,"two",false,{"k":"v"}]],
null, []]
+f_map_variant:
[{"object":{"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}},"array":[1,"two",false,{"k":"v"}]},
null, {}]
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0
index 8118fd30..e35892f7 100644
---
a/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0
@@ -1 +1 @@
-{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_boolean","type":"BOOLEAN"},{"id":2,"name":"f_tinyint","type":"TINYINT"},{"id":3,"name":"f_smallint","type":"SMALLINT"},{"id":4,"name":"f_int","type":"INT"},{"id":5,"name":"f_bigint","type":"BIGINT"},{"id":6,"name":"f_float","type":"FLOAT"},{"id":7,"name":"f_double","type":"DOUBLE"},{"id":8,"name":"f_char","type":"CHAR(8)"},{"id":9,"name":"f_varchar","type":"VARCHAR(64)"},{"id":10,"name":"f_string",
[...]
\ No newline at end of file
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_boolean","type":"BOOLEAN"},{"id":2,"name":"f_tinyint","type":"TINYINT"},{"id":3,"name":"f_smallint","type":"SMALLINT"},{"id":4,"name":"f_int","type":"INT"},{"id":5,"name":"f_bigint","type":"BIGINT"},{"id":6,"name":"f_float","type":"FLOAT"},{"id":7,"name":"f_double","type":"DOUBLE"},{"id":8,"name":"f_char","type":"CHAR(8)"},{"id":9,"name":"f_varchar","type":"VARCHAR(64)"},{"id":10,"name":"f_string",
[...]
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/README.md
b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/README.md
new file mode 100644
index 00000000..e9c06524
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/README.md
@@ -0,0 +1,16 @@
+Schema:
+
+id INT NOT NULL
+f_vector_boolean VECTOR<BOOLEAN, 3>
+f_vector_tinyint VECTOR<TINYINT, 3>
+f_vector_smallint VECTOR<SMALLINT, 3>
+f_vector_int VECTOR<INT, 3>
+f_vector_bigint VECTOR<BIGINT, 3>
+f_vector_float VECTOR<FLOAT, 3>
+f_vector_double VECTOR<DOUBLE, 3>
+
+Options: bucket = -1, file.format = parquet. The table is unpartitioned and
has no primary
+key.
+
+Add: (1, [true,false,true], [-1,0,1], [-1000,0,1000], [-100000,0,100000],
[-10000000000,0,10000000000], [1.25,-2.5,3.75], [1.125,-2.25,3.5])
+Add: (2, [false,true,false], [2,3,4], [2000,3000,4000],
[200000,300000,400000], [20000000000,30000000000,40000000000], [4.25,5.5,6.75],
[4.125,5.25,6.5])
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0
index 3ee403b9..afc1eb70 100644
---
a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0
@@ -1 +1 @@
-{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_vector_boolean","type":{"type":"VECTOR","element":"BOOLEAN","length":3}},{"id":2,"name":"f_vector_tinyint","type":{"type":"VECTOR","element":"TINYINT","length":3}},{"id":3,"name":"f_vector_smallint","type":{"type":"VECTOR","element":"SMALLINT","length":3}},{"id":4,"name":"f_vector_int","type":{"type":"VECTOR","element":"INT","length":3}},{"id":5,"name":"f_vector_bigint","type":{"type":"VECTOR","ele
[...]
\ No newline at end of file
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_vector_boolean","type":{"type":"VECTOR","element":"BOOLEAN","length":3}},{"id":2,"name":"f_vector_tinyint","type":{"type":"VECTOR","element":"TINYINT","length":3}},{"id":3,"name":"f_vector_smallint","type":{"type":"VECTOR","element":"SMALLINT","length":3}},{"id":4,"name":"f_vector_int","type":{"type":"VECTOR","element":"INT","length":3}},{"id":5,"name":"f_vector_bigint","type":{"type":"VECTOR","ele
[...]
