This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 5024bb47 refactor(global-index): extract common sorted index
components (#343)
5024bb47 is described below
commit 5024bb47db7ca8ccf8583f5f4f510522bd6e7273
Author: lxy <[email protected]>
AuthorDate: Wed Sep 16 10:47:15 2026 +0800
refactor(global-index): extract common sorted index components (#343)
---
src/paimon/CMakeLists.txt | 6 +-
src/paimon/common/global_index/CMakeLists.txt | 6 +-
.../btree/btree_compatibility_test.cpp | 56 +++---
.../btree/btree_global_index_integration_test.cpp | 10 +-
.../btree/btree_global_index_reader.cpp | 33 ++--
.../global_index/btree/btree_global_index_reader.h | 10 +-
.../btree/btree_global_index_writer.cpp | 23 +--
.../global_index/btree/btree_global_index_writer.h | 6 +-
.../global_index/btree/btree_global_indexer.cpp | 2 -
.../btree/lazy_filtered_btree_reader.cpp | 30 ++--
.../btree/lazy_filtered_btree_reader.h | 10 +-
.../global_index/{btree => }/key_serializer.cpp | 77 +++++---
.../global_index/{btree => }/key_serializer.h | 36 ++--
.../{btree => }/key_serializer_test.cpp | 194 ++++++++++-----------
..._selector.cpp => sorted_file_meta_selector.cpp} | 139 ++++++++-------
...meta_selector.h => sorted_file_meta_selector.h} | 36 ++--
...test.cpp => sorted_file_meta_selector_test.cpp} | 146 ++++++++--------
...e_index_meta.cpp => sorted_index_file_meta.cpp} | 49 +++---
...btree_index_meta.h => sorted_index_file_meta.h} | 15 +-
...ta_test.cpp => sorted_index_file_meta_test.cpp} | 140 +++++++--------
.../source/primary_key_sorted_index_scan_test.cpp | 26 +--
21 files changed, 552 insertions(+), 498 deletions(-)
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index f0123228..e32b3a9e 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -610,12 +610,12 @@ if(PAIMON_BUILD_TESTS)
common/global_index/bitmap_global_index_result_test.cpp
common/global_index/bitmap_scored_global_index_result_test.cpp
common/global_index/bitmap/bitmap_global_index_test.cpp
- common/global_index/btree/btree_index_meta_test.cpp
+ common/global_index/sorted_index_file_meta_test.cpp
common/global_index/btree/btree_file_footer_test.cpp
- common/global_index/btree/key_serializer_test.cpp
+ common/global_index/key_serializer_test.cpp
common/global_index/btree/btree_global_index_integration_test.cpp
common/global_index/btree/btree_compatibility_test.cpp
- common/global_index/btree/btree_file_meta_selector_test.cpp
+ common/global_index/sorted_file_meta_selector_test.cpp
common/global_index/btree/lazy_filtered_btree_reader_test.cpp
common/global_index/rangebitmap/range_bitmap_global_index_test.cpp
common/global_index/wrap/file_index_reader_wrapper_test.cpp
diff --git a/src/paimon/common/global_index/CMakeLists.txt
b/src/paimon/common/global_index/CMakeLists.txt
index 0bfad32e..18806d58 100644
--- a/src/paimon/common/global_index/CMakeLists.txt
+++ b/src/paimon/common/global_index/CMakeLists.txt
@@ -22,10 +22,10 @@ set(PAIMON_GLOBAL_INDEX_SRC
btree/btree_global_indexer.cpp
btree/btree_global_index_reader.cpp
btree/btree_global_index_writer.cpp
- btree/btree_file_meta_selector.cpp
- btree/btree_index_meta.cpp
+ sorted_file_meta_selector.cpp
+ sorted_index_file_meta.cpp
btree/lazy_filtered_btree_reader.cpp
- btree/key_serializer.cpp
+ key_serializer.cpp
rangebitmap/range_bitmap_global_index.cpp
rangebitmap/range_bitmap_global_index_factory.cpp
offset_global_index_reader.cpp
diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp
b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp
index 84b45175..80c36205 100644
--- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp
+++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp
@@ -25,8 +25,8 @@
#include "arrow/c/bridge.h"
#include "gtest/gtest.h"
#include "paimon/common/global_index/btree/btree_global_indexer.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/string_utils.h"
#include "paimon/fs/file_system.h"
@@ -779,12 +779,14 @@ TEST_F(BTreeCompatibilityTest, DuplicateKeys) {
TEST_F(BTreeCompatibilityTest, MetaDeserialization) {
// Test int_50 meta
{
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(arrow::int32(), pool_));
std::string meta_path = data_dir_ + "/btree_test_int_50.bin.meta";
auto meta_str = ReadFileAsString(meta_path);
std::shared_ptr<Bytes> meta_bytes = Bytes::AllocateBytes(meta_str,
pool_.get());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> meta,
- BTreeIndexMeta::Deserialize(meta_bytes,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> meta,
+ SortedIndexFileMeta::Deserialize(meta_bytes,
pool_.get()));
ASSERT_TRUE(meta);
ASSERT_TRUE(meta->HasNulls());
@@ -792,25 +794,25 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) {
ASSERT_TRUE(meta->FirstKey());
ASSERT_OK_AND_ASSIGN(auto min_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()),
- arrow::int32(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey())));
ASSERT_EQ(min_key, Literal(3));
ASSERT_TRUE(meta->LastKey());
ASSERT_OK_AND_ASSIGN(auto max_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()),
- arrow::int32(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey())));
ASSERT_EQ(max_key, Literal(143));
}
// Test float_50 meta
{
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(arrow::float32(), pool_));
std::string meta_path = data_dir_ + "/btree_test_float_50.bin.meta";
auto meta_str = ReadFileAsString(meta_path);
std::shared_ptr<Bytes> meta_bytes = Bytes::AllocateBytes(meta_str,
pool_.get());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> meta,
- BTreeIndexMeta::Deserialize(meta_bytes,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> meta,
+ SortedIndexFileMeta::Deserialize(meta_bytes,
pool_.get()));
ASSERT_TRUE(meta);
ASSERT_TRUE(meta->HasNulls());
@@ -818,14 +820,12 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) {
ASSERT_TRUE(meta->FirstKey());
ASSERT_OK_AND_ASSIGN(auto min_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()),
- arrow::float32(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey())));
ASSERT_EQ(min_key, Literal(static_cast<float>(-INFINITY)));
ASSERT_TRUE(meta->LastKey());
ASSERT_OK_AND_ASSIGN(auto max_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()),
- arrow::float32(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey())));
ASSERT_EQ(max_key, Literal(static_cast<float>(std::nan(""))));
}
@@ -835,8 +835,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) {
auto meta_str = ReadFileAsString(meta_path);
std::shared_ptr<Bytes> meta_bytes = Bytes::AllocateBytes(meta_str,
pool_.get());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> meta,
- BTreeIndexMeta::Deserialize(meta_bytes,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> meta,
+ SortedIndexFileMeta::Deserialize(meta_bytes,
pool_.get()));
ASSERT_TRUE(meta);
ASSERT_TRUE(meta->HasNulls());
@@ -847,35 +847,37 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) {
// Test no_nulls meta
{
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(arrow::int32(), pool_));
std::string meta_path = data_dir_ +
"/btree_test_int_no_nulls.bin.meta";
auto meta_str = ReadFileAsString(meta_path);
std::shared_ptr<Bytes> meta_bytes = Bytes::AllocateBytes(meta_str,
pool_.get());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> meta,
- BTreeIndexMeta::Deserialize(meta_bytes,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> meta,
+ SortedIndexFileMeta::Deserialize(meta_bytes,
pool_.get()));
ASSERT_TRUE(meta);
ASSERT_TRUE(meta->FirstKey());
ASSERT_OK_AND_ASSIGN(auto min_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()),
- arrow::int32(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey())));
ASSERT_EQ(min_key, Literal(4));
ASSERT_TRUE(meta->LastKey());
ASSERT_OK_AND_ASSIGN(auto max_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()),
- arrow::int32(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey())));
ASSERT_EQ(max_key, Literal(158));
}
// Test string_50 meta
{
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(arrow::utf8(), pool_));
std::string meta_path = data_dir_ + "/btree_test_string_50.bin.meta";
auto meta_str = ReadFileAsString(meta_path);
std::shared_ptr<Bytes> meta_bytes = Bytes::AllocateBytes(meta_str,
pool_.get());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> meta,
- BTreeIndexMeta::Deserialize(meta_bytes,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> meta,
+ SortedIndexFileMeta::Deserialize(meta_bytes,
pool_.get()));
ASSERT_TRUE(meta);
ASSERT_TRUE(meta->HasNulls());
@@ -883,15 +885,13 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) {
ASSERT_TRUE(meta->FirstKey());
ASSERT_OK_AND_ASSIGN(auto min_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()),
- arrow::utf8(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey())));
std::string min_key_str = "test_00000";
ASSERT_EQ(min_key, Literal(FieldType::STRING, min_key_str.data(),
min_key_str.size()));
ASSERT_TRUE(meta->LastKey());
ASSERT_OK_AND_ASSIGN(auto max_key,
-
KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()),
- arrow::utf8(),
pool_.get()));
+
key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey())));
std::string max_key_str = "test_00049";
ASSERT_EQ(max_key, Literal(FieldType::STRING, max_key_str.data(),
max_key_str.size()));
}
diff --git
a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
index ce25b9b4..89d98d43 100644
---
a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
+++
b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
@@ -26,8 +26,8 @@
#include "paimon/common/factories/io_hook.h"
#include "paimon/common/global_index/btree/btree_global_index_writer.h"
#include "paimon/common/global_index/btree/btree_global_indexer.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
#include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/common/options/memory_size.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/data/decimal.h"
@@ -490,8 +490,8 @@ TEST_P(BTreeGlobalIndexIntegrationTest,
WriteEmptyStringKeyMetadata) {
ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish());
ASSERT_EQ(metas.size(), 1);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> meta,
- BTreeIndexMeta::Deserialize(metas[0].metadata,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> meta,
+ SortedIndexFileMeta::Deserialize(metas[0].metadata,
pool_.get()));
ASSERT_TRUE(meta->FirstKey());
ASSERT_EQ(meta->FirstKey()->size(), 0);
ASSERT_TRUE(meta->LastKey());
@@ -1940,7 +1940,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, TestIOException) {
ASSERT_TRUE(run_complete);
}
-// Multiple files with BTreeFileMetaSelector filtering
+// Multiple files with SortedFileMetaSelector filtering
TEST_P(BTreeGlobalIndexIntegrationTest,
WriteAndReadMultiFilesWithMetaSelector) {
auto file_writer = std::make_shared<FakeGlobalIndexFileWriter>(fs_,
base_path_);
auto field = arrow::field("int_field", arrow::int32());
@@ -1982,7 +1982,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest,
WriteAndReadMultiFilesWithMetaSelector)
ASSERT_EQ(all_metas.size(), 3);
// Create reader over all 3 files (internally uses LazyFilteredBTreeReader
+
- // BTreeFileMetaSelector)
+ // SortedFileMetaSelector)
auto file_reader = std::make_shared<FakeGlobalIndexFileReader>(fs_,
base_path_);
auto c_schema = CreateArrowSchema(field);
ASSERT_OK_AND_ASSIGN(auto reader,
diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
index f9a4868f..b9b7f4cf 100644
--- a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
+++ b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
@@ -20,7 +20,7 @@
#include "paimon/common/global_index/btree/btree_global_index_reader.h"
#include "fmt/format.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
#include "paimon/common/memory/memory_slice.h"
#include "paimon/common/memory/memory_slice_input.h"
#include "paimon/global_index/bitmap_global_index_result.h"
@@ -32,36 +32,36 @@ Result<std::shared_ptr<BTreeGlobalIndexReader>>
BTreeGlobalIndexReader::Create(
const std::shared_ptr<SstFileReader>& sst_file_reader, RoaringBitmap64&&
null_bitmap,
const std::optional<MemorySlice>& min_key_slice,
const std::optional<MemorySlice>& max_key_slice,
- const std::shared_ptr<arrow::DataType>& key_type, const
std::shared_ptr<MemoryPool>& pool) {
+ const std::shared_ptr<KeySerializer>& key_serializer) {
+ if (key_serializer == nullptr) {
+ return Status::Invalid("Cannot create BTreeGlobalIndexReader without a
key serializer.");
+ }
std::optional<Literal> min_key;
std::optional<Literal> max_key;
if (min_key_slice) {
- PAIMON_ASSIGN_OR_RAISE(
- min_key, KeySerializer::DeserializeKey(min_key_slice.value(),
key_type, pool.get()));
+ PAIMON_ASSIGN_OR_RAISE(min_key,
key_serializer->Deserialize(min_key_slice.value()));
}
if (max_key_slice) {
- PAIMON_ASSIGN_OR_RAISE(
- max_key, KeySerializer::DeserializeKey(max_key_slice.value(),
key_type, pool.get()));
+ PAIMON_ASSIGN_OR_RAISE(max_key,
key_serializer->Deserialize(max_key_slice.value()));
}
return std::shared_ptr<BTreeGlobalIndexReader>(new BTreeGlobalIndexReader(
sst_file_reader, std::move(null_bitmap), std::move(min_key),
std::move(max_key),
- min_key_slice, max_key_slice, key_type, pool));
+ min_key_slice, max_key_slice, key_serializer));
}
BTreeGlobalIndexReader::BTreeGlobalIndexReader(
const std::shared_ptr<SstFileReader>& sst_file_reader, RoaringBitmap64&&
null_bitmap,
std::optional<Literal> min_key, std::optional<Literal> max_key,
std::optional<MemorySlice> min_key_slice, std::optional<MemorySlice>
max_key_slice,
- const std::shared_ptr<arrow::DataType>& key_type, const
std::shared_ptr<MemoryPool>& pool)
- : pool_(pool),
- sst_file_reader_(sst_file_reader),
+ const std::shared_ptr<KeySerializer>& key_serializer)
+ : sst_file_reader_(sst_file_reader),
+ key_serializer_(key_serializer),
null_bitmap_(std::move(null_bitmap)),
min_key_(std::move(min_key)),
max_key_(std::move(max_key)),
min_key_slice_(std::move(min_key_slice)),
max_key_slice_(std::move(max_key_slice)),
- key_type_(key_type),
- comparator_(KeySerializer::CreateComparator(key_type, pool)) {}
+ comparator_(key_serializer_->CreateComparator()) {}
Result<std::shared_ptr<GlobalIndexResult>>
BTreeGlobalIndexReader::VisitIsNotNull() {
return std::make_shared<BitmapGlobalIndexResult>(
@@ -286,13 +286,12 @@ Result<RoaringBitmap64>
BTreeGlobalIndexReader::RangeQuery(const std::optional<L
// Create an index block iterator to iterate through data blocks
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> from_bytes,
- KeySerializer::SerializeKey(from.value(),
key_type_, pool_.get()));
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> to_bytes,
- KeySerializer::SerializeKey(to.value(), key_type_,
pool_.get()));
+ key_serializer_->Serialize(from.value()));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> to_bytes,
key_serializer_->Serialize(to.value()));
MemorySlice from_slice = MemorySlice::Wrap(from_bytes);
MemorySlice to_slice = MemorySlice::Wrap(to_bytes);
- PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(from_slice,
key_type_));
- PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(to_slice,
key_type_));
+ PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(from_slice));
+ PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(to_slice));
// Determine if we can skip lower/upper bound checks using cached
serialized min/max keys.
// When from == min_key_, all entries are >= from, so skip lower bound
comparison.
diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.h
b/src/paimon/common/global_index/btree/btree_global_index_reader.h
index 1bdbe71e..39bee677 100644
--- a/src/paimon/common/global_index/btree/btree_global_index_reader.h
+++ b/src/paimon/common/global_index/btree/btree_global_index_reader.h
@@ -27,7 +27,7 @@
#include "arrow/api.h"
#include "paimon/common/global_index/btree/btree_defs.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
#include "paimon/common/sst/sst_file_reader.h"
#include "paimon/global_index/global_index_io_meta.h"
#include "paimon/global_index/global_index_reader.h"
@@ -44,7 +44,7 @@ class BTreeGlobalIndexReader : public GlobalIndexReader,
const std::shared_ptr<SstFileReader>& sst_file_reader,
RoaringBitmap64&& null_bitmap,
const std::optional<MemorySlice>& min_key_slice,
const std::optional<MemorySlice>& max_key_slice,
- const std::shared_ptr<arrow::DataType>& key_type, const
std::shared_ptr<MemoryPool>& pool);
+ const std::shared_ptr<KeySerializer>& key_serializer);
Result<std::shared_ptr<GlobalIndexResult>> VisitIsNotNull() override;
@@ -95,8 +95,7 @@ class BTreeGlobalIndexReader : public GlobalIndexReader,
RoaringBitmap64&& null_bitmap,
std::optional<Literal> min_key,
std::optional<Literal> max_key,
std::optional<MemorySlice> min_key_slice,
std::optional<MemorySlice> max_key_slice,
- const std::shared_ptr<arrow::DataType>& key_type,
- const std::shared_ptr<MemoryPool>& pool);
+ const std::shared_ptr<KeySerializer>&
key_serializer);
Result<RoaringBitmap64> RangeQuery(const std::optional<Literal>& from,
const std::optional<Literal>& to, bool
from_inclusive,
@@ -106,8 +105,8 @@ class BTreeGlobalIndexReader : public GlobalIndexReader,
Result<RoaringBitmap64> AllNonNullRows();
- std::shared_ptr<MemoryPool> pool_;
std::shared_ptr<SstFileReader> sst_file_reader_;
+ std::shared_ptr<KeySerializer> key_serializer_;
RoaringBitmap64 null_bitmap_;
std::optional<Literal> min_key_;
std::optional<Literal> max_key_;
@@ -115,7 +114,6 @@ class BTreeGlobalIndexReader : public GlobalIndexReader,
std::optional<MemorySlice> min_key_slice_;
std::optional<MemorySlice> max_key_slice_;
- std::shared_ptr<arrow::DataType> key_type_;
MemorySlice::SliceComparator comparator_;
};
diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp
b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp
index 37b81c41..fd30728d 100644
--- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp
+++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp
@@ -26,8 +26,9 @@
#include "fmt/format.h"
#include "paimon/common/compression/block_compression_factory.h"
#include "paimon/common/global_index/btree/btree_defs.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
#include "paimon/common/global_index/global_index_utils.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/common/memory/memory_slice_output.h"
#include "paimon/common/predicate/literal_converter.h"
#include "paimon/common/utils/arrow/status_utils.h"
@@ -45,6 +46,8 @@ Result<std::shared_ptr<BTreeGlobalIndexWriter>>
BTreeGlobalIndexWriter::Create(
PAIMON_RETURN_NOT_OK(Preconditions::CheckNotNull(
key_field,
fmt::format("field {} not in arrow_array when Create
BTreeGlobalIndexWriter", field_name)));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(key_field->type(), pool));
PAIMON_ASSIGN_OR_RAISE(std::string index_file_name,
file_writer->NewFileName(BtreeDefs::kIdentifier));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output_stream,
@@ -52,19 +55,19 @@ Result<std::shared_ptr<BTreeGlobalIndexWriter>>
BTreeGlobalIndexWriter::Create(
auto sst_file_writer = std::make_unique<SstFileWriter>(output_stream,
/*bloom_filter=*/nullptr,
block_size,
compression_factory, pool);
return std::shared_ptr<BTreeGlobalIndexWriter>(new BTreeGlobalIndexWriter(
- field_name, arrow_type, key_field->type(), file_writer,
index_file_name, output_stream,
+ field_name, arrow_type, key_serializer, file_writer, index_file_name,
output_stream,
std::move(sst_file_writer), pool));
}
BTreeGlobalIndexWriter::BTreeGlobalIndexWriter(
const std::string& field_name, const std::shared_ptr<arrow::DataType>&
arrow_type,
- const std::shared_ptr<arrow::DataType>& key_type,
+ const std::shared_ptr<KeySerializer>& key_serializer,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer, const
std::string& index_file_name,
const std::shared_ptr<OutputStream>& output_stream,
std::unique_ptr<SstFileWriter>&& sst_writer,
const std::shared_ptr<MemoryPool>& pool)
: field_name_(field_name),
arrow_type_(arrow_type),
- key_type_(key_type),
+ key_serializer_(key_serializer),
pool_(pool),
file_writer_(file_writer),
index_file_name_(index_file_name),
@@ -135,7 +138,7 @@ Status BTreeGlobalIndexWriter::Flush() {
current_row_ids_.clear();
assert(last_key_);
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> key_bytes,
- KeySerializer::SerializeKey(last_key_.value(),
key_type_, pool_.get()));
+ key_serializer_->Serialize(last_key_.value()));
return sst_writer_->Write(std::move(key_bytes),
output.ToSlice().CopyBytes(pool_.get()));
}
@@ -190,16 +193,14 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeGlobalIndexWriter::Finish() {
std::shared_ptr<Bytes> first_key_bytes;
std::shared_ptr<Bytes> last_key_bytes;
if (first_key_) {
- PAIMON_ASSIGN_OR_RAISE(first_key_bytes, KeySerializer::SerializeKey(
- first_key_.value(),
key_type_, pool_.get()));
+ PAIMON_ASSIGN_OR_RAISE(first_key_bytes,
key_serializer_->Serialize(first_key_.value()));
}
if (last_key_) {
- PAIMON_ASSIGN_OR_RAISE(
- last_key_bytes, KeySerializer::SerializeKey(last_key_.value(),
key_type_, pool_.get()));
+ PAIMON_ASSIGN_OR_RAISE(last_key_bytes,
key_serializer_->Serialize(last_key_.value()));
}
// Create index meta
- auto index_meta =
- std::make_shared<BTreeIndexMeta>(first_key_bytes, last_key_bytes,
!null_bitmap_.IsEmpty());
+ auto index_meta = std::make_shared<SortedIndexFileMeta>(first_key_bytes,
last_key_bytes,
+
!null_bitmap_.IsEmpty());
auto meta_bytes = index_meta->Serialize(pool_.get());
// Create GlobalIndexIOMeta
diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.h
b/src/paimon/common/global_index/btree/btree_global_index_writer.h
index f8c9ff9c..1078c0b4 100644
--- a/src/paimon/common/global_index/btree/btree_global_index_writer.h
+++ b/src/paimon/common/global_index/btree/btree_global_index_writer.h
@@ -24,13 +24,13 @@
#include "arrow/api.h"
#include "paimon/common/global_index/btree/btree_file_footer.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
#include "paimon/common/sst/sst_file_writer.h"
#include "paimon/global_index/global_index_writer.h"
#include "paimon/global_index/io/global_index_file_writer.h"
#include "paimon/predicate/literal.h"
#include "paimon/utils/roaring_bitmap64.h"
namespace paimon {
+class KeySerializer;
/// Writer for BTree Global Index files.
/// This writer builds an SST file where each key maps to a list of row IDs.
@@ -76,7 +76,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter {
private:
BTreeGlobalIndexWriter(const std::string& field_name,
const std::shared_ptr<arrow::DataType>& arrow_type,
- const std::shared_ptr<arrow::DataType>& key_type,
+ const std::shared_ptr<KeySerializer>&
key_serializer,
const std::shared_ptr<GlobalIndexFileWriter>&
file_writer,
const std::string& index_file_name,
const std::shared_ptr<OutputStream>& output_stream,
@@ -90,7 +90,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter {
private:
std::string field_name_;
std::shared_ptr<arrow::DataType> arrow_type_;
- std::shared_ptr<arrow::DataType> key_type_;
+ std::shared_ptr<KeySerializer> key_serializer_;
std::shared_ptr<MemoryPool> pool_;
std::shared_ptr<GlobalIndexFileWriter> file_writer_;
diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp
b/src/paimon/common/global_index/btree/btree_global_indexer.cpp
index 8996fe59..09b50858 100644
--- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp
+++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp
@@ -27,8 +27,6 @@
#include "paimon/common/compression/block_compression_factory.h"
#include "paimon/common/global_index/btree/btree_file_footer.h"
#include "paimon/common/global_index/btree/btree_global_index_writer.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
#include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h"
#include "paimon/common/memory/memory_slice.h"
#include "paimon/common/memory/memory_slice_input.h"
diff --git
a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
index 4ea7576b..ca5508bf 100644
--- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
+++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
@@ -25,8 +25,8 @@
#include "paimon/common/executor/future.h"
#include "paimon/common/global_index/btree/btree_file_footer.h"
#include "paimon/common/global_index/btree/btree_global_index_reader.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/common/global_index/union_global_index_reader.h"
#include "paimon/common/memory/memory_slice.h"
#include "paimon/common/memory/memory_slice_input.h"
@@ -44,22 +44,24 @@ Result<std::shared_ptr<LazyFilteredBTreeReader>>
LazyFilteredBTreeReader::Create
const std::shared_ptr<GlobalIndexFileReader>& file_reader,
const std::shared_ptr<CacheManager>& cache_manager, const
std::shared_ptr<MemoryPool>& pool,
const std::shared_ptr<Executor>& executor) {
- PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BTreeFileMetaSelector>
file_selector,
- BTreeFileMetaSelector::Create(files, key_type,
pool));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(key_type, pool));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SortedFileMetaSelector>
file_selector,
+ SortedFileMetaSelector::Create(files,
key_serializer));
return std::shared_ptr<LazyFilteredBTreeReader>(
- new LazyFilteredBTreeReader(read_buffer_size,
std::move(file_selector), key_type,
+ new LazyFilteredBTreeReader(read_buffer_size,
std::move(file_selector), key_serializer,
file_reader, cache_manager, pool,
executor));
}
LazyFilteredBTreeReader::LazyFilteredBTreeReader(
- std::optional<int32_t> read_buffer_size,
std::unique_ptr<BTreeFileMetaSelector> file_selector,
- std::shared_ptr<arrow::DataType> key_type,
std::shared_ptr<GlobalIndexFileReader> file_reader,
- std::shared_ptr<CacheManager> cache_manager, std::shared_ptr<MemoryPool>
pool,
- std::shared_ptr<Executor> executor)
+ std::optional<int32_t> read_buffer_size,
std::unique_ptr<SortedFileMetaSelector> file_selector,
+ std::shared_ptr<KeySerializer> key_serializer,
+ std::shared_ptr<GlobalIndexFileReader> file_reader,
std::shared_ptr<CacheManager> cache_manager,
+ std::shared_ptr<MemoryPool> pool, std::shared_ptr<Executor> executor)
: read_buffer_size_(read_buffer_size),
pool_(std::move(pool)),
file_selector_(std::move(file_selector)),
- key_type_(std::move(key_type)),
+ key_serializer_(std::move(key_serializer)),
file_reader_(std::move(file_reader)),
cache_manager_(std::move(cache_manager)),
executor_(std::move(executor)) {}
@@ -224,11 +226,11 @@ Result<std::shared_ptr<GlobalIndexReader>>
LazyFilteredBTreeReader::GetOrCreateR
Result<std::shared_ptr<GlobalIndexReader>>
LazyFilteredBTreeReader::CreateSingleReader(
const GlobalIndexIOMeta& meta) {
// Create comparator based on field type
- auto comparator = KeySerializer::CreateComparator(key_type_, pool_);
+ auto comparator = key_serializer_->CreateComparator();
// Get min/max key slices from meta data (keep as slices; Create() will
deserialize)
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<BTreeIndexMeta> index_meta,
- BTreeIndexMeta::Deserialize(meta.metadata,
pool_.get()));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SortedIndexFileMeta> index_meta,
+ SortedIndexFileMeta::Deserialize(meta.metadata,
pool_.get()));
std::optional<MemorySlice> min_key_slice;
std::optional<MemorySlice> max_key_slice;
if (index_meta->FirstKey()) {
@@ -270,7 +272,7 @@ Result<std::shared_ptr<GlobalIndexReader>>
LazyFilteredBTreeReader::CreateSingle
comparator, block_cache, pool_));
return BTreeGlobalIndexReader::Create(sst_file_reader,
std::move(null_bitmap), min_key_slice,
- max_key_slice, key_type_, pool_);
+ max_key_slice, key_serializer_);
}
Result<RoaringBitmap64> LazyFilteredBTreeReader::ReadNullBitmap(
diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
index 74ed978b..46681c1e 100644
--- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
+++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
@@ -28,7 +28,7 @@
#include "arrow/api.h"
#include "paimon/common/global_index/btree/btree_defs.h"
-#include "paimon/common/global_index/btree/btree_file_meta_selector.h"
+#include "paimon/common/global_index/sorted_file_meta_selector.h"
#include "paimon/common/io/cache/cache_manager.h"
#include "paimon/common/sst/block_cache.h"
#include "paimon/common/sst/block_handle.h"
@@ -80,8 +80,8 @@ class LazyFilteredBTreeReader : public GlobalIndexReader {
private:
LazyFilteredBTreeReader(std::optional<int32_t> read_buffer_size,
- std::unique_ptr<BTreeFileMetaSelector>
file_selector,
- std::shared_ptr<arrow::DataType> key_type,
+ std::unique_ptr<SortedFileMetaSelector>
file_selector,
+ std::shared_ptr<KeySerializer> key_serializer,
std::shared_ptr<GlobalIndexFileReader> file_reader,
std::shared_ptr<CacheManager> cache_manager,
std::shared_ptr<MemoryPool> pool,
std::shared_ptr<Executor> executor);
@@ -102,8 +102,8 @@ class LazyFilteredBTreeReader : public GlobalIndexReader {
private:
std::optional<int32_t> read_buffer_size_;
std::shared_ptr<MemoryPool> pool_;
- std::unique_ptr<BTreeFileMetaSelector> file_selector_;
- std::shared_ptr<arrow::DataType> key_type_;
+ std::unique_ptr<SortedFileMetaSelector> file_selector_;
+ std::shared_ptr<KeySerializer> key_serializer_;
std::shared_ptr<GlobalIndexFileReader> file_reader_;
std::shared_ptr<CacheManager> cache_manager_;
std::map<std::string, std::shared_ptr<GlobalIndexReader>> reader_cache_;
diff --git a/src/paimon/common/global_index/btree/key_serializer.cpp
b/src/paimon/common/global_index/key_serializer.cpp
similarity index 87%
rename from src/paimon/common/global_index/btree/key_serializer.cpp
rename to src/paimon/common/global_index/key_serializer.cpp
index 4b66d1d1..33bcc1c9 100644
--- a/src/paimon/common/global_index/btree/key_serializer.cpp
+++ b/src/paimon/common/global_index/key_serializer.cpp
@@ -17,7 +17,13 @@
* under the License.
*/
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <string_view>
+#include <vector>
#include "fmt/format.h"
#include "paimon/common/memory/memory_slice_input.h"
@@ -124,8 +130,36 @@ Status ValidateDecimal(const MemorySlice& slice,
} // namespace
-Result<std::shared_ptr<Bytes>> KeySerializer::SerializeKey(
- const Literal& literal, const std::shared_ptr<arrow::DataType>& type,
MemoryPool* pool) {
+Result<std::shared_ptr<KeySerializer>> KeySerializer::Create(
+ const std::shared_ptr<arrow::DataType>& type, const
std::shared_ptr<MemoryPool>& pool) {
+ if (type == nullptr) {
+ return Status::Invalid("Cannot create KeySerializer without a key
type.");
+ }
+ if (pool == nullptr) {
+ return Status::Invalid("Cannot create KeySerializer without a memory
pool.");
+ }
+ switch (type->id()) {
+ case arrow::Type::type::BOOL:
+ case arrow::Type::type::INT8:
+ case arrow::Type::type::INT16:
+ case arrow::Type::type::INT32:
+ case arrow::Type::type::INT64:
+ case arrow::Type::type::DATE32:
+ case arrow::Type::type::FLOAT:
+ case arrow::Type::type::DOUBLE:
+ case arrow::Type::type::STRING:
+ case arrow::Type::type::TIMESTAMP:
+ case arrow::Type::type::DECIMAL128:
+ return std::shared_ptr<KeySerializer>(new KeySerializer(type,
pool));
+ default:
+ return Status::Invalid(fmt::format("Data type {} is not supported
by global index now.",
+ type->ToString()));
+ }
+}
+
+Result<std::shared_ptr<Bytes>> KeySerializer::Serialize(const Literal&
literal) const {
+ const std::shared_ptr<arrow::DataType>& type = type_;
+ MemoryPool* pool = pool_.get();
if (literal.IsNull()) {
return Status::Invalid("cannot serialize null in KeySerializer");
}
@@ -182,7 +216,7 @@ Result<std::shared_ptr<Bytes>> KeySerializer::SerializeKey(
case FieldType::TIMESTAMP: {
if (!type || type->id() != arrow::Type::TIMESTAMP) {
return Status::Invalid(
- "ts type cannot cast to arrow::TimestampType in
BTreeGlobalIndex");
+ "timestamp type cannot cast to arrow::TimestampType in
KeySerializer");
}
auto ts_type = checked_pointer_cast<arrow::TimestampType>(type);
MemorySliceOutput output(8, pool);
@@ -199,7 +233,7 @@ Result<std::shared_ptr<Bytes>> KeySerializer::SerializeKey(
case FieldType::DECIMAL: {
if (!type || type->id() != arrow::Type::DECIMAL128) {
return Status::Invalid(
- "decimal type cannot cast to arrow::Decimal128Type in
BTreeGlobalIndex");
+ "decimal type cannot cast to arrow::Decimal128Type in
KeySerializer");
}
auto decimal_type =
checked_pointer_cast<arrow::Decimal128Type>(type);
@@ -218,15 +252,15 @@ Result<std::shared_ptr<Bytes>>
KeySerializer::SerializeKey(
}
default:
return Status::Invalid(
- fmt::format("Not support serialize {} type in
BTreeGlobalIndex",
+ fmt::format("Not support serialize {} type in KeySerializer",
FieldTypeUtils::FieldTypeToString(literal.GetType())));
}
}
-Result<Literal> KeySerializer::DeserializeKey(const MemorySlice& slice,
- const
std::shared_ptr<arrow::DataType>& type,
- MemoryPool* pool) {
- PAIMON_RETURN_NOT_OK(ValidateSerializedKey(slice, type));
+Result<Literal> KeySerializer::Deserialize(const MemorySlice& slice) const {
+ const std::shared_ptr<arrow::DataType>& type = type_;
+ MemoryPool* pool = pool_.get();
+ PAIMON_RETURN_NOT_OK(ValidateSerializedKey(slice));
switch (type->id()) {
case arrow::Type::type::BOOL:
return Literal(slice.ReadByte(0) == 1 ? true : false);
@@ -279,16 +313,13 @@ Result<Literal> KeySerializer::DeserializeKey(const
MemorySlice& slice,
}
}
default:
- return Status::Invalid(fmt::format(
- "Not support deserialize {} type in BTreeGlobalIndex",
type->ToString()));
+ return Status::Invalid(
+ fmt::format("Not support deserialize {} type in
KeySerializer", type->ToString()));
}
}
-Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice,
- const
std::shared_ptr<arrow::DataType>& type) {
- if (type == nullptr) {
- return Status::Invalid("Cannot validate a serialized BTree key without
a key type.");
- }
+Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice) const {
+ const std::shared_ptr<arrow::DataType>& type = type_;
switch (type->id()) {
case arrow::Type::type::BOOL: {
PAIMON_RETURN_NOT_OK(ValidateExactLength(slice, type,
sizeof(int8_t)));
@@ -325,12 +356,13 @@ Status KeySerializer::ValidateSerializedKey(const
MemorySlice& slice,
}
default:
return Status::Invalid(fmt::format(
- "Not support validate serialized {} type in BTreeGlobalIndex",
type->ToString()));
+ "Not support validate serialized {} type in KeySerializer",
type->ToString()));
}
}
-MemorySlice::SliceComparator KeySerializer::CreateComparator(
- const std::shared_ptr<arrow::DataType>& type, const
std::shared_ptr<MemoryPool>& pool) {
+MemorySlice::SliceComparator KeySerializer::CreateComparator() const {
+ const std::shared_ptr<arrow::DataType>& type = type_;
+ const std::shared_ptr<MemoryPool>& pool = pool_;
// Fast paths for integer and string types: direct value comparison
without Literal
// deserialization, avoiding heap allocations entirely.
switch (type->id()) {
@@ -393,8 +425,9 @@ MemorySlice::SliceComparator
KeySerializer::CreateComparator(
// deserialize to Literal and compare.
return
[pool = pool, type = type](const MemorySlice& a, const MemorySlice& b)
-> Result<int32_t> {
- PAIMON_ASSIGN_OR_RAISE(Literal la, DeserializeKey(a, type,
pool.get()));
- PAIMON_ASSIGN_OR_RAISE(Literal lb, DeserializeKey(b, type,
pool.get()));
+ KeySerializer serializer(type, pool);
+ PAIMON_ASSIGN_OR_RAISE(Literal la, serializer.Deserialize(a));
+ PAIMON_ASSIGN_OR_RAISE(Literal lb, serializer.Deserialize(b));
return la.CompareTo(lb);
};
}
diff --git a/src/paimon/common/global_index/btree/key_serializer.h
b/src/paimon/common/global_index/key_serializer.h
similarity index 59%
rename from src/paimon/common/global_index/btree/key_serializer.h
rename to src/paimon/common/global_index/key_serializer.h
index d44b5bdc..09b5298d 100644
--- a/src/paimon/common/global_index/btree/key_serializer.h
+++ b/src/paimon/common/global_index/key_serializer.h
@@ -19,29 +19,41 @@
#pragma once
+#include <memory>
+
#include "arrow/api.h"
#include "paimon/common/memory/memory_slice.h"
#include "paimon/memory/bytes.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/predicate/literal.h"
namespace paimon {
+
+/// Provides core methods to serialize, deserialize, and compare global index
keys.
class KeySerializer {
public:
- KeySerializer() = delete;
- ~KeySerializer() = delete;
+ ~KeySerializer() = default;
- static Result<std::shared_ptr<Bytes>> SerializeKey(const Literal& literal,
- const
std::shared_ptr<arrow::DataType>& type,
- MemoryPool* pool);
+ static Result<std::shared_ptr<KeySerializer>> Create(
+ const std::shared_ptr<arrow::DataType>& type, const
std::shared_ptr<MemoryPool>& pool);
- static Result<Literal> DeserializeKey(const MemorySlice& slice,
- const
std::shared_ptr<arrow::DataType>& type,
- MemoryPool* pool);
+ Result<std::shared_ptr<Bytes>> Serialize(const Literal& literal) const;
- static Status ValidateSerializedKey(const MemorySlice& slice,
- const
std::shared_ptr<arrow::DataType>& type);
+ Result<Literal> Deserialize(const MemorySlice& slice) const;
- static MemorySlice::SliceComparator CreateComparator(
- const std::shared_ptr<arrow::DataType>& type, const
std::shared_ptr<MemoryPool>& pool);
+ Status ValidateSerializedKey(const MemorySlice& slice) const;
+
+ MemorySlice::SliceComparator CreateComparator() const;
+
+ MemoryPool* GetMemoryPool() const {
+ return pool_.get();
+ }
+
+ private:
+ KeySerializer(const std::shared_ptr<arrow::DataType>& type,
+ const std::shared_ptr<MemoryPool>& pool)
+ : type_(type), pool_(pool) {}
+
+ std::shared_ptr<arrow::DataType> type_;
+ std::shared_ptr<MemoryPool> pool_;
};
} // namespace paimon
diff --git a/src/paimon/common/global_index/btree/key_serializer_test.cpp
b/src/paimon/common/global_index/key_serializer_test.cpp
similarity index 53%
rename from src/paimon/common/global_index/btree/key_serializer_test.cpp
rename to src/paimon/common/global_index/key_serializer_test.cpp
index 61322fde..8d29c2fc 100644
--- a/src/paimon/common/global_index/btree/key_serializer_test.cpp
+++ b/src/paimon/common/global_index/key_serializer_test.cpp
@@ -17,7 +17,7 @@
* under the License.
*/
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
#include <cstdint>
#include <string>
@@ -36,6 +36,29 @@ class KeySerializerTest : public ::testing::Test {
pool_ = GetDefaultPool();
}
+ Result<std::shared_ptr<Bytes>> Serialize(const Literal& literal,
+ const
std::shared_ptr<arrow::DataType>& type,
+ MemoryPool* /*pool*/) const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(type, pool_));
+ return serializer->Serialize(literal);
+ }
+
+ Result<Literal> Deserialize(const MemorySlice& slice,
+ const std::shared_ptr<arrow::DataType>& type,
+ MemoryPool* /*pool*/) const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(type, pool_));
+ return serializer->Deserialize(slice);
+ }
+
+ Status ValidateSerializedKey(const MemorySlice& slice,
+ const std::shared_ptr<arrow::DataType>& type)
const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(type, pool_));
+ return serializer->ValidateSerializedKey(slice);
+ }
+
std::shared_ptr<MemoryPool> pool_;
};
@@ -43,63 +66,51 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) {
// BOOLEAN
{
Literal literal(true);
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::boolean(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::boolean(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::boolean(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::boolean(),
pool_.get()));
ASSERT_EQ(result.GetValue<bool>(), true);
Literal literal_false(false);
- ASSERT_OK_AND_ASSIGN(
- bytes, KeySerializer::SerializeKey(literal_false,
arrow::boolean(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(bytes, Serialize(literal_false, arrow::boolean(),
pool_.get()));
slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(result,
- KeySerializer::DeserializeKey(slice,
arrow::boolean(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(result, Deserialize(slice, arrow::boolean(),
pool_.get()));
ASSERT_EQ(result.GetValue<bool>(), false);
}
// TINYINT (int8)
{
Literal literal(static_cast<int8_t>(-42));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::int8(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int8(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::int8(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int8(),
pool_.get()));
ASSERT_EQ(result.GetValue<int8_t>(), -42);
}
// SMALLINT (int16)
{
Literal literal(static_cast<int16_t>(12345));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::int16(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int16(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::int16(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int16(),
pool_.get()));
ASSERT_EQ(result.GetValue<int16_t>(), 12345);
}
// INT (int32)
{
Literal literal(42);
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::int32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int32(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::int32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int32(),
pool_.get()));
ASSERT_EQ(result.GetValue<int32_t>(), 42);
}
// DATE (stored as int32)
{
Literal literal(FieldType::DATE, 18000);
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::date32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::date32(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::date32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::date32(),
pool_.get()));
ASSERT_EQ(result.GetType(), FieldType::DATE);
ASSERT_EQ(result.GetValue<int32_t>(), 18000);
}
@@ -107,44 +118,36 @@ TEST_F(KeySerializerTest,
SerializeAndDeserializeAllTypes) {
// BIGINT (int64)
{
Literal literal(static_cast<int64_t>(123456789012345LL));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::int64(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int64(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::int64(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int64(),
pool_.get()));
ASSERT_EQ(result.GetValue<int64_t>(), 123456789012345LL);
}
// FLOAT
{
Literal literal(3.14f);
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::float32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::float32(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::float32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::float32(),
pool_.get()));
ASSERT_FLOAT_EQ(result.GetValue<float>(), 3.14f);
}
// DOUBLE
{
Literal literal(2.718281828);
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::float64(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::float64(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::float64(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::float64(),
pool_.get()));
ASSERT_DOUBLE_EQ(result.GetValue<double>(), 2.718281828);
}
// STRING
{
Literal literal(FieldType::STRING, "hello world", 11);
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
arrow::utf8(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::utf8(),
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
arrow::utf8(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::utf8(),
pool_.get()));
ASSERT_EQ(result.GetValue<std::string>(), "hello world");
}
@@ -152,11 +155,9 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes)
{
{
auto ts_type = arrow::timestamp(arrow::TimeUnit::MILLI);
Literal literal(Timestamp::FromEpochMillis(1234567890));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal, ts_type,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, ts_type,
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice, ts_type,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, ts_type,
pool_.get()));
ASSERT_EQ(result.GetValue<Timestamp>().GetMillisecond(), 1234567890);
}
@@ -164,11 +165,9 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes)
{
{
auto ts_type = arrow::timestamp(arrow::TimeUnit::NANO);
Literal literal(Timestamp(5000, 123456));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal, ts_type,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, ts_type,
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice, ts_type,
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, ts_type,
pool_.get()));
ASSERT_EQ(result.GetValue<Timestamp>().GetMillisecond(), 5000);
ASSERT_EQ(result.GetValue<Timestamp>().GetNanoOfMillisecond(), 123456);
}
@@ -177,11 +176,9 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes)
{
{
auto decimal_type = arrow::decimal128(10, 2);
Literal literal(Decimal::FromUnscaledLong(12345, 10, 2));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
decimal_type, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, decimal_type,
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
decimal_type, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, decimal_type,
pool_.get()));
ASSERT_EQ(result.GetValue<Decimal>().ToUnscaledLong(), 12345);
}
@@ -189,49 +186,49 @@ TEST_F(KeySerializerTest,
SerializeAndDeserializeAllTypes) {
{
auto decimal_type = arrow::decimal128(25, 3);
Literal literal(Decimal(25, 3, 9999999));
- ASSERT_OK_AND_ASSIGN(auto bytes,
- KeySerializer::SerializeKey(literal,
decimal_type, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, decimal_type,
pool_.get()));
auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes));
- ASSERT_OK_AND_ASSIGN(auto result,
- KeySerializer::DeserializeKey(slice,
decimal_type, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, decimal_type,
pool_.get()));
ASSERT_EQ(result.GetValue<Decimal>().ToString(),
literal.GetValue<Decimal>().ToString());
}
// NULL should fail
{
Literal null_literal(FieldType::INT);
- ASSERT_NOK_WITH_MSG(KeySerializer::SerializeKey(null_literal,
arrow::int32(), pool_.get()),
+ ASSERT_NOK_WITH_MSG(Serialize(null_literal, arrow::int32(),
pool_.get()),
"cannot serialize null in KeySerializer");
}
// unsupported type
{
- Literal literal(FieldType::BINARY, "ab", 2);
- ASSERT_NOK_WITH_MSG(KeySerializer::SerializeKey(literal,
arrow::binary(), pool_.get()),
- "Not support serialize BINARY type in
BTreeGlobalIndex");
+ ASSERT_NOK_WITH_MSG(KeySerializer::Create(arrow::binary(), pool_),
+ "is not supported by global index now");
}
}
+TEST_F(KeySerializerTest, CreateRejectsMissingDependencies) {
+ ASSERT_NOK_WITH_MSG(KeySerializer::Create(nullptr, pool_),
+ "Cannot create KeySerializer without a key type");
+ ASSERT_NOK_WITH_MSG(KeySerializer::Create(arrow::int32(), nullptr),
+ "Cannot create KeySerializer without a memory pool");
+}
+
TEST_F(KeySerializerTest, CanonicalizesFloatingPointNaN) {
const auto float_nan = FloatingPointFromBits<float>(0xffc12345U);
const auto canonical_float_nan =
FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<Bytes> float_bytes,
- KeySerializer::SerializeKey(Literal(float_nan), arrow::float32(),
pool_.get()));
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<Bytes> canonical_float_bytes,
- KeySerializer::SerializeKey(Literal(canonical_float_nan),
arrow::float32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> float_bytes,
+ Serialize(Literal(float_nan), arrow::float32(),
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> canonical_float_bytes,
+ Serialize(Literal(canonical_float_nan),
arrow::float32(), pool_.get()));
ASSERT_EQ(std::string(float_bytes->data(), float_bytes->size()),
std::string(canonical_float_bytes->data(),
canonical_float_bytes->size()));
const auto double_nan =
FloatingPointFromBits<double>(0xfff8123456789abcULL);
const auto canonical_double_nan =
FloatingPointFromBits<double>(kCanonicalDoubleNaNBits);
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<Bytes> double_bytes,
- KeySerializer::SerializeKey(Literal(double_nan), arrow::float64(),
pool_.get()));
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<Bytes> canonical_double_bytes,
- KeySerializer::SerializeKey(Literal(canonical_double_nan),
arrow::float64(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> double_bytes,
+ Serialize(Literal(double_nan), arrow::float64(),
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> canonical_double_bytes,
+ Serialize(Literal(canonical_double_nan),
arrow::float64(), pool_.get()));
ASSERT_EQ(std::string(double_bytes->data(), double_bytes->size()),
std::string(canonical_double_bytes->data(),
canonical_double_bytes->size()));
}
@@ -241,50 +238,46 @@ TEST_F(KeySerializerTest, RejectsMalformedSerializedKeys)
{
return MemorySlice::Wrap(std::make_shared<Bytes>(value, pool_.get()));
};
- ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(std::string(3,
'\0')), arrow::int32()));
- ASSERT_NOK(
- KeySerializer::DeserializeKey(wrap(std::string(3, '\0')),
arrow::int32(), pool_.get()));
+ ASSERT_NOK(ValidateSerializedKey(wrap(std::string(3, '\0')),
arrow::int32()));
+ ASSERT_NOK(Deserialize(wrap(std::string(3, '\0')), arrow::int32(),
pool_.get()));
std::string invalid_boolean(1, static_cast<char>(2));
- ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(invalid_boolean),
arrow::boolean()));
+ ASSERT_NOK(ValidateSerializedKey(wrap(invalid_boolean), arrow::boolean()));
auto nanos_timestamp = arrow::timestamp(arrow::TimeUnit::NANO);
std::string unterminated_timestamp(9, '\0');
unterminated_timestamp[8] = static_cast<char>(0x80);
-
ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(unterminated_timestamp),
nanos_timestamp));
+ ASSERT_NOK(ValidateSerializedKey(wrap(unterminated_timestamp),
nanos_timestamp));
std::string out_of_range_timestamp(11, '\0');
out_of_range_timestamp[8] = static_cast<char>(0xC0);
out_of_range_timestamp[9] = static_cast<char>(0x84);
out_of_range_timestamp[10] = static_cast<char>(0x3D);
-
ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(out_of_range_timestamp),
nanos_timestamp));
+ ASSERT_NOK(ValidateSerializedKey(wrap(out_of_range_timestamp),
nanos_timestamp));
auto non_compact_decimal = arrow::decimal128(25, 3);
- ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(""),
non_compact_decimal));
- ASSERT_NOK(
- KeySerializer::ValidateSerializedKey(wrap(std::string(17, '\0')),
non_compact_decimal));
+ ASSERT_NOK(ValidateSerializedKey(wrap(""), non_compact_decimal));
+ ASSERT_NOK(ValidateSerializedKey(wrap(std::string(17, '\0')),
non_compact_decimal));
auto narrow_decimal = arrow::decimal128(1, 0);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> out_of_range_decimal,
-
KeySerializer::SerializeKey(Literal(Decimal::FromUnscaledLong(10, 1, 0)),
- narrow_decimal,
pool_.get()));
-
ASSERT_NOK(KeySerializer::ValidateSerializedKey(MemorySlice::Wrap(out_of_range_decimal),
- narrow_decimal));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<Bytes> out_of_range_decimal,
+ Serialize(Literal(Decimal::FromUnscaledLong(10, 1, 0)),
narrow_decimal, pool_.get()));
+ ASSERT_NOK(ValidateSerializedKey(MemorySlice::Wrap(out_of_range_decimal),
narrow_decimal));
}
TEST_F(KeySerializerTest, CreateComparator) {
// INT comparator
{
- auto comparator = KeySerializer::CreateComparator(arrow::int32(),
pool_);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(arrow::int32(), pool_));
+ auto comparator = serializer->CreateComparator();
Literal literal_1(1);
Literal literal_2(2);
Literal literal_3(1);
- ASSERT_OK_AND_ASSIGN(auto bytes_1,
- KeySerializer::SerializeKey(literal_1,
arrow::int32(), pool_.get()));
- ASSERT_OK_AND_ASSIGN(auto bytes_2,
- KeySerializer::SerializeKey(literal_2,
arrow::int32(), pool_.get()));
- ASSERT_OK_AND_ASSIGN(auto bytes_3,
- KeySerializer::SerializeKey(literal_3,
arrow::int32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes_1, Serialize(literal_1,
arrow::int32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes_2, Serialize(literal_2,
arrow::int32(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes_3, Serialize(literal_3,
arrow::int32(), pool_.get()));
auto slice_1 = MemorySlice::Wrap(MemorySegment::Wrap(bytes_1));
auto slice_2 = MemorySlice::Wrap(MemorySegment::Wrap(bytes_2));
auto slice_3 = MemorySlice::Wrap(MemorySegment::Wrap(bytes_3));
@@ -299,16 +292,15 @@ TEST_F(KeySerializerTest, CreateComparator) {
// STRING comparator
{
- auto comparator = KeySerializer::CreateComparator(arrow::utf8(),
pool_);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(arrow::utf8(), pool_));
+ auto comparator = serializer->CreateComparator();
Literal literal_a(FieldType::STRING, "apple", 5);
Literal literal_b(FieldType::STRING, "banana", 6);
Literal literal_c(FieldType::STRING, "apple", 5);
- ASSERT_OK_AND_ASSIGN(auto bytes_a,
- KeySerializer::SerializeKey(literal_a,
arrow::utf8(), pool_.get()));
- ASSERT_OK_AND_ASSIGN(auto bytes_b,
- KeySerializer::SerializeKey(literal_b,
arrow::utf8(), pool_.get()));
- ASSERT_OK_AND_ASSIGN(auto bytes_c,
- KeySerializer::SerializeKey(literal_c,
arrow::utf8(), pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes_a, Serialize(literal_a, arrow::utf8(),
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes_b, Serialize(literal_b, arrow::utf8(),
pool_.get()));
+ ASSERT_OK_AND_ASSIGN(auto bytes_c, Serialize(literal_c, arrow::utf8(),
pool_.get()));
auto slice_a = MemorySlice::Wrap(MemorySegment::Wrap(bytes_a));
auto slice_b = MemorySlice::Wrap(MemorySegment::Wrap(bytes_b));
auto slice_c = MemorySlice::Wrap(MemorySegment::Wrap(bytes_c));
diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp
b/src/paimon/common/global_index/sorted_file_meta_selector.cpp
similarity index 51%
rename from src/paimon/common/global_index/btree/btree_file_meta_selector.cpp
rename to src/paimon/common/global_index/sorted_file_meta_selector.cpp
index aa2f23c7..365a0966 100644
--- a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp
+++ b/src/paimon/common/global_index/sorted_file_meta_selector.cpp
@@ -17,47 +17,45 @@
* under the License.
*/
-#include "paimon/common/global_index/btree/btree_file_meta_selector.h"
+#include "paimon/common/global_index/sorted_file_meta_selector.h"
#include "fmt/format.h"
#include "paimon/common/memory/memory_slice.h"
namespace paimon {
-Result<std::unique_ptr<BTreeFileMetaSelector>> BTreeFileMetaSelector::Create(
- const std::vector<GlobalIndexIOMeta>& files, const
std::shared_ptr<arrow::DataType>& key_type,
- const std::shared_ptr<MemoryPool>& pool) {
- if (key_type == nullptr) {
- return Status::Invalid("Cannot create a BTree file metadata selector
without a key type.");
- }
- if (pool == nullptr) {
+Result<std::unique_ptr<SortedFileMetaSelector>> SortedFileMetaSelector::Create(
+ const std::vector<GlobalIndexIOMeta>& files,
+ const std::shared_ptr<KeySerializer>& key_serializer) {
+ if (key_serializer == nullptr) {
return Status::Invalid(
- "Cannot create a BTree file metadata selector without a memory
pool.");
+ "Cannot create a sorted file metadata selector without a key
serializer.");
}
- std::vector<std::pair<GlobalIndexIOMeta, std::shared_ptr<BTreeIndexMeta>>>
decoded_files;
+ std::vector<std::pair<GlobalIndexIOMeta,
std::shared_ptr<SortedIndexFileMeta>>> decoded_files;
decoded_files.reserve(files.size());
- MemorySlice::SliceComparator comparator =
KeySerializer::CreateComparator(key_type, pool);
+ MemorySlice::SliceComparator comparator =
key_serializer->CreateComparator();
for (const auto& file : files) {
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<BTreeIndexMeta> index_meta,
- BTreeIndexMeta::Deserialize(file.metadata,
pool.get()));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<SortedIndexFileMeta> index_meta,
+ SortedIndexFileMeta::Deserialize(file.metadata,
key_serializer->GetMemoryPool()));
bool has_first_key = index_meta->FirstKey() != nullptr;
bool has_last_key = index_meta->LastKey() != nullptr;
if (has_first_key != has_last_key) {
return Status::Invalid(fmt::format(
- "BTree index metadata for {} must contain both boundary keys
or neither.",
+ "Sorted index file metadata for {} must contain both boundary
keys or neither.",
file.file_path));
}
if (!has_first_key && !index_meta->HasNulls()) {
- return Status::Invalid(
- fmt::format("BTree index metadata for {} has no boundary keys
or null values.",
- file.file_path));
+ return Status::Invalid(fmt::format(
+ "Sorted index file metadata for {} has no boundary keys or
null values.",
+ file.file_path));
}
if (index_meta->FirstKey() != nullptr) {
- PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(
- WrapKeySlice(index_meta->FirstKey()), key_type));
+ PAIMON_RETURN_NOT_OK(
+
key_serializer->ValidateSerializedKey(WrapKeySlice(index_meta->FirstKey())));
}
if (index_meta->LastKey() != nullptr) {
- PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(
- WrapKeySlice(index_meta->LastKey()), key_type));
+ PAIMON_RETURN_NOT_OK(
+
key_serializer->ValidateSerializedKey(WrapKeySlice(index_meta->LastKey())));
}
if (index_meta->FirstKey() != nullptr && index_meta->LastKey() !=
nullptr) {
PAIMON_ASSIGN_OR_RAISE(int32_t comparison,
@@ -65,35 +63,35 @@ Result<std::unique_ptr<BTreeFileMetaSelector>>
BTreeFileMetaSelector::Create(
WrapKeySlice(index_meta->LastKey())));
if (comparison > 0) {
return Status::Invalid(fmt::format(
- "BTree index metadata for {} has a first key greater than
its last key.",
+ "Sorted index file metadata for {} has a first key greater
than its last key.",
file.file_path));
}
}
decoded_files.emplace_back(file, std::move(index_meta));
}
- return std::unique_ptr<BTreeFileMetaSelector>(
- new BTreeFileMetaSelector(std::move(decoded_files), key_type, pool));
+ return std::unique_ptr<SortedFileMetaSelector>(
+ new SortedFileMetaSelector(std::move(decoded_files), key_serializer));
}
-BTreeFileMetaSelector::BTreeFileMetaSelector(
- std::vector<std::pair<GlobalIndexIOMeta, std::shared_ptr<BTreeIndexMeta>>>
files,
- std::shared_ptr<arrow::DataType> key_type, std::shared_ptr<MemoryPool>
pool)
+SortedFileMetaSelector::SortedFileMetaSelector(
+ std::vector<std::pair<GlobalIndexIOMeta,
std::shared_ptr<SortedIndexFileMeta>>> files,
+ std::shared_ptr<KeySerializer> key_serializer)
: files_(std::move(files)),
- key_type_(std::move(key_type)),
- pool_(std::move(pool)),
- comparator_(KeySerializer::CreateComparator(key_type_, pool_)) {}
+ key_serializer_(std::move(key_serializer)),
+ comparator_(key_serializer_->CreateComparator()) {}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitIsNotNull()
{
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
!meta.OnlyNulls(); });
+Result<std::vector<GlobalIndexIOMeta>>
SortedFileMetaSelector::VisitIsNotNull() {
+ return Filter(
+ [](const SortedIndexFileMeta& meta) -> Result<bool> { return
!meta.OnlyNulls(); });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitIsNull() {
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
meta.HasNulls(); });
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitIsNull() {
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
meta.HasNulls(); });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitEqual(const
Literal& literal) {
+Result<std::vector<GlobalIndexIOMeta>>
SortedFileMetaSelector::VisitEqual(const Literal& literal) {
PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice,
SerializeLiteral(literal));
- return Filter([this, &literal_slice](const BTreeIndexMeta& meta) ->
Result<bool> {
+ return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) ->
Result<bool> {
if (meta.OnlyNulls()) {
return false;
}
@@ -101,16 +99,16 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitEqual(const L
});
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitNotEqual(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitNotEqual(
const Literal& literal) {
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
true; });
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
true; });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitLessThan(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitLessThan(
const Literal& literal) {
// file.minKey < literal
PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice,
SerializeLiteral(literal));
- return Filter([this, &literal_slice](const BTreeIndexMeta& meta) ->
Result<bool> {
+ return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) ->
Result<bool> {
if (meta.OnlyNulls()) {
return false;
}
@@ -119,11 +117,11 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitLessThan(
});
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitLessOrEqual(
+Result<std::vector<GlobalIndexIOMeta>>
SortedFileMetaSelector::VisitLessOrEqual(
const Literal& literal) {
// file.minKey <= literal
PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice,
SerializeLiteral(literal));
- return Filter([this, &literal_slice](const BTreeIndexMeta& meta) ->
Result<bool> {
+ return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) ->
Result<bool> {
if (meta.OnlyNulls()) {
return false;
}
@@ -132,11 +130,11 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitLessOrEqual(
});
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitGreaterThan(
+Result<std::vector<GlobalIndexIOMeta>>
SortedFileMetaSelector::VisitGreaterThan(
const Literal& literal) {
// file.maxKey > literal
PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice,
SerializeLiteral(literal));
- return Filter([this, &literal_slice](const BTreeIndexMeta& meta) ->
Result<bool> {
+ return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) ->
Result<bool> {
if (meta.OnlyNulls()) {
return false;
}
@@ -145,11 +143,11 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitGreaterThan(
});
}
-Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitGreaterOrEqual(
+Result<std::vector<GlobalIndexIOMeta>>
SortedFileMetaSelector::VisitGreaterOrEqual(
const Literal& literal) {
// file.maxKey >= literal
PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice,
SerializeLiteral(literal));
- return Filter([this, &literal_slice](const BTreeIndexMeta& meta) ->
Result<bool> {
+ return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) ->
Result<bool> {
if (meta.OnlyNulls()) {
return false;
}
@@ -158,7 +156,7 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitGreaterOrEqua
});
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitIn(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitIn(
const std::vector<Literal>& literals) {
std::vector<MemorySlice> literal_slices;
literal_slices.reserve(literals.size());
@@ -166,7 +164,7 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitIn(
PAIMON_ASSIGN_OR_RAISE(MemorySlice slice, SerializeLiteral(literal));
literal_slices.push_back(std::move(slice));
}
- return Filter([this, &literal_slices](const BTreeIndexMeta& meta) ->
Result<bool> {
+ return Filter([this, &literal_slices](const SortedIndexFileMeta& meta) ->
Result<bool> {
if (meta.OnlyNulls()) {
return false;
}
@@ -180,31 +178,32 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitIn(
});
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitNotIn(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitNotIn(
const std::vector<Literal>& literals) {
// Cannot filter any file by NOT IN condition
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
true; });
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
true; });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitStartsWith(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitStartsWith(
const Literal& prefix) {
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
true; });
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
true; });
}
-Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::VisitEndsWith(const Literal& suffix) {
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
true; });
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitEndsWith(
+ const Literal& suffix) {
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
true; });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitContains(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitContains(
const Literal& literal) {
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
true; });
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
true; });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitLike(const
Literal& literal) {
- return Filter([](const BTreeIndexMeta& meta) -> Result<bool> { return
true; });
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::VisitLike(const
Literal& literal) {
+ return Filter([](const SortedIndexFileMeta& meta) -> Result<bool> { return
true; });
}
-Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::Filter(
+Result<std::vector<GlobalIndexIOMeta>> SortedFileMetaSelector::Filter(
const MetaPredicate& predicate) const {
std::vector<GlobalIndexIOMeta> result;
for (const auto& [io_meta, index_meta] : files_) {
@@ -216,8 +215,9 @@ Result<std::vector<GlobalIndexIOMeta>>
BTreeFileMetaSelector::Filter(
return result;
}
-Result<bool> BTreeFileMetaSelector::Overlaps(const BTreeIndexMeta& meta, const
MemorySlice& from,
- const MemorySlice& to) const {
+Result<bool> SortedFileMetaSelector::Overlaps(const SortedIndexFileMeta& meta,
+ const MemorySlice& from,
+ const MemorySlice& to) const {
if (meta.FirstKey()) {
PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(to,
WrapKeySlice(meta.FirstKey())));
if (cmp < 0) {
@@ -233,31 +233,30 @@ Result<bool> BTreeFileMetaSelector::Overlaps(const
BTreeIndexMeta& meta, const M
return true;
}
-Result<int32_t> BTreeFileMetaSelector::CompareFirstKey(const BTreeIndexMeta&
meta,
- const MemorySlice&
literal) const {
+Result<int32_t> SortedFileMetaSelector::CompareFirstKey(const
SortedIndexFileMeta& meta,
+ const MemorySlice&
literal) const {
if (!meta.FirstKey()) {
return -1;
}
return comparator_(WrapKeySlice(meta.FirstKey()), literal);
}
-Result<int32_t> BTreeFileMetaSelector::CompareLastKey(const BTreeIndexMeta&
meta,
- const MemorySlice&
literal) const {
+Result<int32_t> SortedFileMetaSelector::CompareLastKey(const
SortedIndexFileMeta& meta,
+ const MemorySlice&
literal) const {
if (!meta.LastKey()) {
return 1;
}
return comparator_(WrapKeySlice(meta.LastKey()), literal);
}
-MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr<Bytes>&
key) {
+MemorySlice SortedFileMetaSelector::WrapKeySlice(const std::shared_ptr<Bytes>&
key) {
return MemorySlice::Wrap(MemorySegment::WrapView(key->data(),
key->size()));
}
-Result<MemorySlice> BTreeFileMetaSelector::SerializeLiteral(const Literal&
literal) const {
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes,
- KeySerializer::SerializeKey(literal, key_type_,
pool_.get()));
+Result<MemorySlice> SortedFileMetaSelector::SerializeLiteral(const Literal&
literal) const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes,
key_serializer_->Serialize(literal));
MemorySlice slice = MemorySlice::Wrap(bytes);
- PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(slice,
key_type_));
+ PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(slice));
return slice;
}
diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.h
b/src/paimon/common/global_index/sorted_file_meta_selector.h
similarity index 71%
rename from src/paimon/common/global_index/btree/btree_file_meta_selector.h
rename to src/paimon/common/global_index/sorted_file_meta_selector.h
index ad066278..871bbab3 100644
--- a/src/paimon/common/global_index/btree/btree_file_meta_selector.h
+++ b/src/paimon/common/global_index/sorted_file_meta_selector.h
@@ -19,24 +19,27 @@
#pragma once
+#include <functional>
#include <memory>
#include <utility>
#include <vector>
-#include "paimon/common/global_index/btree/btree_index_meta.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/common/memory/memory_slice.h"
#include "paimon/global_index/global_index_io_meta.h"
#include "paimon/predicate/function_visitor.h"
namespace paimon {
-/// Selects candidate BTree index files based on filter predicates.
-class BTreeFileMetaSelector : public
FunctionVisitor<std::vector<GlobalIndexIOMeta>> {
+/// Selects candidate global index files by per-index-file min/max metadata.
+///
+/// All files are expected to belong to the same field.
+class SortedFileMetaSelector : public
FunctionVisitor<std::vector<GlobalIndexIOMeta>> {
public:
- static Result<std::unique_ptr<BTreeFileMetaSelector>> Create(
+ static Result<std::unique_ptr<SortedFileMetaSelector>> Create(
const std::vector<GlobalIndexIOMeta>& files,
- const std::shared_ptr<arrow::DataType>& key_type, const
std::shared_ptr<MemoryPool>& pool);
+ const std::shared_ptr<KeySerializer>& key_serializer);
Result<std::vector<GlobalIndexIOMeta>> VisitIsNotNull() override;
Result<std::vector<GlobalIndexIOMeta>> VisitIsNull() override;
@@ -55,20 +58,22 @@ class BTreeFileMetaSelector : public
FunctionVisitor<std::vector<GlobalIndexIOMe
Result<std::vector<GlobalIndexIOMeta>> VisitLike(const Literal& literal)
override;
private:
- BTreeFileMetaSelector(
- std::vector<std::pair<GlobalIndexIOMeta,
std::shared_ptr<BTreeIndexMeta>>> files,
- std::shared_ptr<arrow::DataType> key_type, std::shared_ptr<MemoryPool>
pool);
+ SortedFileMetaSelector(
+ std::vector<std::pair<GlobalIndexIOMeta,
std::shared_ptr<SortedIndexFileMeta>>> files,
+ std::shared_ptr<KeySerializer> key_serializer);
- using MetaPredicate = std::function<Result<bool>(const BTreeIndexMeta&)>;
+ using MetaPredicate = std::function<Result<bool>(const
SortedIndexFileMeta&)>;
Result<std::vector<GlobalIndexIOMeta>> Filter(const MetaPredicate&
predicate) const;
- Result<bool> Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from,
+ Result<bool> Overlaps(const SortedIndexFileMeta& meta, const MemorySlice&
from,
const MemorySlice& to) const;
- Result<int32_t> CompareFirstKey(const BTreeIndexMeta& meta, const
MemorySlice& literal) const;
+ Result<int32_t> CompareFirstKey(const SortedIndexFileMeta& meta,
+ const MemorySlice& literal) const;
- Result<int32_t> CompareLastKey(const BTreeIndexMeta& meta, const
MemorySlice& literal) const;
+ Result<int32_t> CompareLastKey(const SortedIndexFileMeta& meta,
+ const MemorySlice& literal) const;
Result<MemorySlice> SerializeLiteral(const Literal& literal) const;
@@ -76,9 +81,8 @@ class BTreeFileMetaSelector : public
FunctionVisitor<std::vector<GlobalIndexIOMe
/// avoiding shared_ptr reference-count overhead.
static MemorySlice WrapKeySlice(const std::shared_ptr<Bytes>& key);
- std::vector<std::pair<GlobalIndexIOMeta, std::shared_ptr<BTreeIndexMeta>>>
files_;
- std::shared_ptr<arrow::DataType> key_type_;
- std::shared_ptr<MemoryPool> pool_;
+ std::vector<std::pair<GlobalIndexIOMeta,
std::shared_ptr<SortedIndexFileMeta>>> files_;
+ std::shared_ptr<KeySerializer> key_serializer_;
MemorySlice::SliceComparator comparator_;
};
diff --git
a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp
b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp
similarity index 62%
rename from
src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp
rename to src/paimon/common/global_index/sorted_file_meta_selector_test.cpp
index bb025c11..eb0a843c 100644
--- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp
+++ b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp
@@ -17,7 +17,7 @@
* under the License.
*/
-#include "paimon/common/global_index/btree/btree_file_meta_selector.h"
+#include "paimon/common/global_index/sorted_file_meta_selector.h"
#include <algorithm>
#include <set>
@@ -25,31 +25,37 @@
#include <vector>
#include "gtest/gtest.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
-class BTreeFileMetaSelectorTest : public ::testing::Test {
+class SortedFileMetaSelectorTest : public ::testing::Test {
public:
void SetUp() override {
pool_ = GetDefaultPool();
key_type_ = arrow::int32();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(key_type_, pool_));
+ key_serializer_ = std::move(key_serializer);
// file1: keys [1, 10], has_nulls=true
// file2: keys [15, 20], has_nulls=false
// file3: keys [21, 30], has_nulls=true
// file4: keys [1, 5], has_nulls=false
// file5: keys [19, 25], has_nulls=true
- auto meta1 = std::make_shared<BTreeIndexMeta>(SerializeInt(1),
SerializeInt(10), true);
- auto meta2 = std::make_shared<BTreeIndexMeta>(SerializeInt(15),
SerializeInt(20), false);
- auto meta3 = std::make_shared<BTreeIndexMeta>(SerializeInt(21),
SerializeInt(30), true);
- auto meta4 = std::make_shared<BTreeIndexMeta>(SerializeInt(1),
SerializeInt(5), false);
- auto meta5 = std::make_shared<BTreeIndexMeta>(SerializeInt(19),
SerializeInt(25), true);
+ auto meta1 = std::make_shared<SortedIndexFileMeta>(SerializeInt(1),
SerializeInt(10), true);
+ auto meta2 =
+ std::make_shared<SortedIndexFileMeta>(SerializeInt(15),
SerializeInt(20), false);
+ auto meta3 =
+ std::make_shared<SortedIndexFileMeta>(SerializeInt(21),
SerializeInt(30), true);
+ auto meta4 = std::make_shared<SortedIndexFileMeta>(SerializeInt(1),
SerializeInt(5), false);
+ auto meta5 =
+ std::make_shared<SortedIndexFileMeta>(SerializeInt(19),
SerializeInt(25), true);
// file6: only-nulls file (no keys, has_nulls=true)
- auto meta6 = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, true);
+ auto meta6 = std::make_shared<SortedIndexFileMeta>(nullptr, nullptr,
true);
files_ = {
GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get())),
GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get())),
@@ -62,8 +68,7 @@ class BTreeFileMetaSelectorTest : public ::testing::Test {
std::shared_ptr<Bytes> SerializeInt(int32_t value) const {
Literal literal(value);
- EXPECT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result,
- KeySerializer::SerializeKey(literal, key_type_,
pool_.get()));
+ EXPECT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result,
key_serializer_->Serialize(literal));
return result;
}
@@ -83,12 +88,13 @@ class BTreeFileMetaSelectorTest : public ::testing::Test {
private:
std::shared_ptr<MemoryPool> pool_;
std::shared_ptr<arrow::DataType> key_type_;
+ std::shared_ptr<KeySerializer> key_serializer_;
std::vector<GlobalIndexIOMeta> files_;
};
-TEST_F(BTreeFileMetaSelectorTest, TestVisitLessThan) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitLessThan) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// minKey < 8: file1(1), file4(1)
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessThan(Literal(8)));
@@ -103,9 +109,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitLessThan) {
ASSERT_TRUE(result.empty());
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitLessOrEqual) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitLessOrEqual) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// minKey <= 20: file1(1), file2(15), file4(1), file5(19)
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessOrEqual(Literal(20)));
@@ -116,9 +122,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitLessOrEqual) {
CheckResult(result, {"file1", "file2", "file4"});
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterThan) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitGreaterThan) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// maxKey > 20: file3(30), file5(25)
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterThan(Literal(20)));
@@ -129,9 +135,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterThan) {
ASSERT_TRUE(result.empty());
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterOrEqual) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitGreaterOrEqual) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// maxKey >= 5: all non-null files (file1..file5)
ASSERT_OK_AND_ASSIGN(auto result,
selector->VisitGreaterOrEqual(Literal(5)));
@@ -142,9 +148,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterOrEqual) {
CheckResult(result, {"file2", "file3", "file5"});
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitEqual) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitEqual) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// 22 in [21,30] and [19,25]
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(22)));
@@ -162,36 +168,36 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitEqual) {
ASSERT_NOK(selector->VisitEqual(Literal(static_cast<int8_t>(1))));
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitNotEqual) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitNotEqual) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// NotEqual cannot prune any file, returns all
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitNotEqual(Literal(22)));
CheckResult(result, {"file1", "file2", "file3", "file4", "file5",
"file6"});
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNull) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitIsNull) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// has_nulls: file1, file3, file5, file6
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNull());
CheckResult(result, {"file1", "file3", "file5", "file6"});
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNotNull) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitIsNotNull) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// !onlyNulls: file1..file5 (file6 is only-nulls)
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNotNull());
CheckResult(result, {"file1", "file2", "file3", "file4", "file5"});
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitIn) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// IN(1, 2, 3, 26, 27, 28):
// 1 in [1,10]=file1, [1,5]=file4
@@ -209,9 +215,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) {
ASSERT_TRUE(result.empty());
}
-TEST_F(BTreeFileMetaSelectorTest, TestVisitNotIn) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestVisitNotIn) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// NotIn cannot prune any file
ASSERT_OK_AND_ASSIGN(auto result,
@@ -219,9 +225,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitNotIn) {
CheckResult(result, {"file1", "file2", "file3", "file4", "file5",
"file6"});
}
-TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) {
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files_, key_type_,
pool_));
+TEST_F(SortedFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files_,
key_serializer_));
// file6 is only-nulls, should be excluded from all range/equality queries
ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(1)));
@@ -242,29 +248,30 @@ TEST_F(BTreeFileMetaSelectorTest,
TestOnlyNullsFileExcludedFromRangeQueries) {
ASSERT_EQ(names.count("file6"), 1u);
}
-TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) {
+TEST_F(SortedFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) {
std::shared_ptr<MemoryPool> pool = GetDefaultPool();
std::shared_ptr<arrow::DataType> key_type = arrow::utf8();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(key_type, pool));
auto serialize = [&](const char* value, size_t size) {
Literal literal(FieldType::STRING, value, size);
- EXPECT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result,
- KeySerializer::SerializeKey(literal, key_type,
pool.get()));
+ EXPECT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result,
key_serializer->Serialize(literal));
return result;
};
- auto empty_meta =
- std::make_shared<BTreeIndexMeta>(serialize("", 0),
serialize("www.example.com", 15), false);
- auto normal_meta =
- std::make_shared<BTreeIndexMeta>(serialize("aaa.com", 7),
serialize("zzz.com", 7), false);
- auto null_meta = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, true);
+ auto empty_meta = std::make_shared<SortedIndexFileMeta>(
+ serialize("", 0), serialize("www.example.com", 15), false);
+ auto normal_meta =
std::make_shared<SortedIndexFileMeta>(serialize("aaa.com", 7),
+
serialize("zzz.com", 7), false);
+ auto null_meta = std::make_shared<SortedIndexFileMeta>(nullptr, nullptr,
true);
std::vector<GlobalIndexIOMeta> files = {
GlobalIndexIOMeta("file_empty", 1, empty_meta->Serialize(pool.get())),
GlobalIndexIOMeta("file_normal", 1,
normal_meta->Serialize(pool.get())),
GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())),
};
- ASSERT_OK_AND_ASSIGN(std::unique_ptr<BTreeFileMetaSelector> selector,
- BTreeFileMetaSelector::Create(files, key_type, pool));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<SortedFileMetaSelector> selector,
+ SortedFileMetaSelector::Create(files,
key_serializer));
ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> result,
selector->VisitEqual(Literal(FieldType::STRING,
"www.example.com", 15)));
@@ -282,38 +289,41 @@ TEST_F(BTreeFileMetaSelectorTest,
TestEmptyStringKeyDoesNotCrash) {
CheckResult(result, {"file_empty", "file_normal"});
}
-TEST_F(BTreeFileMetaSelectorTest, RejectsMalformedFileMetadata) {
+TEST_F(SortedFileMetaSelectorTest, RejectsMalformedFileMetadata) {
std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(arrow::int32(), pool));
std::vector<GlobalIndexIOMeta> files = {GlobalIndexIOMeta("missing", 1,
/*metadata=*/nullptr)};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
auto truncated = std::make_shared<Bytes>(std::string(4, '\0'), pool.get());
files = {GlobalIndexIOMeta("truncated", 1, truncated)};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
auto short_key = std::make_shared<Bytes>(std::string(1, '\0'), pool.get());
- auto invalid_key_meta = std::make_shared<BTreeIndexMeta>(short_key,
short_key, false);
+ auto invalid_key_meta = std::make_shared<SortedIndexFileMeta>(short_key,
short_key, false);
files = {GlobalIndexIOMeta("invalid-key", 1,
invalid_key_meta->Serialize(pool.get()))};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
- auto reversed_meta = std::make_shared<BTreeIndexMeta>(SerializeInt(10),
SerializeInt(1), false);
+ auto reversed_meta =
+ std::make_shared<SortedIndexFileMeta>(SerializeInt(10),
SerializeInt(1), false);
files = {GlobalIndexIOMeta("reversed-range", 1,
reversed_meta->Serialize(pool.get()))};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
auto only_first_meta =
- std::make_shared<BTreeIndexMeta>(SerializeInt(1),
/*last_key=*/nullptr, false);
+ std::make_shared<SortedIndexFileMeta>(SerializeInt(1),
/*last_key=*/nullptr, false);
files = {GlobalIndexIOMeta("only-first-key", 1,
only_first_meta->Serialize(pool.get()))};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
auto only_last_meta =
- std::make_shared<BTreeIndexMeta>(/*first_key=*/nullptr,
SerializeInt(1), false);
+ std::make_shared<SortedIndexFileMeta>(/*first_key=*/nullptr,
SerializeInt(1), false);
files = {GlobalIndexIOMeta("only-last-key", 1,
only_last_meta->Serialize(pool.get()))};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
auto empty_nonnull_meta =
- std::make_shared<BTreeIndexMeta>(/*first_key=*/nullptr,
/*last_key=*/nullptr, false);
+ std::make_shared<SortedIndexFileMeta>(/*first_key=*/nullptr,
/*last_key=*/nullptr, false);
files = {GlobalIndexIOMeta("empty-nonnull", 1,
empty_nonnull_meta->Serialize(pool.get()))};
- ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool));
+ ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer));
}
} // namespace paimon::test
diff --git a/src/paimon/common/global_index/btree/btree_index_meta.cpp
b/src/paimon/common/global_index/sorted_index_file_meta.cpp
similarity index 72%
rename from src/paimon/common/global_index/btree/btree_index_meta.cpp
rename to src/paimon/common/global_index/sorted_index_file_meta.cpp
index 4b6f6ce0..a05e7b18 100644
--- a/src/paimon/common/global_index/btree/btree_index_meta.cpp
+++ b/src/paimon/common/global_index/sorted_index_file_meta.cpp
@@ -17,7 +17,7 @@
* under the License.
*/
-#include "paimon/common/global_index/btree/btree_index_meta.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include <cstddef>
#include <limits>
@@ -32,14 +32,14 @@ Result<std::shared_ptr<Bytes>> ReadKey(MemorySliceInput*
input, int32_t key_leng
int32_t required_remaining, const char*
key_name,
MemoryPool* pool) {
if (key_length < 0) {
- return Status::Invalid(
- fmt::format("BTree index metadata has a negative {} length {}.",
key_name, key_length));
+ return Status::Invalid(fmt::format(
+ "Sorted index file metadata has a negative {} length {}.",
key_name, key_length));
}
if (input->Available() < required_remaining ||
key_length > input->Available() - required_remaining) {
- return Status::Invalid(
- fmt::format("BTree index metadata {} length {} exceeds the
available payload bytes.",
- key_name, key_length));
+ return Status::Invalid(fmt::format(
+ "Sorted index file metadata {} length {} exceeds the available
payload bytes.",
+ key_name, key_length));
}
if (key_length == 0) {
return std::make_shared<Bytes>(0, pool);
@@ -49,24 +49,25 @@ Result<std::shared_ptr<Bytes>> ReadKey(MemorySliceInput*
input, int32_t key_leng
} // namespace
-Result<std::shared_ptr<BTreeIndexMeta>> BTreeIndexMeta::Deserialize(
+Result<std::shared_ptr<SortedIndexFileMeta>> SortedIndexFileMeta::Deserialize(
const std::shared_ptr<Bytes>& meta, paimon::MemoryPool* pool) {
if (meta == nullptr) {
- return Status::Invalid("Cannot deserialize BTree index metadata from a
null buffer.");
+ return Status::Invalid("Cannot deserialize Sorted index file metadata
from a null buffer.");
}
if (pool == nullptr) {
- return Status::Invalid("Cannot deserialize BTree index metadata with a
null memory pool.");
+ return Status::Invalid(
+ "Cannot deserialize Sorted index file metadata with a null memory
pool.");
}
// Legacy metadata contains two int32 lengths and one has-nulls byte.
constexpr size_t kMinimumMetadataSize = 2 * sizeof(int32_t) +
sizeof(int8_t);
if (meta->size() < kMinimumMetadataSize) {
return Status::Invalid(fmt::format(
- "BTree index metadata is truncated: expected at least {} bytes,
but found {}.",
+ "Sorted index file metadata is truncated: expected at least {}
bytes, but found {}.",
kMinimumMetadataSize, meta->size()));
}
if (meta->size() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
return Status::Invalid(
- fmt::format("BTree index metadata size {} exceeds the supported
maximum {}.",
+ fmt::format("Sorted index file metadata size {} exceeds the
supported maximum {}.",
meta->size(), std::numeric_limits<int32_t>::max()));
}
@@ -83,8 +84,8 @@ Result<std::shared_ptr<BTreeIndexMeta>>
BTreeIndexMeta::Deserialize(
ReadKey(&input, last_key_len,
kRequiredAfterLastKey, "last key", pool));
int8_t has_nulls_byte = input.ReadByte();
if (has_nulls_byte != 0 && has_nulls_byte != 1) {
- return Status::Invalid(
- fmt::format("BTree index metadata has invalid has-nulls value
{}.", has_nulls_byte));
+ return Status::Invalid(fmt::format(
+ "Sorted index file metadata has invalid has-nulls value {}.",
has_nulls_byte));
}
bool has_nulls = has_nulls_byte == 1;
@@ -92,23 +93,25 @@ Result<std::shared_ptr<BTreeIndexMeta>>
BTreeIndexMeta::Deserialize(
int8_t format_version = input.ReadByte();
if (format_version != kFormatVersionWithNullFlags) {
return Status::Invalid(
- fmt::format("Unsupported BTree index metadata version {}.",
format_version));
+ fmt::format("Unsupported Sorted index file metadata version
{}.", format_version));
}
int8_t null_key_flags = input.ReadByte();
constexpr int8_t kKnownNullFlags = kFirstKeyIsNull | kLastKeyIsNull;
if ((null_key_flags & ~kKnownNullFlags) != 0) {
- return Status::Invalid(
- fmt::format("BTree index metadata has invalid null-key flags
{}.", null_key_flags));
+ return Status::Invalid(fmt::format(
+ "Sorted index file metadata has invalid null-key flags {}.",
null_key_flags));
}
if ((null_key_flags & kFirstKeyIsNull) != 0) {
if (first_key_len != 0) {
- return Status::Invalid("BTree index metadata marks a non-empty
first key as null.");
+ return Status::Invalid(
+ "Sorted index file metadata marks a non-empty first key as
null.");
}
first_key.reset();
}
if ((null_key_flags & kLastKeyIsNull) != 0) {
if (last_key_len != 0) {
- return Status::Invalid("BTree index metadata marks a non-empty
last key as null.");
+ return Status::Invalid(
+ "Sorted index file metadata marks a non-empty last key as
null.");
}
last_key.reset();
}
@@ -118,16 +121,16 @@ Result<std::shared_ptr<BTreeIndexMeta>>
BTreeIndexMeta::Deserialize(
first_key.reset();
last_key.reset();
} else if (input.Available() != 0) {
- return Status::Invalid(fmt::format("BTree index metadata has {}
unexpected trailing bytes.",
- input.Available()));
+ return Status::Invalid(fmt::format(
+ "Sorted index file metadata has {} unexpected trailing bytes.",
input.Available()));
}
- return std::make_shared<BTreeIndexMeta>(first_key, last_key, has_nulls);
+ return std::make_shared<SortedIndexFileMeta>(first_key, last_key,
has_nulls);
}
-std::shared_ptr<Bytes> BTreeIndexMeta::Serialize(paimon::MemoryPool* pool)
const {
+std::shared_ptr<Bytes> SortedIndexFileMeta::Serialize(paimon::MemoryPool*
pool) const {
int32_t first_key_size = first_key_ ? first_key_->size() : 0;
int32_t last_key_size = last_key_ ? last_key_->size() : 0;
- int32_t total_size = Size();
+ int32_t total_size = MemorySize();
MemorySliceOutput output(total_size, pool);
int8_t null_key_flags = 0;
diff --git a/src/paimon/common/global_index/btree/btree_index_meta.h
b/src/paimon/common/global_index/sorted_index_file_meta.h
similarity index 82%
rename from src/paimon/common/global_index/btree/btree_index_meta.h
rename to src/paimon/common/global_index/sorted_index_file_meta.h
index eb8158f1..88a7cdb7 100644
--- a/src/paimon/common/global_index/btree/btree_index_meta.h
+++ b/src/paimon/common/global_index/sorted_index_file_meta.h
@@ -24,21 +24,22 @@
#include "paimon/common/memory/memory_slice_input.h"
#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
#include "paimon/result.h"
namespace paimon {
-/// Index metadata for each BTree index file.
+/// Manifest-level min/max metadata for one global index file.
///
/// Empty serialized keys are valid, so null boundary keys are encoded
separately with flags.
-class BTreeIndexMeta {
+class SortedIndexFileMeta {
public:
- static Result<std::shared_ptr<BTreeIndexMeta>> Deserialize(const
std::shared_ptr<Bytes>& meta,
-
paimon::MemoryPool* pool);
+ static Result<std::shared_ptr<SortedIndexFileMeta>> Deserialize(
+ const std::shared_ptr<Bytes>& meta, paimon::MemoryPool* pool);
std::shared_ptr<Bytes> Serialize(paimon::MemoryPool* pool) const;
public:
- BTreeIndexMeta(const std::shared_ptr<Bytes>& first_key, const
std::shared_ptr<Bytes>& last_key,
- bool has_nulls)
+ SortedIndexFileMeta(const std::shared_ptr<Bytes>& first_key,
+ const std::shared_ptr<Bytes>& last_key, bool has_nulls)
: first_key_(first_key), last_key_(last_key), has_nulls_(has_nulls) {}
const std::shared_ptr<Bytes>& FirstKey() const {
@@ -58,7 +59,7 @@ class BTreeIndexMeta {
}
private:
- int32_t Size() const {
+ int32_t MemorySize() const {
// 11 bytes => key lengths (8) + has_nulls (1) + format version (1) +
null flags (1).
return (first_key_ ? first_key_->size() : 0) + (last_key_ ?
last_key_->size() : 0) + 11;
}
diff --git a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp
b/src/paimon/common/global_index/sorted_index_file_meta_test.cpp
similarity index 67%
rename from src/paimon/common/global_index/btree/btree_index_meta_test.cpp
rename to src/paimon/common/global_index/sorted_index_file_meta_test.cpp
index b0d78fa3..6c62fe5b 100644
--- a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp
+++ b/src/paimon/common/global_index/sorted_index_file_meta_test.cpp
@@ -17,7 +17,7 @@
* under the License.
*/
-#include "paimon/common/global_index/btree/btree_index_meta.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include <cstddef>
#include <string>
@@ -29,7 +29,7 @@
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
-class BTreeIndexMetaTest : public ::testing::Test {
+class SortedIndexFileMetaTest : public ::testing::Test {
protected:
void SetUp() override {
pool_ = GetDefaultPool();
@@ -78,10 +78,10 @@ class BTreeIndexMetaTest : public ::testing::Test {
std::shared_ptr<MemoryPool> pool_;
};
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) {
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeNormalKeys) {
auto first_key = std::make_shared<Bytes>("first_key_data", pool_.get());
auto last_key = std::make_shared<Bytes>("last_key_data", pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(first_key, last_key, true);
+ auto meta = std::make_shared<SortedIndexFileMeta>(first_key, last_key,
true);
// Serialize
auto serialized = meta->Serialize(pool_.get());
@@ -89,8 +89,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) {
ASSERT_GT(serialized->size(), 0u);
// Deserialize
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized);
// Verify first_key
@@ -108,16 +108,16 @@ TEST_F(BTreeIndexMetaTest,
SerializeDeserializeNormalKeys) {
ASSERT_TRUE(deserialized->HasNulls());
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) {
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeEmptyFirstKey) {
auto empty_key = std::make_shared<Bytes>(0, pool_.get());
auto last_key = std::make_shared<Bytes>("last_key_data", pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(empty_key, last_key, false);
+ auto meta = std::make_shared<SortedIndexFileMeta>(empty_key, last_key,
false);
auto serialized = meta->Serialize(pool_.get());
ASSERT_EQ(serialized->size(), 11 + last_key->size());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized->FirstKey());
ASSERT_EQ(deserialized->FirstKey()->size(), 0);
ASSERT_TRUE(deserialized->LastKey());
@@ -127,15 +127,15 @@ TEST_F(BTreeIndexMetaTest,
SerializeDeserializeEmptyFirstKey) {
ASSERT_FALSE(deserialized->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls)
{
+TEST_F(SortedIndexFileMetaTest,
SerializeDeserializeEmptyFirstAndLastKeysWithNulls) {
auto empty_key = std::make_shared<Bytes>(0, pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(empty_key, empty_key, true);
+ auto meta = std::make_shared<SortedIndexFileMeta>(empty_key, empty_key,
true);
auto serialized = meta->Serialize(pool_.get());
ASSERT_EQ(serialized->size(), 11);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized->FirstKey());
ASSERT_EQ(deserialized->FirstKey()->size(), 0);
ASSERT_TRUE(deserialized->LastKey());
@@ -144,14 +144,14 @@ TEST_F(BTreeIndexMetaTest,
SerializeDeserializeEmptyFirstAndLastKeysWithNulls) {
ASSERT_FALSE(deserialized->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) {
- auto meta = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, true);
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeOnlyNulls) {
+ auto meta = std::make_shared<SortedIndexFileMeta>(nullptr, nullptr, true);
auto serialized = meta->Serialize(pool_.get());
ASSERT_EQ(serialized->size(), 11);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized);
ASSERT_FALSE(deserialized->FirstKey());
ASSERT_FALSE(deserialized->LastKey());
@@ -159,25 +159,25 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls)
{
ASSERT_TRUE(deserialized->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, DeserializeLegacyOnlyNulls) {
+TEST_F(SortedIndexFileMetaTest, DeserializeLegacyOnlyNulls) {
auto empty_key = std::make_shared<Bytes>(0, pool_.get());
auto serialized = LegacyMetaBytes(empty_key, empty_key, true);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_FALSE(deserialized->FirstKey());
ASSERT_FALSE(deserialized->LastKey());
ASSERT_TRUE(deserialized->HasNulls());
ASSERT_TRUE(deserialized->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) {
+TEST_F(SortedIndexFileMetaTest, DeserializeLegacyEmptyFirstKey) {
auto empty_key = std::make_shared<Bytes>(0, pool_.get());
auto last_key = std::make_shared<Bytes>("last_key_data", pool_.get());
auto serialized = LegacyMetaBytes(empty_key, last_key, false);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized->FirstKey());
ASSERT_EQ(deserialized->FirstKey()->size(), 0);
ASSERT_TRUE(deserialized->LastKey());
@@ -187,12 +187,12 @@ TEST_F(BTreeIndexMetaTest,
DeserializeLegacyEmptyFirstKey) {
ASSERT_FALSE(deserialized->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls)
{
+TEST_F(SortedIndexFileMetaTest,
DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) {
auto empty_key = std::make_shared<Bytes>(0, pool_.get());
auto serialized = LegacyMetaBytes(empty_key, empty_key, false);
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized->FirstKey());
ASSERT_EQ(deserialized->FirstKey()->size(), 0);
ASSERT_TRUE(deserialized->LastKey());
@@ -201,109 +201,109 @@ TEST_F(BTreeIndexMetaTest,
DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) {
ASSERT_FALSE(deserialized->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, DeserializeRejectsNullAndTruncatedMetadata) {
- ASSERT_NOK(BTreeIndexMeta::Deserialize(nullptr, pool_.get()));
+TEST_F(SortedIndexFileMetaTest, DeserializeRejectsNullAndTruncatedMetadata) {
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(nullptr, pool_.get()));
for (size_t size = 0; size < 9; size++) {
auto truncated = std::make_shared<Bytes>(std::string(size, '\0'),
pool_.get());
- ASSERT_NOK(BTreeIndexMeta::Deserialize(truncated, pool_.get()));
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(truncated, pool_.get()));
}
}
-TEST_F(BTreeIndexMetaTest, DeserializeRejectsInvalidKeyLengths) {
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+TEST_F(SortedIndexFileMetaTest, DeserializeRejectsInvalidKeyLengths) {
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(/*first_key_length=*/-1, "", /*last_key_length=*/0, "",
/*has_nulls=*/0),
pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(/*first_key_length=*/100, "", /*last_key_length=*/0, "",
/*has_nulls=*/0),
pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/-1, "",
/*has_nulls=*/0),
pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/100, "",
/*has_nulls=*/0),
pool_.get()));
}
-TEST_F(BTreeIndexMetaTest, DeserializeRejectsMalformedFlagsAndTrailingBytes) {
- ASSERT_NOK(
- BTreeIndexMeta::Deserialize(MetadataBytes(0, "", 0, "",
/*has_nulls=*/2), pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+TEST_F(SortedIndexFileMetaTest,
DeserializeRejectsMalformedFlagsAndTrailingBytes) {
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(MetadataBytes(0, "", 0, "",
/*has_nulls=*/2),
+ pool_.get()));
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(0, "", 0, "", /*has_nulls=*/0,
{/*truncated_version=*/1}), pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(0, "", 0, "", /*has_nulls=*/0,
{/*unsupported_version=*/2, /*flags=*/0}),
pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1,
/*unknown_flags=*/4}),
pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1,
/*flags=*/0, /*trailing=*/0}),
pool_.get()));
- ASSERT_NOK(BTreeIndexMeta::Deserialize(
+ ASSERT_NOK(SortedIndexFileMeta::Deserialize(
MetadataBytes(1, "x", 0, "", /*has_nulls=*/0, {/*version=*/1,
/*first_key_is_null=*/1}),
pool_.get()));
}
-TEST_F(BTreeIndexMetaTest, HasNullsAndOnlyNulls) {
+TEST_F(SortedIndexFileMetaTest, HasNullsAndOnlyNulls) {
// Case 1: Has nulls with keys
auto meta1 =
- std::make_shared<BTreeIndexMeta>(std::make_shared<Bytes>("key",
pool_.get()),
- std::make_shared<Bytes>("key",
pool_.get()), true);
+ std::make_shared<SortedIndexFileMeta>(std::make_shared<Bytes>("key",
pool_.get()),
+ std::make_shared<Bytes>("key",
pool_.get()), true);
ASSERT_TRUE(meta1->HasNulls());
ASSERT_FALSE(meta1->OnlyNulls());
// Case 2: No nulls with keys
auto meta2 =
- std::make_shared<BTreeIndexMeta>(std::make_shared<Bytes>("key",
pool_.get()),
- std::make_shared<Bytes>("key",
pool_.get()), false);
+ std::make_shared<SortedIndexFileMeta>(std::make_shared<Bytes>("key",
pool_.get()),
+ std::make_shared<Bytes>("key",
pool_.get()), false);
ASSERT_FALSE(meta2->HasNulls());
ASSERT_FALSE(meta2->OnlyNulls());
// Case 3: Only nulls (no keys)
- auto meta3 = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, true);
+ auto meta3 = std::make_shared<SortedIndexFileMeta>(nullptr, nullptr, true);
ASSERT_TRUE(meta3->HasNulls());
ASSERT_TRUE(meta3->OnlyNulls());
// Case 4: No nulls and no keys (edge case)
- auto meta4 = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, false);
+ auto meta4 = std::make_shared<SortedIndexFileMeta>(nullptr, nullptr,
false);
ASSERT_FALSE(meta4->HasNulls());
ASSERT_TRUE(meta4->OnlyNulls());
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeNoNulls) {
- // Create a BTreeIndexMeta without nulls
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeNoNulls) {
+ // Create a SortedIndexFileMeta without nulls
auto first_key = std::make_shared<Bytes>("abc", pool_.get());
auto last_key = std::make_shared<Bytes>("xyz", pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(first_key, last_key, false);
+ auto meta = std::make_shared<SortedIndexFileMeta>(first_key, last_key,
false);
// Serialize
auto serialized = meta->Serialize(pool_.get());
ASSERT_TRUE(serialized);
// Deserialize
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized);
// Verify has_nulls is false
ASSERT_FALSE(deserialized->HasNulls());
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyFirstKey) {
- // Create a BTreeIndexMeta with only first_key (edge case)
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeWithOnlyFirstKey) {
+ // Create a SortedIndexFileMeta with only first_key (edge case)
auto first_key = std::make_shared<Bytes>("first", pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(first_key, nullptr, false);
+ auto meta = std::make_shared<SortedIndexFileMeta>(first_key, nullptr,
false);
// Serialize
auto serialized = meta->Serialize(pool_.get());
ASSERT_TRUE(serialized);
// Deserialize
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized);
// Verify first_key
@@ -315,18 +315,18 @@ TEST_F(BTreeIndexMetaTest,
SerializeDeserializeWithOnlyFirstKey) {
ASSERT_FALSE(deserialized->LastKey());
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyLastKey) {
- // Create a BTreeIndexMeta with only last_key (edge case)
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeWithOnlyLastKey) {
+ // Create a SortedIndexFileMeta with only last_key (edge case)
auto last_key = std::make_shared<Bytes>("last", pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(nullptr, last_key, false);
+ auto meta = std::make_shared<SortedIndexFileMeta>(nullptr, last_key,
false);
// Serialize
auto serialized = meta->Serialize(pool_.get());
ASSERT_TRUE(serialized);
// Deserialize
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized);
// Verify first_key is null
@@ -338,21 +338,21 @@ TEST_F(BTreeIndexMetaTest,
SerializeDeserializeWithOnlyLastKey) {
ASSERT_EQ(std::string(deserialized_last->data(),
deserialized_last->size()), "last");
}
-TEST_F(BTreeIndexMetaTest, SerializeDeserializeBinaryKeys) {
- // Create a BTreeIndexMeta with binary keys containing null bytes
+TEST_F(SortedIndexFileMetaTest, SerializeDeserializeBinaryKeys) {
+ // Create a SortedIndexFileMeta with binary keys containing null bytes
std::string binary_first = std::string("key\0with\0nulls", 14);
std::string binary_last = std::string("last\0key", 8);
auto first_key = std::make_shared<Bytes>(binary_first, pool_.get());
auto last_key = std::make_shared<Bytes>(binary_last, pool_.get());
- auto meta = std::make_shared<BTreeIndexMeta>(first_key, last_key, true);
+ auto meta = std::make_shared<SortedIndexFileMeta>(first_key, last_key,
true);
// Serialize
auto serialized = meta->Serialize(pool_.get());
ASSERT_TRUE(serialized);
// Deserialize
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<BTreeIndexMeta> deserialized,
- BTreeIndexMeta::Deserialize(serialized, pool_.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<SortedIndexFileMeta> deserialized,
+ SortedIndexFileMeta::Deserialize(serialized,
pool_.get()));
ASSERT_TRUE(deserialized);
// Verify first_key
diff --git
a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
index 7b14a5c4..e4231cf5 100644
--- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
+++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp
@@ -29,8 +29,8 @@
#include "arrow/api.h"
#include "fmt/format.h"
#include "gtest/gtest.h"
-#include "paimon/common/global_index/btree/btree_index_meta.h"
-#include "paimon/common/global_index/btree/key_serializer.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/global_index/sorted_index_file_meta.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/core/global_index/indexed_split_impl.h"
#include "paimon/core/index/pk/primary_key_index_definitions.h"
@@ -735,19 +735,21 @@ TEST_F(PrimaryKeySortedIndexScanTest,
InvalidRowRangePayloadFallsBack) {
TEST_F(PrimaryKeySortedIndexScanTest, MalformedBTreeMetadataFallsBack) {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> payload,
BuildPayload());
const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<KeySerializer> key_serializer,
+ KeySerializer::Create(arrow::int64(), pool_));
auto short_key = std::make_shared<Bytes>(std::string(1, '\0'),
pool_.get());
- auto invalid_key_meta = std::make_shared<BTreeIndexMeta>(short_key,
short_key, false);
+ auto invalid_key_meta = std::make_shared<SortedIndexFileMeta>(short_key,
short_key, false);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> first_key,
-
KeySerializer::SerializeKey(Literal(static_cast<int64_t>(10)),
- arrow::int64(),
pool_.get()));
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<Bytes> last_key,
- KeySerializer::SerializeKey(Literal(static_cast<int64_t>(1)),
arrow::int64(), pool_.get()));
- auto reversed_meta = std::make_shared<BTreeIndexMeta>(first_key, last_key,
false);
- auto only_first_meta = std::make_shared<BTreeIndexMeta>(first_key,
/*last_key=*/nullptr, false);
- auto only_last_meta =
std::make_shared<BTreeIndexMeta>(/*first_key=*/nullptr, last_key, false);
+
key_serializer->Serialize(Literal(static_cast<int64_t>(10))));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> last_key,
+
key_serializer->Serialize(Literal(static_cast<int64_t>(1))));
+ auto reversed_meta = std::make_shared<SortedIndexFileMeta>(first_key,
last_key, false);
+ auto only_first_meta =
+ std::make_shared<SortedIndexFileMeta>(first_key, /*last_key=*/nullptr,
false);
+ auto only_last_meta =
+ std::make_shared<SortedIndexFileMeta>(/*first_key=*/nullptr, last_key,
false);
auto empty_nonnull_meta =
- std::make_shared<BTreeIndexMeta>(/*first_key=*/nullptr,
/*last_key=*/nullptr, false);
+ std::make_shared<SortedIndexFileMeta>(/*first_key=*/nullptr,
/*last_key=*/nullptr, false);
std::vector<std::shared_ptr<Bytes>> malformed_metadata = {
nullptr,
std::make_shared<Bytes>(std::string(4, '\0'), pool_.get()),
