This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b494574ce3a [opt](store) allow to set storage_page_size for tables
(#42680)
b494574ce3a is described below
commit b494574ce3a295121087b98455744d8be9eab112
Author: zzzxl <[email protected]>
AuthorDate: Thu Nov 7 14:04:40 2024 +0800
[opt](store) allow to set storage_page_size for tables (#42680)
storage_page_size can only be set between 64k and 1M.
---
be/src/cloud/pb_convert.cpp | 4 +
be/src/olap/compaction.cpp | 2 +
be/src/olap/delta_writer_v2.cpp | 3 +
be/src/olap/rowset/rowset_writer_context.h | 1 +
be/src/olap/rowset/segment_v2/options.h | 1 +
be/src/olap/rowset/segment_v2/segment_writer.cpp | 29 ++++
.../rowset/segment_v2/vertical_segment_writer.cpp | 29 ++++
be/src/olap/rowset_builder.cpp | 1 +
be/src/olap/tablet_meta.cpp | 13 +-
be/src/olap/tablet_meta.h | 10 +-
.../org/apache/doris/alter/CloudRollupJobV2.java | 3 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../analysis/ModifyTablePropertiesClause.java | 2 +
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 6 +
.../java/org/apache/doris/catalog/OlapTable.java | 14 ++
.../org/apache/doris/catalog/TableProperty.java | 14 ++
.../cloud/datasource/CloudInternalCatalog.java | 7 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 21 +++
.../apache/doris/datasource/InternalCatalog.java | 16 +-
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../org/apache/doris/task/CreateReplicaTask.java | 6 +-
.../apache/doris/common/PropertyAnalyzerTest.java | 48 ++++++
.../java/org/apache/doris/task/AgentTaskTest.java | 4 +-
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 1 +
.../data/query_p0/system/test_table_properties.out | 8 +-
.../test_storage_page_size_fault.groovy | 71 ++++++++
.../query_p0/system/test_storage_page_size.groovy | 190 +++++++++++++++++++++
31 files changed, 504 insertions(+), 17 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 1f780824e32..b6b8d3934ee 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -503,6 +503,7 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out,
const TabletMetaPB& in)
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ out->set_storage_page_size(in.storage_page_size());
}
void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
@@ -569,6 +570,7 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out,
TabletMetaPB&& in) {
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ out->set_storage_page_size(in.storage_page_size());
}
TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
@@ -644,6 +646,7 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const
TabletMetaCloudPB& in)
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ out->set_storage_page_size(in.storage_page_size());
}
void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
@@ -710,6 +713,7 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out,
TabletMetaCloudPB&& in) {
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ out->set_storage_page_size(in.storage_page_size());
}
} // namespace doris::cloud
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index a7418f5bbdd..85e0a94c874 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -821,6 +821,7 @@ Status
CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx)
ctx.tablet_schema = _cur_tablet_schema;
ctx.newest_write_timestamp = _newest_write_timestamp;
ctx.write_type = DataWriteType::TYPE_COMPACTION;
+ ctx.storage_page_size = _tablet->tablet_meta()->storage_page_size();
_output_rs_writer = DORIS_TRY(_tablet->create_rowset_writer(ctx,
_is_vertical));
_pending_rs_guard = _engine.add_pending_rowset(ctx);
return Status::OK();
@@ -1123,6 +1124,7 @@ Status
CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext&
ctx.tablet_schema = _cur_tablet_schema;
ctx.newest_write_timestamp = _newest_write_timestamp;
ctx.write_type = DataWriteType::TYPE_COMPACTION;
+ ctx.storage_page_size = _tablet->tablet_meta()->storage_page_size();
auto compaction_policy = _tablet->tablet_meta()->compaction_policy();
if (_tablet->tablet_meta()->time_series_compaction_level_threshold() >= 2)
{
diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index a6fb0154489..b770d3c0790 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -125,6 +125,9 @@ Status DeltaWriterV2::init() {
context.partial_update_info = _partial_update_info;
context.memtable_on_sink_support_index_v2 = true;
+ auto tablet =
DORIS_TRY(ExecEnv::GetInstance()->storage_engine().get_tablet(_req.tablet_id));
+ context.storage_page_size = tablet->tablet_meta()->storage_page_size();
+
_rowset_writer = std::make_shared<BetaRowsetWriterV2>(_streams);
RETURN_IF_ERROR(_rowset_writer->init(context));
std::shared_ptr<WorkloadGroup> wg_sptr = nullptr;
diff --git a/be/src/olap/rowset/rowset_writer_context.h
b/be/src/olap/rowset/rowset_writer_context.h
index cb0fda83e60..df85c11cd35 100644
--- a/be/src/olap/rowset/rowset_writer_context.h
+++ b/be/src/olap/rowset/rowset_writer_context.h
@@ -108,6 +108,7 @@ struct RowsetWriterContext {
std::shared_ptr<std::mutex> schema_lock;
int64_t compaction_level = 0;
+ int64_t storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE;
// For local rowset
std::string tablet_path;
diff --git a/be/src/olap/rowset/segment_v2/options.h
b/be/src/olap/rowset/segment_v2/options.h
index 93ec03df452..33d1a24ece3 100644
--- a/be/src/olap/rowset/segment_v2/options.h
+++ b/be/src/olap/rowset/segment_v2/options.h
@@ -25,6 +25,7 @@ namespace segment_v2 {
static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M
constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store
page size: 16KB
+static constexpr size_t STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536;
struct PageBuilderOptions {
size_t data_page_size = DEFAULT_PAGE_SIZE;
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 648379d35c4..db95d3b62e7 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -245,6 +245,35 @@ Status SegmentWriter::_create_column_writer(uint32_t cid,
const TabletColumn& co
#undef CHECK_FIELD_TYPE
+ if (_opts.rowset_ctx != nullptr) {
+ int64_t storage_page_size = _opts.rowset_ctx->storage_page_size;
+ // storage_page_size must be between 4KB and 10MB.
+ if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
+ opts.data_page_size = storage_page_size;
+ }
+ }
+
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
+ auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
+ INT_MIN);
+ auto target_data_page_size =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size",
+ "storage_page_size", INT_MIN);
+ if (table_id == INT_MIN || target_data_page_size == INT_MIN) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Debug point parameters missing: either 'table_id' or
'storage_page_size' not "
+ "set.");
+ }
+ if (table_id == _tablet_schema->table_id() &&
+ opts.data_page_size != target_data_page_size) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Mismatch in 'storage_page_size': expected size does not
match the current "
+ "data page size. "
+ "Expected: " +
+ std::to_string(target_data_page_size) +
+ ", Actual: " + std::to_string(opts.data_page_size) + ".");
+ }
+ })
if (column.is_row_store_column()) {
// smaller page size for row store column
auto page_size = _tablet_schema->row_store_page_size();
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 2243da78556..0ac9f349769 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -238,6 +238,35 @@ Status
VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
#undef CHECK_FIELD_TYPE
+ if (_opts.rowset_ctx != nullptr) {
+ int64_t storage_page_size = _opts.rowset_ctx->storage_page_size;
+ // storage_page_size must be between 4KB and 10MB.
+ if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
+ opts.data_page_size = storage_page_size;
+ }
+ }
+
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
+ auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
+ INT_MIN);
+ auto target_data_page_size =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+
"VerticalSegmentWriter._create_column_writer.storage_page_size",
+ "storage_page_size", INT_MIN);
+ if (table_id == INT_MIN || target_data_page_size == INT_MIN) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Debug point parameters missing: either 'table_id' or
'storage_page_size' not "
+ "set.");
+ }
+ if (table_id == _tablet_schema->table_id() &&
+ opts.data_page_size != target_data_page_size) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "Mismatch in 'storage_page_size': expected size does not
match the current "
+ "data page size. "
+ "Expected: " +
+ std::to_string(target_data_page_size) +
+ ", Actual: " + std::to_string(opts.data_page_size) + ".");
+ }
+ })
if (column.is_row_store_column()) {
// smaller page size for row store column
auto page_size = _tablet_schema->row_store_page_size();
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 112986d31d9..93b10512461 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -232,6 +232,7 @@ Status RowsetBuilder::init() {
context.mow_context = mow_context;
context.write_file_cache = _req.write_file_cache;
context.partial_update_info = _partial_update_info;
+ context.storage_page_size = _tablet->tablet_meta()->storage_page_size();
_rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false));
_pending_rs_guard = _engine.pending_local_rowsets().add(context.rowset_id);
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 91f3b7dd816..d153069babd 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -97,7 +97,8 @@ TabletMetaSharedPtr TabletMeta::create(
request.time_series_compaction_file_count_threshold,
request.time_series_compaction_time_threshold_seconds,
request.time_series_compaction_empty_rowsets_threshold,
- request.time_series_compaction_level_threshold,
inverted_index_file_storage_format);
+ request.time_series_compaction_level_threshold,
inverted_index_file_storage_format,
+ request.storage_page_size);
}
TabletMeta::TabletMeta()
@@ -118,7 +119,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
int64_t time_series_compaction_time_threshold_seconds,
int64_t time_series_compaction_empty_rowsets_threshold,
int64_t time_series_compaction_level_threshold,
- TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format)
+ TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format,
+ int64_t storage_page_size)
: _tablet_uid(0, 0),
_schema(new TabletSchema),
_delete_bitmap(new DeleteBitmap(tablet_id)) {
@@ -150,6 +152,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
time_series_compaction_empty_rowsets_threshold);
tablet_meta_pb.set_time_series_compaction_level_threshold(
time_series_compaction_level_threshold);
+ tablet_meta_pb.set_storage_page_size(storage_page_size);
TabletSchemaPB* schema = tablet_meta_pb.mutable_schema();
schema->set_num_short_key_columns(tablet_schema.short_key_column_count);
schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block);
@@ -377,7 +380,8 @@ TabletMeta::TabletMeta(const TabletMeta& b)
b._time_series_compaction_time_threshold_seconds),
_time_series_compaction_empty_rowsets_threshold(
b._time_series_compaction_empty_rowsets_threshold),
-
_time_series_compaction_level_threshold(b._time_series_compaction_level_threshold)
{};
+
_time_series_compaction_level_threshold(b._time_series_compaction_level_threshold),
+ _storage_page_size(b._storage_page_size) {};
void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn&
tcolumn,
ColumnPB* column) {
@@ -685,6 +689,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
tablet_meta_pb.time_series_compaction_empty_rowsets_threshold();
_time_series_compaction_level_threshold =
tablet_meta_pb.time_series_compaction_level_threshold();
+ _storage_page_size = tablet_meta_pb.storage_page_size();
}
void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
@@ -776,6 +781,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
time_series_compaction_empty_rowsets_threshold());
tablet_meta_pb->set_time_series_compaction_level_threshold(
time_series_compaction_level_threshold());
+ tablet_meta_pb->set_storage_page_size(storage_page_size());
}
int64_t TabletMeta::mem_size() const {
@@ -983,6 +989,7 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) {
return false;
if (a._time_series_compaction_level_threshold !=
b._time_series_compaction_level_threshold)
return false;
+ if (a._storage_page_size != b._storage_page_size) return false;
return true;
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index d56e529e42b..0d9645e0190 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -114,7 +114,8 @@ public:
int64_t time_series_compaction_empty_rowsets_threshold = 5,
int64_t time_series_compaction_level_threshold = 1,
TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format =
- TInvertedIndexFileStorageFormat::V2);
+ TInvertedIndexFileStorageFormat::V2,
+ int64_t storage_page_size = 65536);
// If need add a filed in TableMeta, filed init copy in copy construct
function
TabletMeta(const TabletMeta& tablet_meta);
TabletMeta(TabletMeta&& tablet_meta) = delete;
@@ -293,6 +294,11 @@ public:
int64_t avg_rs_meta_serialize_size() const { return
_avg_rs_meta_serialize_size; }
+ void set_storage_page_size(int64_t storage_page_size) {
+ _storage_page_size = storage_page_size;
+ }
+ int64_t storage_page_size() const { return _storage_page_size; }
+
private:
Status _save_meta(DataDir* data_dir);
@@ -353,6 +359,8 @@ private:
// cloud
int64_t _ttl_seconds = 0;
+ int64_t _storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+
mutable std::shared_mutex _meta_lock;
};
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index b38921f4fc9..37638319e3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -232,7 +232,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
tbl.getEnableMowLightDelete(), null,
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(), null);
+ tbl.variantEnableFlattenNested(), null,
+ tbl.storagePageSize());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 07b8323951d..92a74e46b92 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -277,7 +277,8 @@ public class CloudSchemaChangeJobV2 extends
SchemaChangeJobV2 {
tbl.getEnableMowLightDelete(),
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(),
clusterKeyIndexes);
+ tbl.variantEnableFlattenNested(),
clusterKeyIndexes,
+ tbl.storagePageSize());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 49838446cd5..8eb87050bf6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -270,7 +270,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
tbl.getRowStoreColumnsUniqueIds(tbl.getTableProperty().getCopiedRowStoreColumns()),
objectPool,
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested());
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index c8754aa5d62..522433dfe89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -330,7 +330,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool,
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested());
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index 71acd3b217b..1fd7480c06c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -366,6 +366,8 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN))
{
// do nothing, will be analyzed when creating alter job
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
+ throw new AnalysisException("You can not modify
storage_page_size");
} else {
throw new AnalysisException("Unknown table property: " +
properties.keySet());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 8a627299a82..e4537489177 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1353,7 +1353,8 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool,
localTbl.rowStorePageSize(),
- localTbl.variantEnableFlattenNested());
+ localTbl.variantEnableFlattenNested(),
+ localTbl.storagePageSize());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
if (baseTabletRef != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index cd0c0e80d8f..822c1954540 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3604,6 +3604,12 @@ public class Env {
sb.append(olapTable.rowStorePageSize()).append("\"");
}
+ // storage page size
+ if (olapTable.storagePageSize() !=
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE) {
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE).append("\"
= \"");
+ sb.append(olapTable.storagePageSize()).append("\"");
+ }
+
// skip inverted index on load
if (olapTable.skipWriteIndexOnLoad()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).append("\"
= \"");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 20033f9330e..28a9dfce15e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2763,6 +2763,20 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
}
+ public void setStoragePageSize(long storagePageSize) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
+ Long.valueOf(storagePageSize).toString());
+ tableProperty.buildStoragePageSize();
+ }
+
+ public long storagePageSize() {
+ if (tableProperty != null) {
+ return tableProperty.storagePageSize();
+ }
+ return PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ }
+
public void setStorageFormat(TStorageFormat storageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT,
storageFormat.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 8517c4fa58a..1ac556c6846 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -107,6 +107,8 @@ public class TableProperty implements Writable,
GsonPostProcessable {
private long rowStorePageSize =
PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+ private long storagePageSize =
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+
private String compactionPolicy =
PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY;
private long timeSeriesCompactionGoalSizeMbytes
@@ -324,6 +326,17 @@ public class TableProperty implements Writable,
GsonPostProcessable {
return rowStorePageSize;
}
+ public TableProperty buildStoragePageSize() {
+ storagePageSize = Long.parseLong(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
+
Long.toString(PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE)));
+ return this;
+ }
+
+ public long storagePageSize() {
+ return storagePageSize;
+ }
+
public TableProperty buildSkipWriteIndexOnLoad() {
skipWriteIndexOnLoad = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
"false"));
@@ -724,6 +737,7 @@ public class TableProperty implements Writable,
GsonPostProcessable {
buildStoreRowColumn();
buildRowStoreColumns();
buildRowStorePageSize();
+ buildStoragePageSize();
buildSkipWriteIndexOnLoad();
buildCompactionPolicy();
buildTimeSeriesCompactionGoalSizeMbytes();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index fab1ce58775..b5197a60786 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -180,7 +180,8 @@ public class CloudInternalCatalog extends InternalCatalog {
tbl.getEnableMowLightDelete(),
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(), clusterKeyIndexes);
+ tbl.variantEnableFlattenNested(), clusterKeyIndexes,
+ tbl.storagePageSize());
requestBuilder.addTabletMetas(builder);
}
if (!storageVaultIdSet && ((CloudEnv)
Env.getCurrentEnv()).getEnableStorageVault()) {
@@ -230,7 +231,8 @@ public class CloudInternalCatalog extends InternalCatalog {
Long timeSeriesCompactionLevelThreshold, boolean
disableAutoCompaction,
List<Integer> rowStoreColumnUniqueIds, boolean
enableMowLightDelete,
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat,
long pageSize,
- boolean variantEnableFlattenNested, List<Integer> clusterKeyIdxes)
throws DdlException {
+ boolean variantEnableFlattenNested, List<Integer> clusterKeyIdxes,
+ long storagePageSize) throws DdlException {
OlapFile.TabletMetaCloudPB.Builder builder =
OlapFile.TabletMetaCloudPB.newBuilder();
builder.setTableId(tableId);
builder.setIndexId(indexId);
@@ -265,6 +267,7 @@ public class CloudInternalCatalog extends InternalCatalog {
builder.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
builder.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
builder.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold);
+ builder.setStoragePageSize(storagePageSize);
OlapFile.TabletSchemaCloudPB.Builder schemaBuilder =
OlapFile.TabletSchemaCloudPB.newBuilder();
schemaBuilder.setSchemaVersion(schemaVersion);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index b4384f8a7fa..5721db0c27e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -99,6 +99,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_ROW_STORE_PAGE_SIZE =
"row_store_page_size";
public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384L;
+ public static final String PROPERTIES_STORAGE_PAGE_SIZE =
"storage_page_size";
+ public static final long STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536L;
+
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE =
"distribution_type";
@@ -1072,6 +1075,24 @@ public class PropertyAnalyzer {
return rowStorePageSize;
}
+ public static long analyzeStoragePageSize(Map<String, String> properties)
throws AnalysisException {
+ long storagePageSize = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ if (properties != null &&
properties.containsKey(PROPERTIES_STORAGE_PAGE_SIZE)) {
+ String storagePageSizeStr =
properties.get(PROPERTIES_STORAGE_PAGE_SIZE);
+ try {
+ storagePageSize = Long.parseLong(storagePageSizeStr);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid storage page size: " +
storagePageSizeStr);
+ }
+ if (storagePageSize < 4096 || storagePageSize > 10485760) {
+ throw new AnalysisException("Storage page size must be between
4KB and 10MB.");
+ }
+ storagePageSize = alignTo4K(storagePageSize);
+ properties.remove(PROPERTIES_STORAGE_PAGE_SIZE);
+ }
+ return storagePageSize;
+ }
+
// analyzeStorageFormat will parse the storage format from properties
// sql: alter table tablet_name set ("storage_format" = "v2")
// Use this sql to convert all tablets(base and rollup index) to a new
format segment
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index f8183028c6a..0ab2460f7ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1641,6 +1641,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
properties.put(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE,
Long.toString(olapTable.rowStorePageSize()));
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
+ Long.toString(olapTable.storagePageSize()));
+ }
if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD))
{
properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
olapTable.skipWriteIndexOnLoad().toString());
@@ -2168,7 +2172,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.storeRowColumn(), binlogConfig,
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool, tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested());
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
@@ -2655,9 +2660,16 @@ public class InternalCatalog implements
CatalogIf<Database> {
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
-
olapTable.setRowStorePageSize(rowStorePageSize);
+ long storagePageSize =
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ try {
+ storagePageSize =
PropertyAnalyzer.analyzeStoragePageSize(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ olapTable.setStoragePageSize(storagePageSize);
+
// check data sort properties
int keyColumnSize =
CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames()) ?
keysDesc.keysColumnSize() :
keysDesc.getClusterKeysColumnNames().size();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index bc51590ccb5..c5c72eae3c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -974,7 +974,8 @@ public class ReportHandler extends Daemon {
olapTable.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool,
olapTable.rowStorePageSize(),
-
olapTable.variantEnableFlattenNested());
+
olapTable.variantEnableFlattenNested(),
+ olapTable.storagePageSize());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
.getInvertedIndexFileStorageFormat());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index c410f37e5c9..4824994c3b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -66,6 +66,7 @@ public class CreateReplicaTask extends AgentTask {
private TStorageMedium storageMedium;
private TCompressionType compressionType;
private long rowStorePageSize;
+ private long storagePageSize;
private List<Column> columns;
@@ -156,7 +157,8 @@ public class CreateReplicaTask extends AgentTask {
List<Integer> rowStoreColumnUniqueIds,
Map<Object, Object> objectPool,
long rowStorePageSize,
- boolean variantEnableFlattenNested) {
+ boolean variantEnableFlattenNested,
+ long storagePageSize) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
this.replicaId = replicaId;
@@ -204,6 +206,7 @@ public class CreateReplicaTask extends AgentTask {
this.objectPool = objectPool;
this.rowStorePageSize = rowStorePageSize;
this.variantEnableFlattenNested = variantEnableFlattenNested;
+ this.storagePageSize = storagePageSize;
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -412,6 +415,7 @@ public class CreateReplicaTask extends AgentTask {
createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
createTabletReq.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
createTabletReq.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold);
+ createTabletReq.setStoragePageSize(storagePageSize);
if (binlogConfig != null) {
createTabletReq.setBinlogConfig(binlogConfig.toThrift());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index eb3500d13d9..041ca89bfc5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -188,4 +188,52 @@ public class PropertyAnalyzerTest {
Assert.assertEquals(1, tagMap.size());
Assert.assertEquals(Tag.DEFAULT_BACKEND_TAG.value,
tagMap.get(Tag.TYPE_LOCATION));
}
+
+ @Test
+ public void testStoragePageSize() throws AnalysisException {
+ Map<String, String> properties = Maps.newHashMap();
+
+ // Test default value
+ Assert.assertEquals(PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE,
+ PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test valid value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "8192");
// 8KB
+ Assert.assertEquals(8192,
PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test lower boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "4096");
// 4KB
+ Assert.assertEquals(4096,
PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test upper boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
"10485760"); // 10MB
+ Assert.assertEquals(10485760,
PropertyAnalyzer.analyzeStoragePageSize(properties));
+
+ // Test invalid number format
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
"invalid");
+ try {
+ PropertyAnalyzer.analyzeStoragePageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid storage page
size"));
+ }
+
+ // Test value below minimum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "1024");
// 1KB
+ try {
+ PropertyAnalyzer.analyzeStoragePageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage page size must
be between 4KB and 10MB"));
+ }
+
+ // Test value above maximum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
"20971520"); // 20MB
+ try {
+ PropertyAnalyzer.analyzeStoragePageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage page size must
be between 4KB and 10MB"));
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index 6f2996207d0..f6c6201a934 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -74,6 +74,7 @@ public class AgentTaskTest {
private TStorageType storageType = TStorageType.COLUMN;
private long rowStorePageSize = 16384L;
+ private long storagePageSize = 65536L;
private List<Column> columns;
private MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(3);
@@ -108,7 +109,8 @@ public class AgentTaskTest {
createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId,
partitionId,
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
- TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false);
+ TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false,
+ storagePageSize);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 30f74af7d67..7e65a4a9bf2 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -516,6 +516,7 @@ message TabletMetaPB {
optional int64 time_series_compaction_time_threshold_seconds = 31 [default
= 3600];
optional int64 time_series_compaction_empty_rowsets_threshold = 32
[default = 5];
optional int64 time_series_compaction_level_threshold = 33 [default = 1];
+ optional int64 storage_page_size = 34 [default=65536];
// For cloud
optional int64 index_id = 1000;
@@ -572,6 +573,7 @@ message TabletMetaCloudPB {
optional int64 group_commit_data_bytes = 36 [default = 134217728];
optional int64 time_series_compaction_empty_rowsets_threshold = 37
[default = 5];
optional int64 time_series_compaction_level_threshold = 38 [default = 1];
+ optional int64 storage_page_size = 39 [default=65536];
// Use for selectdb-cloud
optional string table_name = 101;
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index f02b8c0f886..9094c6eea2a 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -184,6 +184,7 @@ struct TCreateTabletReq {
27: optional i64 time_series_compaction_level_threshold = 1
28: optional TInvertedIndexStorageFormat inverted_index_storage_format =
TInvertedIndexStorageFormat.DEFAULT // Deprecated
29: optional Types.TInvertedIndexFileStorageFormat
inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2
+ 30: optional i64 storage_page_size = 65536
// For cloud
1000: optional bool is_in_memory = false
diff --git a/regression-test/data/query_p0/system/test_table_properties.out
b/regression-test/data/query_p0/system/test_table_properties.out
index 276ca4daeb5..812b7d2d4b7 100644
--- a/regression-test/data/query_p0/system/test_table_properties.out
+++ b/regression-test/data/query_p0/system/test_table_properties.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select_check_1 --
-102
+105
-- !select_check_2 --
internal test_table_properties_db duplicate_table _auto_bucket
false
@@ -29,6 +29,7 @@ internal test_table_properties_db duplicate_table
row_store_page_size 16384
internal test_table_properties_db duplicate_table
skip_write_index_on_load false
internal test_table_properties_db duplicate_table storage_format
V2
internal test_table_properties_db duplicate_table storage_medium
HDD
+internal test_table_properties_db duplicate_table
storage_page_size 65536
internal test_table_properties_db duplicate_table
store_row_column false
internal test_table_properties_db duplicate_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db duplicate_table
time_series_compaction_file_count_threshold 2000
@@ -62,6 +63,7 @@ internal test_table_properties_db listtable
row_store_page_size 16384
internal test_table_properties_db listtable
skip_write_index_on_load false
internal test_table_properties_db listtable storage_format
V2
internal test_table_properties_db listtable storage_medium
HDD
+internal test_table_properties_db listtable
storage_page_size 65536
internal test_table_properties_db listtable
store_row_column false
internal test_table_properties_db listtable
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db listtable
time_series_compaction_file_count_threshold 2000
@@ -95,6 +97,7 @@ internal test_table_properties_db unique_table
row_store_page_size 16384
internal test_table_properties_db unique_table
skip_write_index_on_load false
internal test_table_properties_db unique_table storage_format
V2
internal test_table_properties_db unique_table storage_medium
HDD
+internal test_table_properties_db unique_table
storage_page_size 65536
internal test_table_properties_db unique_table
store_row_column false
internal test_table_properties_db unique_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db unique_table
time_series_compaction_file_count_threshold 2000
@@ -130,6 +133,7 @@ internal test_table_properties_db duplicate_table
row_store_page_size 16384
internal test_table_properties_db duplicate_table
skip_write_index_on_load false
internal test_table_properties_db duplicate_table storage_format
V2
internal test_table_properties_db duplicate_table storage_medium
HDD
+internal test_table_properties_db duplicate_table
storage_page_size 65536
internal test_table_properties_db duplicate_table
store_row_column false
internal test_table_properties_db duplicate_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db duplicate_table
time_series_compaction_file_count_threshold 2000
@@ -163,6 +167,7 @@ internal test_table_properties_db unique_table
row_store_page_size 16384
internal test_table_properties_db unique_table
skip_write_index_on_load false
internal test_table_properties_db unique_table storage_format
V2
internal test_table_properties_db unique_table storage_medium
HDD
+internal test_table_properties_db unique_table
storage_page_size 65536
internal test_table_properties_db unique_table
store_row_column false
internal test_table_properties_db unique_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db unique_table
time_series_compaction_file_count_threshold 2000
@@ -200,6 +205,7 @@ internal test_table_properties_db duplicate_table
row_store_page_size 16384
internal test_table_properties_db duplicate_table
skip_write_index_on_load false
internal test_table_properties_db duplicate_table storage_format
V2
internal test_table_properties_db duplicate_table storage_medium
HDD
+internal test_table_properties_db duplicate_table
storage_page_size 65536
internal test_table_properties_db duplicate_table
store_row_column false
internal test_table_properties_db duplicate_table
time_series_compaction_empty_rowsets_threshold 5
internal test_table_properties_db duplicate_table
time_series_compaction_file_count_threshold 2000
diff --git
a/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy
b/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy
new file mode 100644
index 00000000000..76b666ecba7
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_storage_page_size_fault", "nonConcurrent") {
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ boolean disableAutoCompaction = false
+
+ def set_be_config = { key, value ->
+ for (String backend_id: backendId_to_backendIP.keySet()) {
+ def (code, out, err) =
update_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), key, value)
+ logger.info("update config: code=" + code + ", out=" + out + ",
err=" + err)
+ }
+ }
+
+ def dbName = "regression_test_fault_injection_p0"
+ def tableName = "test_storage_page_size_fault"
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `@timestamp` int(11) NULL COMMENT "",
+ `clientip` varchar(20) NULL COMMENT "",
+ `request` text NULL COMMENT "",
+ `status` int(11) NULL COMMENT "",
+ `size` int(11) NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`@timestamp`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true",
+ "storage_page_size" = "65537"
+ );
+ """
+
+ def tableId = getTableId(dbName, tableName)
+ if (tableId == null) {
+ throw new IllegalStateException("Table ID not found for table:
${tableName}")
+ }
+ logger.info("tableId: " + tableId)
+
+ try {
+
GetDebugPoint().enableDebugPointForAllBEs("VerticalSegmentWriter._create_column_writer.storage_page_size",
["table_id": tableId, "storage_page_size": 69632])
+ sql """ INSERT INTO ${tableName} VALUES (893964617, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+
+ set_be_config.call("enable_vertical_segment_writer", "false")
+ sql """ INSERT INTO ${tableName} VALUES (893964617, '40.135.0.0', 'GET
/images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+ set_be_config.call("enable_vertical_segment_writer", "true")
+
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("VerticalSegmentWriter._create_column_writer.storage_page_size")
+ }
+}
\ No newline at end of file
diff --git
a/regression-test/suites/query_p0/system/test_storage_page_size.groovy
b/regression-test/suites/query_p0/system/test_storage_page_size.groovy
new file mode 100644
index 00000000000..9d7b129226d
--- /dev/null
+++ b/regression-test/suites/query_p0/system/test_storage_page_size.groovy
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_storage_page_size") {
+ sql """ DROP TABLE IF EXISTS table_1; """
+ sql """
+ create table table_1 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1");
+ """
+ test {
+ sql "show create table table_1;"
+ check { result, exception, startTime, endTime ->
+ assertFalse(result[0][1].contains("\"storage_page_size\" =
\"65536\""))
+ }
+ }
+
+ // min - 1
+ sql """ DROP TABLE IF EXISTS table_2; """
+ test {
+ sql """
+ create table table_2 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "4095");
+ """
+ check { result, exception, startTime, endTime ->
+ if (exception) {
+ assert exception.message.contains("Storage page size must be
between 4KB and 10MB.")
+ }
+ }
+ }
+
+ // min
+ sql """ DROP TABLE IF EXISTS table_3; """
+ sql """
+ create table table_3 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "4096");
+ """
+ test {
+ sql "show create table table_3;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"4096\""))
+ }
+ }
+
+
+ // min + 1
+ sql """ DROP TABLE IF EXISTS table_4; """
+ sql """
+ create table table_4 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "4097");
+ """
+ test {
+ sql "show create table table_4;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"8192\""))
+ }
+ }
+
+ // 65537
+ sql """ DROP TABLE IF EXISTS table_5; """
+ sql """
+ create table table_5 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "65537");
+ """
+ test {
+ sql "show create table table_5;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"69632\""))
+ }
+ }
+ test {
+ sql """ alter table table_5 set ("storage_page_size" = "65535"); """
+ check { result, exception, startTime, endTime ->
+ if (exception) {
+ assert exception.message.contains("You can not modify
storage_page_size")
+ }
+ }
+ }
+
+ // max - 1
+ sql """ DROP TABLE IF EXISTS table_6; """
+ sql """
+ create table table_6 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "10485759");
+ """
+ test {
+ sql "show create table table_6;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"10485760\""))
+ }
+ }
+
+ // max
+ sql """ DROP TABLE IF EXISTS table_7; """
+ sql """
+ create table table_7 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" = "10485760");
+ """
+ test {
+ sql "show create table table_7;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"storage_page_size\" =
\"10485760\""))
+ }
+ }
+
+ // max + 1
+ sql """ DROP TABLE IF EXISTS table_8; """
+ test {
+ sql """
+ create table table_8 (
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ duplicate key (k1)
+ distributed BY random buckets 1
+ properties("replication_num" = "1", "storage_page_size" =
"10485761");
+ """
+ check { result, exception, startTime, endTime ->
+ if (exception) {
+ assert exception.message.contains("Storage page size must be
between 4KB and 10MB.")
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
