This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 49875c3074d [fix](cloud) schema change from not null to null (#32913)
49875c3074d is described below
commit 49875c3074da11be455b32ac28764f7aa6cf351b
Author: Yongqiang YANG <[email protected]>
AuthorDate: Thu Mar 28 12:23:30 2024 +0800
[fix](cloud) schema change from not null to null (#32913)
1. Use equals instead of == for type comparing
2. null bitmap size is reisze by size of ref column.
---
be/src/agent/task_worker_pool.cpp | 3 ++-
be/src/cloud/cloud_schema_change_job.cpp | 4 ++++
be/src/olap/schema_change.cpp | 27 ++++++++++++++++------
be/src/olap/schema_change.h | 4 +++-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 2 +-
5 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index b9a55518c46..183a5744aae 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -190,7 +190,8 @@ void alter_tablet(StorageEngine& engine, const
TAgentTaskRequest& agent_task_req
Status res = Status::OK();
try {
DCHECK(agent_task_req.alter_tablet_req_v2.__isset.job_id);
- SchemaChangeJob job(engine, agent_task_req.alter_tablet_req_v2);
+ SchemaChangeJob job(engine, agent_task_req.alter_tablet_req_v2,
+
std::to_string(agent_task_req.alter_tablet_req_v2.job_id));
status =
job.process_alter_tablet(agent_task_req.alter_tablet_req_v2);
} catch (const Exception& e) {
status = e.to_status();
diff --git a/be/src/cloud/cloud_schema_change_job.cpp
b/be/src/cloud/cloud_schema_change_job.cpp
index e919e3f37aa..d1d323e19a9 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -201,6 +201,10 @@ Status
CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam
"Don't support to add materialized view by linked schema
change");
}
+ LOG(INFO) << "schema change type, sc_sorting: " << sc_sorting
+ << ", sc_directly: " << sc_directly << ", base_tablet=" <<
_base_tablet->tablet_id()
+ << ", new_tablet=" << _new_tablet->tablet_id();
+
// 2. Generate historical data converter
auto sc_procedure = get_sc_procedure(changer, sc_sorting);
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 1a0836014ae..a4e81184b10 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -347,7 +347,7 @@ Status BlockChanger::change_block(vectorized::Block*
ref_block,
assert_cast<vectorized::ColumnNullable*>(new_col->assume_mutable().get());
new_nullable_col->change_nested_column(ref_col);
-
new_nullable_col->get_null_map_data().resize_fill(new_nullable_col->size());
+
new_nullable_col->get_null_map_data().resize_fill(ref_col->size());
} else {
// nullable to not nullable:
// suppose column `c_phone` is originally varchar(16) NOT NULL,
@@ -394,11 +394,22 @@ Status
BlockChanger::_check_cast_valid(vectorized::ColumnPtr ref_column,
return Status::DataQualityError("Null data is changed to not
nullable");
}
} else {
- const auto* new_null_map =
+ const auto& null_map_column =
vectorized::check_and_get_column<vectorized::ColumnNullable>(new_column)
- ->get_null_map_column()
- .get_data()
- .data();
+ ->get_null_map_column();
+ const auto& nested_column =
+
vectorized::check_and_get_column<vectorized::ColumnNullable>(new_column)
+ ->get_nested_column();
+ const auto* new_null_map = null_map_column.get_data().data();
+
+ if (null_map_column.size() != new_column->size() ||
+ nested_column.size() != new_column->size()) {
+ DCHECK(false);
+ return Status::InternalError(
+ "null_map_column size is changed,
null_map_column_size={}, "
+ "new_column_size={}",
+ null_map_column.size(), new_column->size());
+ }
bool is_changed = false;
for (size_t i = 0; i < ref_column->size(); i++) {
@@ -713,7 +724,7 @@ Status SchemaChangeJob::process_alter_tablet(const
TAlterTabletReqV2& request) {
}
SchemaChangeJob::SchemaChangeJob(StorageEngine& local_storage_engine,
- const TAlterTabletReqV2& request)
+ const TAlterTabletReqV2& request, const
std::string& job_id)
: _local_storage_engine(local_storage_engine) {
_base_tablet =
_local_storage_engine.tablet_manager()->get_tablet(request.base_tablet_id);
_new_tablet =
_local_storage_engine.tablet_manager()->get_tablet(request.new_tablet_id);
@@ -726,6 +737,7 @@ SchemaChangeJob::SchemaChangeJob(StorageEngine&
local_storage_engine,
// the complete variant is constructed by reading all the sub-columns
of the variant.
_new_tablet_schema =
_new_tablet->tablet_schema()->copy_without_extracted_columns();
}
+ _job_id = job_id;
}
// In the past schema change and rollup will create new tablet and will wait
for txns starting before the task to finished
@@ -1017,7 +1029,7 @@ Status SchemaChangeJob::_convert_historical_rowsets(const
SchemaChangeParams& sc
int64_t*
real_alter_version) {
LOG(INFO) << "begin to convert historical rowsets for new_tablet from
base_tablet."
<< " base_tablet=" << _base_tablet->tablet_id()
- << ", new_tablet=" << _new_tablet->tablet_id();
+ << ", new_tablet=" << _new_tablet->tablet_id() << ", job_id=" <<
_job_id;
// find end version
int32_t end_version = -1;
@@ -1291,6 +1303,7 @@ Status SchemaChangeJob::parse_request(const
SchemaChangeParams& sc_params,
// use directly schema change instead.
if (!(*sc_directly) && !(*sc_sorting)) {
// check has remote rowset
+ // work for cloud and cold storage
for (const auto& rs_reader : sc_params.ref_rowset_readers) {
if (!rs_reader->rowset()->is_local()) {
*sc_directly = true;
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 2e1198e181a..dc77cff9d06 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -274,7 +274,8 @@ struct SchemaChangeParams {
class SchemaChangeJob {
public:
- SchemaChangeJob(StorageEngine& local_storage_engine, const
TAlterTabletReqV2& request);
+ SchemaChangeJob(StorageEngine& local_storage_engine, const
TAlterTabletReqV2& request,
+ const std::string& job_id);
Status process_alter_tablet(const TAlterTabletReqV2& request);
bool tablet_in_converting(int64_t tablet_id);
@@ -323,5 +324,6 @@ private:
std::shared_mutex _mutex;
std::unordered_set<int64_t> _tablet_ids_in_converting;
std::set<std::string> _supported_functions;
+ std::string _job_id;
};
} // namespace doris
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index fccc517f7a9..2960d9a0744 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -459,7 +459,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
if
(indexColumnMap.containsKey(SchemaChangeHandler.SHADOW_NAME_PREFIX +
column.getName())) {
Column newColumn = indexColumnMap
.get(SchemaChangeHandler.SHADOW_NAME_PREFIX + column.getName());
- if (newColumn.getType() != column.getType()) {
+ if (!newColumn.getType().equals(column.getType()))
{
try {
SlotRef slot = new SlotRef(destSlotDesc);
slot.setCol(column.getName());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
