kangpinghuang commented on a change in pull request #2212: Add rowset convert
URL: https://github.com/apache/incubator-doris/pull/2212#discussion_r351967983
##########
File path: be/src/olap/snapshot_manager.cpp
##########
@@ -467,39 +468,114 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
new_tablet_meta->revise_rs_metas(rs_metas);
}
if (snapshot_version < PREFERRED_SNAPSHOT_VERSION) {
- set<string> exist_old_files;
- Status ret = FileUtils::list_dirs_files(schema_full_path, nullptr,
&exist_old_files,
- Env::Default());
- if (!ret.ok()) {
- res = OLAP_ERR_DISK_FAILURE;
- LOG(WARNING) << "failed to dir walk when convert old files.
dir="
- << schema_full_path << ", error:" <<
ret.to_string();
- break;
- }
- OlapSnapshotConverter converter;
- TabletMetaPB tablet_meta_pb;
- OLAPHeaderMessage olap_header_msg;
- new_tablet_meta->to_meta_pb(&tablet_meta_pb);
- res = converter.to_old_snapshot(tablet_meta_pb, schema_full_path,
schema_full_path, &olap_header_msg);
- if (res != OLAP_SUCCESS) {
- break;
- }
- // convert new version files to old version files successuflly,
then should remove the old files
- vector<string> files_to_delete;
- for (auto file_name : exist_old_files) {
- string full_file_path = schema_full_path + "/" + file_name;
- files_to_delete.push_back(full_file_path);
- }
- // remove all files
- ret = FileUtils::remove_paths(files_to_delete);
- if (!ret.ok()) {
- res = OLAP_ERR_IO_ERROR;
- LOG(WARNING) << "remove paths failed. error: " <<
ret.to_string();
- break;
+ if (snapshot_version < ALPHA_ROWSET_VERSION) {
+ // convert old segment file name to new name
+ // from tabletid_schemahash_segmentgroupid_segmentid to
rowsetid_segmentgroupid_segmentid
+ set<string> exist_old_files;
+ Status ret = FileUtils::list_dirs_files(schema_full_path,
nullptr, &exist_old_files, Env::Default());
+ if (!ret.ok()) {
+ res = OLAP_ERR_DISK_FAILURE;
+ LOG(WARNING) << "failed to dir walk when convert old
files. dir="
+ << schema_full_path << ", error:" << ret.to_string();
+ break;
+ }
+ OlapSnapshotConverter converter;
+ TabletMetaPB tablet_meta_pb;
+ OLAPHeaderMessage olap_header_msg;
+ new_tablet_meta->to_meta_pb(&tablet_meta_pb);
+ res = converter.to_old_snapshot(
+ tablet_meta_pb, schema_full_path, schema_full_path,
+ &olap_header_msg);
+ if (res != OLAP_SUCCESS) {
+ break;
+ }
+ // convert new version files to old version files
successuflly, then should remove the old files
+ vector<string> files_to_delete;
+ for (auto file_name : exist_old_files) {
+ string full_file_path = schema_full_path + "/" + file_name;
+ files_to_delete.push_back(full_file_path);
+ }
+ // remove all files
+ ret = FileUtils::remove_paths(files_to_delete);
+ if (!ret.ok()) {
+ res = OLAP_ERR_IO_ERROR;
+ LOG(WARNING) << "remove paths failed. error: " <<
ret.to_string();
+ break;
+ }
+ // save new header to snapshot header path
+ res = converter.save(header_path, olap_header_msg);
+ LOG(INFO) << "finished convert new snapshot to old snapshot,
res=" << res;
+ } else if (snapshot_version == ALPHA_ROWSET_VERSION) {
+ // convert beta rowset to alpha rowset
+ LOG(INFO) << "start to convert beta rowset to alpha rowset for
snapshot";
+ RowsetConverter rowset_converter(new_tablet_meta);
+ std::vector<RowsetMetaSharedPtr> new_rowsets;
+ bool modified = false;
+ if (request.__isset.missing_version) {
+ for (auto& inc_meta : new_tablet_meta->all_inc_rs_metas())
{
+ if (inc_meta->rowset_type() == BETA_ROWSET) {
+ modified = true;
+ RowsetMetaPB rowset_meta_pb;
+ LOG(INFO) << "convert inc beta rowset:"
+ << inc_meta->rowset_id() << ", tablet_id:"
+ << new_tablet_meta->tablet_id() << " to
alpha";
+ auto st =
rowset_converter.convert_beta_to_alpha(inc_meta,
+ schema_full_path, &rowset_meta_pb);
+ if (st != OLAP_SUCCESS) {
+ res = st;
+ LOG(WARNING) << "convert beta to alpha failed.
error: " << st;
+ break;
+ }
+ RowsetMetaSharedPtr new_rowset(new
AlphaRowsetMeta());
+ bool ret =
new_rowset->init_from_pb(rowset_meta_pb);
+ if (!ret) {
+ res = OLAP_ERR_INIT_FAILED;
+ break;
+ }
+ new_rowsets.push_back(new_rowset);
+ } else {
+ new_rowsets.push_back(inc_meta);
+ }
+ }
+ if (res == OLAP_SUCCESS && modified) {
+ res =
new_tablet_meta->revise_inc_rs_metas(new_rowsets);
+ }
+ } else {
+ for (auto& rowset_meta : new_tablet_meta->all_rs_metas()) {
+ if (rowset_meta->rowset_type() == BETA_ROWSET) {
+ modified = true;
+ RowsetMetaPB rowset_meta_pb;
+ LOG(INFO) << "convert beta rowset:"
+ << rowset_meta->rowset_id() << ",
tablet_id:"
+ << new_tablet_meta->tablet_id() << " to
alpha";
+ auto st = rowset_converter.convert_beta_to_alpha(
+ rowset_meta, schema_full_path,
&rowset_meta_pb);
+ if (st != OLAP_SUCCESS) {
+ res = st;
+ LOG(WARNING) << "convert beta to alpha failed.
error: " << st;
+ break;
+ }
+ RowsetMetaSharedPtr new_rowset(new
AlphaRowsetMeta());
+ bool ret =
new_rowset->init_from_pb(rowset_meta_pb);
+ if (!ret) {
+ res = OLAP_ERR_INIT_FAILED;
+ break;
+ }
+ new_rowsets.push_back(new_rowset);
+ } else {
+ new_rowsets.push_back(rowset_meta);
+ }
+ }
+ if (res == OLAP_SUCCESS && modified) {
+ res = new_tablet_meta->revise_rs_metas(new_rowsets);
+ }
Review comment:
No, it not equal.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
