This is an automated email from the ASF dual-hosted git repository.
lichaoyong pushed a commit to branch revert-3762-tablet-manager2
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 71b207b2d52524bbc0de2dfc9483eb25b748c85e
Author: lichaoyong <lichaoy...@baidu.com>
AuthorDate: Tue Jun 23 16:17:00 2020 +0800
Revert "[Memory Engine] MemTablet creation and compatibility handling in BE
(#3762)"
This reverts commit ca96ea30560c9e9837c28cfd2cdd8ed24196f787.
---
be/src/agent/task_worker_pool.cpp | 13 +-
be/src/http/action/meta_action.cpp | 5 +-
be/src/olap/base_tablet.cpp | 30 +---
be/src/olap/base_tablet.h | 138 ---------------
be/src/olap/data_dir.cpp | 4 +-
be/src/olap/data_dir.h | 5 +-
be/src/olap/memory/mem_tablet.cpp | 34 +---
be/src/olap/memory/mem_tablet.h | 19 +--
be/src/olap/storage_engine.cpp | 12 +-
be/src/olap/tablet.cpp | 25 ++-
be/src/olap/tablet.h | 146 ++++++++++++++--
be/src/olap/tablet_manager.cpp | 286 +++++++++-----------------------
be/src/olap/tablet_manager.h | 30 ++--
be/src/olap/tablet_meta.cpp | 2 +-
be/test/olap/memory/mem_tablet_test.cpp | 2 +-
gensrc/thrift/MasterService.thrift | 1 -
16 files changed, 276 insertions(+), 476 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index dbf175c..e96c847 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -344,9 +344,8 @@ void*
TaskWorkerPool::_create_tablet_worker_thread_callback(void* arg_this) {
} else {
++_s_report_version;
// get path hash of the created tablet
- BaseTabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->
- get_base_tablet(create_tablet_req.tablet_id,
-
create_tablet_req.tablet_schema.schema_hash);
+ TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(
+ create_tablet_req.tablet_id,
create_tablet_req.tablet_schema.schema_hash);
DCHECK(tablet != nullptr);
TTabletInfo tablet_info;
tablet_info.tablet_id = tablet->table_id();
@@ -400,8 +399,8 @@ void*
TaskWorkerPool::_drop_tablet_worker_thread_callback(void* arg_this) {
TStatusCode::type status_code = TStatusCode::OK;
vector<string> error_msgs;
TStatus task_status;
- BaseTabletSharedPtr dropped_tablet =
StorageEngine::instance()->tablet_manager()->
- get_base_tablet(drop_tablet_req.tablet_id,
drop_tablet_req.schema_hash);
+ TabletSharedPtr dropped_tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(
+ drop_tablet_req.tablet_id, drop_tablet_req.schema_hash);
if (dropped_tablet != nullptr) {
OLAPStatus drop_status =
StorageEngine::instance()->tablet_manager()->drop_tablet(
drop_tablet_req.tablet_id, drop_tablet_req.schema_hash);
@@ -829,8 +828,8 @@ void*
TaskWorkerPool::_update_tablet_meta_worker_thread_callback(void* arg_this)
TStatus task_status;
for (auto tablet_meta_info : update_tablet_meta_req.tabletMetaInfos) {
- BaseTabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->
- get_base_tablet(tablet_meta_info.tablet_id,
tablet_meta_info.schema_hash);
+ TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(
+ tablet_meta_info.tablet_id, tablet_meta_info.schema_hash);
if (tablet == nullptr) {
LOG(WARNING) << "could not find tablet when update partition
id"
<< " tablet_id=" << tablet_meta_info.tablet_id
diff --git a/be/src/http/action/meta_action.cpp
b/be/src/http/action/meta_action.cpp
index 90c90a0..2fad2a9 100644
--- a/be/src/http/action/meta_action.cpp
+++ b/be/src/http/action/meta_action.cpp
@@ -53,8 +53,9 @@ Status MetaAction::_handle_header(HttpRequest* req,
std::string* json_meta) {
<< ", schema_hash:" << req_schema_hash;
return Status::InternalError(strings::Substitute("convert failed, $0",
e.what()));
}
- BaseTabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_base_tablet(
- tablet_id, schema_hash);
+
+ TabletSharedPtr tablet =
+ StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id,
schema_hash);
if (tablet == nullptr) {
LOG(WARNING) << "no tablet for tablet_id:" << tablet_id << " schema
hash:" << schema_hash;
return Status::InternalError("no tablet exist");
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 7346715..d544f36 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -26,31 +26,12 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta,
DataDir* data_dir)
: _state(tablet_meta->tablet_state()),
_tablet_meta(tablet_meta),
_schema(tablet_meta->tablet_schema()),
- _data_dir(data_dir),
- _is_bad(false) {
+ _data_dir(data_dir) {
_gen_tablet_path();
}
BaseTablet::~BaseTablet() {}
-
-OLAPStatus BaseTablet::init() {
- return _init_once.call([this] { return _init_once_action(); });
-}
-
-// should save tablet meta to remote meta store
-// if it's a primary replica
-void BaseTablet::save_meta() {
- auto res = _tablet_meta->save_meta(_data_dir);
- CHECK_EQ(res, OLAP_SUCCESS) << "fail to save tablet_meta. res=" << res
- << ", root=" << _data_dir->path();
- // User could directly update tablet schema by _tablet_meta,
- // So we need to refetch schema again
- _schema = _tablet_meta->tablet_schema();
- // TODO: update _mem_schema too?
-}
-
-
OLAPStatus BaseTablet::set_tablet_state(TabletState state) {
if (_tablet_meta->tablet_state() == TABLET_SHUTDOWN && state !=
TABLET_SHUTDOWN) {
LOG(WARNING) << "could not change tablet state from shutdown to " <<
state;
@@ -71,13 +52,4 @@ void BaseTablet::_gen_tablet_path() {
}
}
-OLAPStatus BaseTablet::set_partition_id(int64_t partition_id) {
- return _tablet_meta->set_partition_id(partition_id);
-}
-
-TabletInfo BaseTablet::get_tablet_info() const {
- return TabletInfo(tablet_id(), schema_hash(), tablet_uid());
-}
-
-
} /* namespace doris */
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index f3ebfb0..f3b0c2d 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -20,19 +20,13 @@
#include <memory>
-#include "gen_cpp/AgentService_types.h"
-#include "gen_cpp/MasterService_types.h"
-#include "olap/data_dir.h"
#include "olap/olap_define.h"
#include "olap/tablet_meta.h"
#include "olap/utils.h"
-#include "util/once.h"
namespace doris {
class DataDir;
-class BaseTablet;
-using BaseTabletSharedPtr = std::shared_ptr<BaseTablet>;
// Base class for all tablet classes, currently only olap/Tablet and
// olap/memory/MemTablet.
@@ -66,54 +60,11 @@ public:
inline void set_creation_time(int64_t creation_time);
inline bool equal(int64_t tablet_id, int32_t schema_hash);
- OLAPStatus init();
- inline bool init_succeeded();
-
- bool is_used();
-
- void save_meta();
-
- void register_tablet_into_dir();
- void deregister_tablet_from_dir();
-
-
// properties encapsulated in TabletSchema
inline const TabletSchema& tablet_schema() const;
- inline size_t tablet_footprint(); // disk space occupied by tablet
- inline size_t num_rows();
- inline int version_count() const;
- inline Version max_version() const;
-
- // propreties encapsulated in TabletSchema
- inline KeysType keys_type() const;
- inline size_t num_columns() const;
- inline size_t num_null_columns() const;
- inline size_t num_key_columns() const;
- inline size_t num_short_key_columns() const;
- inline size_t num_rows_per_row_block() const;
- inline CompressKind compress_kind() const;
- inline double bloom_filter_fpp() const;
- inline size_t next_unique_id() const;
- inline size_t row_size() const;
- inline size_t field_index(const string& field_name) const;
-
- OLAPStatus set_partition_id(int64_t partition_id);
-
- TabletInfo get_tablet_info() const;
-
- // meta lock
- inline void obtain_header_rdlock() { _meta_lock.rdlock(); }
- inline void obtain_header_wrlock() { _meta_lock.wrlock(); }
- inline void release_header_lock() { _meta_lock.unlock(); }
- inline RWMutex* get_header_lock_ptr() { return &_meta_lock; }
-
- virtual void build_tablet_report_info(TTabletInfo* tablet_info) = 0;
-
- virtual void delete_all_files() = 0;
protected:
void _gen_tablet_path();
- virtual OLAPStatus _init_once_action() = 0;
protected:
TabletState _state;
@@ -123,13 +74,6 @@ protected:
DataDir* _data_dir;
std::string _tablet_path;
- DorisCallOnce<OLAPStatus> _init_once;
- // TODO(lingbin): There is a _meta_lock TabletMeta too, there should be a
comment to
- // explain how these two locks work together.
- mutable RWMutex _meta_lock;
- // if this tablet is broken, set to true. default is false
- std::atomic<bool> _is_bad;
-
private:
DISALLOW_COPY_AND_ASSIGN(BaseTablet);
};
@@ -197,88 +141,6 @@ inline const TabletSchema& BaseTablet::tablet_schema()
const {
return _schema;
}
-inline bool BaseTablet::init_succeeded() {
- return _init_once.has_called() && _init_once.stored_result() ==
OLAP_SUCCESS;
-}
-
-inline bool BaseTablet::is_used() {
- return !_is_bad && _data_dir->is_used();
-}
-
-inline void BaseTablet::register_tablet_into_dir() {
- _data_dir->register_tablet(this);
-}
-
-inline void BaseTablet::deregister_tablet_from_dir() {
- _data_dir->deregister_tablet(this);
-}
-
-// TODO(lingbin): Why other methods that need to get information from
_tablet_meta
-// are not locked, here needs a comment to explain.
-inline size_t BaseTablet::tablet_footprint() {
- ReadLock rdlock(&_meta_lock);
- return _tablet_meta->tablet_footprint();
-}
-
-// TODO(lingbin): Why other methods which need to get information from
_tablet_meta
-// are not locked, here needs a comment to explain.
-inline size_t BaseTablet::num_rows() {
- ReadLock rdlock(&_meta_lock);
- return _tablet_meta->num_rows();
-}
-
-inline int BaseTablet::version_count() const {
- return _tablet_meta->version_count();
-}
-
-inline Version BaseTablet::max_version() const {
- return _tablet_meta->max_version();
-}
-
-inline KeysType BaseTablet::keys_type() const {
- return _schema.keys_type();
-}
-
-inline size_t BaseTablet::num_columns() const {
- return _schema.num_columns();
-}
-
-inline size_t BaseTablet::num_null_columns() const {
- return _schema.num_null_columns();
-}
-
-inline size_t BaseTablet::num_key_columns() const {
- return _schema.num_key_columns();
-}
-
-inline size_t BaseTablet::num_short_key_columns() const {
- return _schema.num_short_key_columns();
-}
-
-inline size_t BaseTablet::num_rows_per_row_block() const {
- return _schema.num_rows_per_row_block();
-}
-
-inline CompressKind BaseTablet::compress_kind() const {
- return _schema.compress_kind();
-}
-
-inline double BaseTablet::bloom_filter_fpp() const {
- return _schema.bloom_filter_fpp();
-}
-
-inline size_t BaseTablet::next_unique_id() const {
- return _schema.next_column_unique_id();
-}
-
-inline size_t BaseTablet::field_index(const string& field_name) const {
- return _schema.field_index(field_name);
-}
-
-inline size_t BaseTablet::row_size() const {
- return _schema.row_size();
-}
-
} /* namespace doris */
#endif /* DORIS_BE_SRC_OLAP_BASE_TABLET_H */
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 8cabfd9..f577f97 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -334,14 +334,14 @@ OLAPStatus DataDir::get_shard(uint64_t* shard) {
return OLAP_SUCCESS;
}
-void DataDir::register_tablet(BaseTablet* tablet) {
+void DataDir::register_tablet(Tablet* tablet) {
TabletInfo tablet_info(tablet->tablet_id(), tablet->schema_hash(),
tablet->tablet_uid());
std::lock_guard<std::mutex> l(_mutex);
_tablet_set.emplace(std::move(tablet_info));
}
-void DataDir::deregister_tablet(BaseTablet* tablet) {
+void DataDir::deregister_tablet(Tablet* tablet) {
TabletInfo tablet_info(tablet->tablet_id(), tablet->schema_hash(),
tablet->tablet_uid());
std::lock_guard<std::mutex> l(_mutex);
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index f8df04a..58992ed 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -32,7 +32,6 @@
namespace doris {
-class BaseTablet;
class Tablet;
class TabletManager;
class TabletMeta;
@@ -81,8 +80,8 @@ public:
TStorageMedium::type storage_medium() const { return _storage_medium; }
- void register_tablet(BaseTablet* tablet);
- void deregister_tablet(BaseTablet* tablet);
+ void register_tablet(Tablet* tablet);
+ void deregister_tablet(Tablet* tablet);
void clear_tablets(std::vector<TabletInfo>* tablet_infos);
std::string get_absolute_shard_path(int64_t shard_id);
diff --git a/be/src/olap/memory/mem_tablet.cpp
b/be/src/olap/memory/mem_tablet.cpp
index 0e5f33b..03a9d45 100644
--- a/be/src/olap/memory/mem_tablet.cpp
+++ b/be/src/olap/memory/mem_tablet.cpp
@@ -36,18 +36,11 @@ std::shared_ptr<MemTablet>
MemTablet::create_tablet_from_meta(TabletMetaSharedPt
return std::make_shared<MemTablet>(tablet_meta, data_dir);
}
-OLAPStatus MemTablet::_init_once_action() {
+Status MemTablet::init() {
_max_version = 0;
- Status ret = MemSubTablet::create(0, *_mem_schema.get(), &_sub_tablet);
- if (ret.ok()) {
- return OLAP_SUCCESS;
- } else {
- // TODO: Status/OLAPStatus compatibility
- return OLAP_ERR_INIT_FAILED;
- }
+ return MemSubTablet::create(0, *_mem_schema.get(), &_sub_tablet);
}
-
Status MemTablet::scan(std::unique_ptr<ScanSpec>* spec,
std::unique_ptr<MemTabletScan>* scan) {
uint64_t version = (*spec)->version();
if (version == UINT64_MAX) {
@@ -93,28 +86,5 @@ Status MemTablet::commit_write_txn(WriteTxn* wtxn, uint64_t
version) {
return Status::OK();
}
-void MemTablet::build_tablet_report_info(TTabletInfo* tablet_info) {
- ReadLock rdlock(&_meta_lock);
- tablet_info->tablet_id = _tablet_meta->tablet_id();
- tablet_info->schema_hash = _tablet_meta->schema_hash();
- tablet_info->row_count = _tablet_meta->num_rows();
- tablet_info->data_size = _tablet_meta->tablet_footprint();
- tablet_info->version = _max_version;
- tablet_info->version_hash = 0;
- tablet_info->__set_partition_id(_tablet_meta->partition_id());
- tablet_info->__set_storage_medium(_data_dir->storage_medium());
- tablet_info->__set_version_count(_tablet_meta->version_count());
- tablet_info->__set_path_hash(_data_dir->path_hash());
-
tablet_info->__set_is_in_memory(_tablet_meta->tablet_schema().is_in_memory());
- tablet_info->__set_tablet_type(_tablet_meta->tablet_type() ==
TabletTypePB::TABLET_TYPE_DISK ?
- TTabletType::TABLET_TYPE_DISK : TTabletType::TABLET_TYPE_MEMORY);
-}
-
-void MemTablet::delete_all_files() {
- // TODO:
-}
-
-
-
} // namespace memory
} // namespace doris
diff --git a/be/src/olap/memory/mem_tablet.h b/be/src/olap/memory/mem_tablet.h
index 03c3933..dfafa27 100644
--- a/be/src/olap/memory/mem_tablet.h
+++ b/be/src/olap/memory/mem_tablet.h
@@ -27,15 +27,6 @@ class MemSubTablet;
class ScanSpec;
class MemTabletScan;
class WriteTxn;
-class MemTablet;
-using MemTabletSharedPtr = std::shared_ptr<MemTablet>;
-
-inline MemTabletSharedPtr to_mem_tablet(const BaseTabletSharedPtr& base) {
- if (base->is_memory()) {
- return std::static_pointer_cast<MemTablet>(base);
- }
- return MemTabletSharedPtr();
-}
// Tablet class for memory-optimized storage engine.
//
@@ -58,6 +49,9 @@ public:
virtual ~MemTablet();
+ // Initialize
+ Status init();
+
// Scan the tablet, return a MemTabletScan object scan, user can specify
projections
// using ScanSpec, currently only support full scan with projection, will
support
// filter/aggregation in the future.
@@ -76,13 +70,6 @@ public:
// Note: commit is done sequentially, protected by internal write lock
Status commit_write_txn(WriteTxn* wtxn, uint64_t version);
- virtual void build_tablet_report_info(TTabletInfo* tablet_info);
-
- virtual void delete_all_files();
-
-protected:
- virtual OLAPStatus _init_once_action();
-
private:
friend class MemTabletScan;
// memory::Schema is used internally rather than TabletSchema, so we need
an extra
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 165b4c2..61d7ab9 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -907,9 +907,9 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) {
vector<TabletInfo> tablet_infos;
task->get_related_tablets(&tablet_infos);
sort(tablet_infos.begin(), tablet_infos.end());
- vector<BaseTabletSharedPtr> related_tablets;
+ vector<TabletSharedPtr> related_tablets;
for (TabletInfo& tablet_info : tablet_infos) {
- BaseTabletSharedPtr tablet = _tablet_manager->get_base_tablet(
+ TabletSharedPtr tablet = _tablet_manager->get_tablet(
tablet_info.tablet_id, tablet_info.schema_hash);
if (tablet != nullptr) {
related_tablets.push_back(tablet);
@@ -921,7 +921,7 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) {
}
// add write lock to all related tablets
OLAPStatus prepare_status = task->prepare();
- for (auto& tablet : related_tablets) {
+ for (TabletSharedPtr& tablet : related_tablets) {
tablet->release_header_lock();
}
if (prepare_status != OLAP_SUCCESS) {
@@ -943,9 +943,9 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) {
// related tablets may be changed after execute task, so that get them
here again
task->get_related_tablets(&tablet_infos);
sort(tablet_infos.begin(), tablet_infos.end());
- vector<BaseTabletSharedPtr> related_tablets;
+ vector<TabletSharedPtr> related_tablets;
for (TabletInfo& tablet_info : tablet_infos) {
- auto tablet = _tablet_manager->get_base_tablet(
+ TabletSharedPtr tablet = _tablet_manager->get_tablet(
tablet_info.tablet_id, tablet_info.schema_hash);
if (tablet != nullptr) {
related_tablets.push_back(tablet);
@@ -957,7 +957,7 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) {
}
// add write lock to all related tablets
OLAPStatus fin_status = task->finish();
- for (auto& tablet : related_tablets) {
+ for (TabletSharedPtr& tablet : related_tablets) {
tablet->release_header_lock();
}
return fin_status;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 429c37a..e48ca02 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -55,6 +55,7 @@ TabletSharedPtr
Tablet::create_tablet_from_meta(TabletMetaSharedPtr tablet_meta,
Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir) :
BaseTablet(tablet_meta, data_dir),
+ _is_bad(false),
_last_cumu_compaction_failure_millis(0),
_last_base_compaction_failure_millis(0),
_last_cumu_compaction_success_millis(0),
@@ -101,6 +102,20 @@ OLAPStatus Tablet::_init_once_action() {
return res;
}
+OLAPStatus Tablet::init() {
+ return _init_once.call([this] { return _init_once_action(); });
+}
+
+// should save tablet meta to remote meta store
+// if it's a primary replica
+void Tablet::save_meta() {
+ auto res = _tablet_meta->save_meta(_data_dir);
+ CHECK_EQ(res, OLAP_SUCCESS) << "fail to save tablet_meta. res=" << res <<
", root=" << _data_dir->path();
+ // User could directly update tablet schema by _tablet_meta,
+ // So we need to refetch schema again
+ _schema = _tablet_meta->tablet_schema();
+}
+
OLAPStatus Tablet::revise_tablet_meta(
const vector<RowsetMetaSharedPtr>& rowsets_to_clone,
const vector<Version>& versions_to_delete) {
@@ -829,6 +844,14 @@ OLAPStatus Tablet::_contains_version(const Version&
version) {
return OLAP_SUCCESS;
}
+OLAPStatus Tablet::set_partition_id(int64_t partition_id) {
+ return _tablet_meta->set_partition_id(partition_id);
+}
+
+TabletInfo Tablet::get_tablet_info() const {
+ return TabletInfo(tablet_id(), schema_hash(), tablet_uid());
+}
+
void Tablet::pick_candicate_rowsets_to_cumulative_compaction(int64_t
skip_window_sec,
std::vector<RowsetSharedPtr>* candidate_rowsets) {
int64_t now = UnixSeconds();
@@ -1022,8 +1045,6 @@ void Tablet::build_tablet_report_info(TTabletInfo*
tablet_info) {
tablet_info->__set_version_count(_tablet_meta->version_count());
tablet_info->__set_path_hash(_data_dir->path_hash());
tablet_info->__set_is_in_memory(_tablet_meta->tablet_schema().is_in_memory());
- tablet_info->__set_tablet_type(_tablet_meta->tablet_type() ==
TabletTypePB::TABLET_TYPE_DISK ?
- TTabletType::TABLET_TYPE_DISK : TTabletType::TABLET_TYPE_MEMORY);
}
// should use this method to get a copy of current tablet meta
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index e7bd244..40cac13 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -46,13 +46,6 @@ class TabletMeta;
using TabletSharedPtr = std::shared_ptr<Tablet>;
-inline TabletSharedPtr to_tablet(const BaseTabletSharedPtr& base) {
- if (base->is_memory()) {
- return TabletSharedPtr();
- }
- return std::static_pointer_cast<Tablet>(base);
-}
-
class Tablet : public BaseTablet {
public:
static TabletSharedPtr create_tablet_from_meta(TabletMetaSharedPtr
tablet_meta,
@@ -60,6 +53,15 @@ public:
Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir);
+ OLAPStatus init();
+ inline bool init_succeeded();
+
+ bool is_used();
+
+ void register_tablet_into_dir();
+ void deregister_tablet_from_dir();
+
+ void save_meta();
// Used in clone task, to update local meta when finishing a clone job
OLAPStatus revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
const std::vector<Version>&
versions_to_delete);
@@ -67,6 +69,24 @@ public:
inline const int64_t cumulative_layer_point() const;
inline void set_cumulative_layer_point(int64_t new_point);
+ inline size_t tablet_footprint(); // disk space occupied by tablet
+ inline size_t num_rows();
+ inline int version_count() const;
+ inline Version max_version() const;
+
+ // propreties encapsulated in TabletSchema
+ inline KeysType keys_type() const;
+ inline size_t num_columns() const;
+ inline size_t num_null_columns() const;
+ inline size_t num_key_columns() const;
+ inline size_t num_short_key_columns() const;
+ inline size_t num_rows_per_row_block() const;
+ inline CompressKind compress_kind() const;
+ inline double bloom_filter_fpp() const;
+ inline size_t next_unique_id() const;
+ inline size_t row_size() const;
+ inline size_t field_index(const string& field_name) const;
+
// operation in rowsets
OLAPStatus add_rowset(RowsetSharedPtr rowset, bool need_persist = true);
void modify_rowsets(const vector<RowsetSharedPtr>& to_add,
@@ -108,6 +128,12 @@ public:
void delete_alter_task();
OLAPStatus set_alter_state(AlterTabletState state);
+ // meta lock
+ inline void obtain_header_rdlock() { _meta_lock.rdlock(); }
+ inline void obtain_header_wrlock() { _meta_lock.wrlock(); }
+ inline void release_header_lock() { _meta_lock.unlock(); }
+ inline RWMutex* get_header_lock_ptr() { return &_meta_lock; }
+
// ingest lock
inline void obtain_push_lock() { _ingest_lock.lock(); }
inline void release_push_lock() { _ingest_lock.unlock(); }
@@ -167,9 +193,15 @@ public:
int64_t last_base_compaction_success_time() { return
_last_base_compaction_success_millis; }
void set_last_base_compaction_success_time(int64_t millis) {
_last_base_compaction_success_millis = millis; }
+ void delete_all_files();
+
bool check_path(const std::string& check_path) const;
bool check_rowset_id(const RowsetId& rowset_id);
+ OLAPStatus set_partition_id(int64_t partition_id);
+
+ TabletInfo get_tablet_info() const;
+
void pick_candicate_rowsets_to_cumulative_compaction(int64_t
skip_window_sec,
std::vector<RowsetSharedPtr>* candidate_rowsets);
void
pick_candicate_rowsets_to_base_compaction(std::vector<RowsetSharedPtr>*
candidate_rowsets);
@@ -187,24 +219,20 @@ public:
bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta);
+ void build_tablet_report_info(TTabletInfo* tablet_info);
+
void generate_tablet_meta_copy(TabletMetaSharedPtr new_tablet_meta) const;
// return a json string to show the compaction status of this tablet
void get_compaction_status(std::string* json_result);
- virtual void build_tablet_report_info(TTabletInfo* tablet_info);
-
- virtual void delete_all_files();
-
-protected:
- virtual OLAPStatus _init_once_action();
-
private:
+ OLAPStatus _init_once_action();
void _print_missed_versions(const std::vector<Version>& missed_versions)
const;
bool _contains_rowset(const RowsetId rowset_id);
OLAPStatus _contains_version(const Version& version);
void _max_continuous_version_from_begining_unlocked(Version* version,
- VersionHash* v_hash)
const;
+ VersionHash* v_hash)
const ;
RowsetSharedPtr _rowset_with_largest_size();
void _delete_inc_rowset_by_version(const Version& version, const
VersionHash& version_hash);
OLAPStatus _capture_consistent_rowsets_unlocked(const vector<Version>&
version_path,
@@ -215,6 +243,7 @@ private:
RowsetGraph _rs_graph;
+ DorisCallOnce<OLAPStatus> _init_once;
// meta store lock is used for prevent 2 threads do checkpoint concurrently
// it will be used in econ-mode in the future
RWMutex _meta_store_lock;
@@ -223,6 +252,9 @@ private:
Mutex _cumulative_lock;
RWMutex _migration_lock;
+ // TODO(lingbin): There is a _meta_lock TabletMeta too, there should be a
comment to
+ // explain how these two locks work together.
+ mutable RWMutex _meta_lock;
// A new load job will produce a new rowset, which will be inserted into
both _rs_version_map
// and _inc_rs_version_map. Only the most recent rowsets are kept in
_inc_rs_version_map to
// reduce the amount of data that needs to be copied during the clone task.
@@ -236,6 +268,8 @@ private:
std::unordered_map<Version, RowsetSharedPtr, HashOfVersion>
_rs_version_map;
std::unordered_map<Version, RowsetSharedPtr, HashOfVersion>
_inc_rs_version_map;
+ // if this tablet is broken, set to true. default is false
+ std::atomic<bool> _is_bad;
// timestamp of last cumu compaction failure
std::atomic<int64_t> _last_cumu_compaction_failure_millis;
// timestamp of last base compaction failure
@@ -251,6 +285,22 @@ private:
DISALLOW_COPY_AND_ASSIGN(Tablet);
};
+inline bool Tablet::init_succeeded() {
+ return _init_once.has_called() && _init_once.stored_result() ==
OLAP_SUCCESS;
+}
+
+inline bool Tablet::is_used() {
+ return !_is_bad && _data_dir->is_used();
+}
+
+inline void Tablet::register_tablet_into_dir() {
+ _data_dir->register_tablet(this);
+}
+
+inline void Tablet::deregister_tablet_from_dir() {
+ _data_dir->deregister_tablet(this);
+}
+
inline const int64_t Tablet::cumulative_layer_point() const {
return _cumulative_point;
@@ -261,6 +311,72 @@ inline void Tablet::set_cumulative_layer_point(int64_t
new_point) {
}
+// TODO(lingbin): Why other methods that need to get information from
_tablet_meta
+// are not locked, here needs a comment to explain.
+inline size_t Tablet::tablet_footprint() {
+ ReadLock rdlock(&_meta_lock);
+ return _tablet_meta->tablet_footprint();
+}
+
+// TODO(lingbin): Why other methods which need to get information from
_tablet_meta
+// are not locked, here needs a comment to explain.
+inline size_t Tablet::num_rows() {
+ ReadLock rdlock(&_meta_lock);
+ return _tablet_meta->num_rows();
+}
+
+inline int Tablet::version_count() const {
+ return _tablet_meta->version_count();
+}
+
+inline Version Tablet::max_version() const {
+ return _tablet_meta->max_version();
+}
+
+inline KeysType Tablet::keys_type() const {
+ return _schema.keys_type();
+}
+
+inline size_t Tablet::num_columns() const {
+ return _schema.num_columns();
+}
+
+inline size_t Tablet::num_null_columns() const {
+ return _schema.num_null_columns();
+}
+
+inline size_t Tablet::num_key_columns() const {
+ return _schema.num_key_columns();
+}
+
+inline size_t Tablet::num_short_key_columns() const {
+ return _schema.num_short_key_columns();
+}
+
+inline size_t Tablet::num_rows_per_row_block() const {
+ return _schema.num_rows_per_row_block();
+}
+
+inline CompressKind Tablet::compress_kind() const {
+ return _schema.compress_kind();
+}
+
+inline double Tablet::bloom_filter_fpp() const {
+ return _schema.bloom_filter_fpp();
+}
+
+inline size_t Tablet::next_unique_id() const {
+ return _schema.next_column_unique_id();
+}
+
+inline size_t Tablet::field_index(const string& field_name) const {
+ return _schema.field_index(field_name);
+}
+
+inline size_t Tablet::row_size() const {
+ return _schema.row_size();
+}
+
}
#endif // DORIS_BE_SRC_OLAP_TABLET_H
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index aafd7d9..3c3fbe5 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -41,7 +41,6 @@
#include "olap/rowset/column_data_writer.h"
#include "olap/rowset/rowset_factory.h"
#include "olap/rowset/rowset_id_generator.h"
-#include "olap/memory/mem_tablet.h"
#include "olap/schema_change.h"
#include "olap/tablet.h"
#include "olap/tablet_meta.h"
@@ -63,7 +62,7 @@ using strings::Substitute;
namespace doris {
-static bool _cmp_tablet_by_create_time(const BaseTabletSharedPtr& a, const
BaseTabletSharedPtr& b) {
+static bool _cmp_tablet_by_create_time(const TabletSharedPtr& a, const
TabletSharedPtr& b) {
return a->creation_time() < b->creation_time();
}
@@ -82,15 +81,15 @@ TabletManager::~TabletManager() {
}
OLAPStatus TabletManager::_add_tablet_unlocked(TTabletId tablet_id, SchemaHash
schema_hash,
- const BaseTabletSharedPtr&
base_tablet,
+ const TabletSharedPtr& tablet,
bool update_meta, bool force) {
OLAPStatus res = OLAP_SUCCESS;
VLOG(3) << "begin to add tablet to TabletManager. " << "tablet_id=" <<
tablet_id
<< ", schema_hash=" << schema_hash << ", force=" << force;
- BaseTabletSharedPtr existed_tablet = nullptr;
+ TabletSharedPtr existed_tablet = nullptr;
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
- for (auto& item : tablet_map[tablet_id].table_arr) {
+ for (TabletSharedPtr item : tablet_map[tablet_id].table_arr) {
if (item->equal(tablet_id, schema_hash)) {
existed_tablet = item;
break;
@@ -99,34 +98,26 @@ OLAPStatus TabletManager::_add_tablet_unlocked(TTabletId
tablet_id, SchemaHash s
if (existed_tablet == nullptr) {
return _add_tablet_to_map_unlocked(tablet_id, schema_hash,
- base_tablet, update_meta,
+ tablet, update_meta,
false /*keep_files*/, false
/*drop_old*/);
}
if (!force) {
- if (existed_tablet->tablet_path() == base_tablet->tablet_path()) {
+ if (existed_tablet->tablet_path() == tablet->tablet_path()) {
LOG(WARNING) << "add the same tablet twice! tablet_id=" <<
tablet_id
<< ", schema_hash=" << schema_hash
- << ", tablet_path=" << base_tablet->tablet_path();
+ << ", tablet_path=" << tablet->tablet_path();
return OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE;
}
- if (existed_tablet->data_dir() == base_tablet->data_dir()) {
+ if (existed_tablet->data_dir() == tablet->data_dir()) {
LOG(WARNING) << "add tablet with same data dir twice! tablet_id="
<< tablet_id
<< ", schema_hash=" << schema_hash;
return OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE;
}
}
- if (base_tablet->is_memory() || existed_tablet->is_memory()) {
- LOG(WARNING) << "add the same MemTablet twice! tablet_id=" << tablet_id
- << ", schema_hash=" << schema_hash
- << ", tablet_path=" << base_tablet->tablet_path();
- return OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE;
- }
-
- TabletSharedPtr tablet = to_tablet(base_tablet);
existed_tablet->obtain_header_rdlock();
- const RowsetSharedPtr old_rowset =
to_tablet(existed_tablet)->rowset_with_max_version();
+ const RowsetSharedPtr old_rowset =
existed_tablet->rowset_with_max_version();
const RowsetSharedPtr new_rowset = tablet->rowset_with_max_version();
// If new tablet is empty, it is a newly created schema change tablet.
@@ -172,7 +163,7 @@ OLAPStatus TabletManager::_add_tablet_unlocked(TTabletId
tablet_id, SchemaHash s
}
OLAPStatus TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id,
SchemaHash schema_hash,
- const
BaseTabletSharedPtr& tablet,
+ const TabletSharedPtr&
tablet,
bool update_meta, bool
keep_files,
bool drop_old) {
// check if new tablet's meta is in store and add new tablet's meta to
meta store
@@ -233,7 +224,7 @@ OLAPStatus TabletManager::create_tablet(const
TCreateTabletReq& request,
// tablet_id exist but with different schema_hash, return an error(report
task will
// eventually trigger its deletion).
if (_check_tablet_id_exist_unlocked(tablet_id)) {
- BaseTabletSharedPtr tablet = _get_base_tablet_unlocked(tablet_id,
schema_hash);
+ TabletSharedPtr tablet = _get_tablet_unlocked(tablet_id, schema_hash);
if (tablet != nullptr) {
LOG(INFO) << "success to create tablet. tablet already exist.
tablet_id=" << tablet_id;
return OLAP_SUCCESS;
@@ -250,7 +241,6 @@ OLAPStatus TabletManager::create_tablet(const
TCreateTabletReq& request,
// If the CreateTabletReq has base_tablet_id then it is a alter-tablet
request
if (request.__isset.base_tablet_id && request.base_tablet_id > 0) {
is_schema_change = true;
- // MemTablet does not support schema change, so it's safe to use
TabletSharedPtr
base_tablet = _get_tablet_unlocked(request.base_tablet_id,
request.base_schema_hash);
if (base_tablet == nullptr) {
LOG(WARNING) << "fail to create tablet(change schema), base tablet
does not exist. "
@@ -268,7 +258,7 @@ OLAPStatus TabletManager::create_tablet(const
TCreateTabletReq& request,
}
// set alter type to schema-change. it is useless
- auto tablet = _internal_create_tablet_unlocked(
+ TabletSharedPtr tablet = _internal_create_tablet_unlocked(
AlterTabletType::SCHEMA_CHANGE, request, is_schema_change,
base_tablet.get(), stores);
if (tablet == nullptr) {
LOG(WARNING) << "fail to create tablet. tablet_id=" <<
request.tablet_id;
@@ -281,7 +271,7 @@ OLAPStatus TabletManager::create_tablet(const
TCreateTabletReq& request,
return OLAP_SUCCESS;
}
-BaseTabletSharedPtr TabletManager::_internal_create_tablet_unlocked(
+TabletSharedPtr TabletManager::_internal_create_tablet_unlocked(
const AlterTabletType alter_type, const TCreateTabletReq& request,
const bool is_schema_change, const Tablet* base_tablet,
const std::vector<DataDir*>& data_dirs) {
@@ -322,22 +312,17 @@ BaseTabletSharedPtr
TabletManager::_internal_create_tablet_unlocked(
// 1. !is_schema_change: not in schema-change state;
// 2. request.base_tablet_id > 0: in schema-change state;
if (!is_schema_change || (request.__isset.base_tablet_id &&
request.base_tablet_id > 0)) {
- if (!tablet->is_memory()) {
- // Create init version if this is not a restore mode replica
and request.version is set
- // bool in_restore_mode = request.__isset.in_restore_mode &&
request.in_restore_mode;
- // if (!in_restore_mode && request.__isset.version) {
- // create inital rowset before add it to storage engine could
omit many locks
- res = _create_inital_rowset_unlocked(request,
to_tablet(tablet).get());
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to create initial version for
tablet. res=" << res;
- break;
- }
+ // Create init version if this is not a restore mode replica and
request.version is set
+ // bool in_restore_mode = request.__isset.in_restore_mode &&
request.in_restore_mode;
+ // if (!in_restore_mode && request.__isset.version) {
+ // create inital rowset before add it to storage engine could omit
many locks
+ res = _create_inital_rowset_unlocked(request, tablet.get());
+ if (res != OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to create initial version for tablet.
res=" << res;
+ break;
}
}
if (is_schema_change) {
- if (tablet->is_memory()) {
- LOG(FATAL) << "MemTablet schema change not supported";
- }
if (request.__isset.base_tablet_id && request.base_tablet_id > 0) {
LOG(INFO) << "request for alter-tablet v2, do not add alter
task to tablet";
// if this is a new alter tablet, has to set its state to not
ready
@@ -346,10 +331,8 @@ BaseTabletSharedPtr
TabletManager::_internal_create_tablet_unlocked(
tablet->set_tablet_state(TabletState::TABLET_NOTREADY);
} else {
// add alter task to new tablet if it is a new tablet during
schema change
- to_tablet(tablet)->add_alter_task(base_tablet->tablet_id(),
- base_tablet->schema_hash(),
- vector<Version>(),
- alter_type);
+ tablet->add_alter_task(base_tablet->tablet_id(),
base_tablet->schema_hash(),
+ vector<Version>(), alter_type);
}
// 有可能出现以下2种特殊情况：
// 1. 因为操作系统时间跳变，导致新生成的表的creation_time小于旧表的creation_time时间
@@ -376,7 +359,7 @@ BaseTabletSharedPtr
TabletManager::_internal_create_tablet_unlocked(
// TODO(lingbin): The following logic seems useless, can be removed?
// Because if _add_tablet_unlocked() return OK, we must can get it
from map.
- BaseTabletSharedPtr tablet_ptr =
_get_base_tablet_unlocked(new_tablet_id, new_schema_hash);
+ TabletSharedPtr tablet_ptr = _get_tablet_unlocked(new_tablet_id,
new_schema_hash);
if (tablet_ptr == nullptr) {
res = OLAP_ERR_TABLE_NOT_FOUND;
LOG(WARNING) << "fail to get tablet. res=" << res;
@@ -408,7 +391,7 @@ static string _gen_tablet_dir(const string& dir, int16_t
shard_id, int64_t table
return path;
}
-BaseTabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked(
+TabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked(
const TCreateTabletReq& request, const bool is_schema_change,
const Tablet* base_tablet, const std::vector<DataDir*>& data_dirs) {
string pending_id = StrCat(TABLET_ID_PREFIX, request.tablet_id);
@@ -444,25 +427,16 @@ BaseTabletSharedPtr
TabletManager::_create_tablet_meta_and_dir_unlocked(
} else {
data_dir->add_pending_ids(pending_id);
Status st = FileUtils::create_dir(schema_hash_dir);
- if (!st.ok()) {
+ if(!st.ok()) {
LOG(WARNING) << "create dir fail. path=" << schema_hash_dir
<< " error=" << st.to_string();
continue;
}
}
- TTabletType::type ttype = request.__isset.tablet_type ?
- request.tablet_type : TTabletType::TABLET_TYPE_DISK;
- if (ttype == TTabletType::TABLET_TYPE_DISK) {
- TabletSharedPtr new_tablet =
Tablet::create_tablet_from_meta(tablet_meta, data_dir);
- DCHECK(new_tablet != nullptr);
- return std::static_pointer_cast<BaseTablet>(new_tablet);
- } else {
- memory::MemTabletSharedPtr new_tablet =
memory::MemTablet::create_tablet_from_meta(
- tablet_meta, data_dir);
- DCHECK(new_tablet != nullptr);
- return std::static_pointer_cast<BaseTablet>(new_tablet);
- }
+ TabletSharedPtr new_tablet =
Tablet::create_tablet_from_meta(tablet_meta, data_dir);
+ DCHECK(new_tablet != nullptr);
+ return new_tablet;
}
return nullptr;
}
@@ -488,18 +462,13 @@ OLAPStatus TabletManager::_drop_tablet_unlocked(
DorisMetrics::instance()->drop_tablet_requests_total.increment(1);
// Fetch tablet which need to be droped
- BaseTabletSharedPtr to_drop_base_tablet =
_get_base_tablet_unlocked(tablet_id, schema_hash);
- if (to_drop_base_tablet == nullptr) {
+ TabletSharedPtr to_drop_tablet = _get_tablet_unlocked(tablet_id,
schema_hash);
+ if (to_drop_tablet == nullptr) {
LOG(WARNING) << "fail to drop tablet because it does not exist. "
<< "tablet_id=" << tablet_id << ", schema_hash=" <<
schema_hash;
return OLAP_SUCCESS;
}
- if (to_drop_base_tablet->is_memory()) {
- return _drop_tablet_directly_unlocked(tablet_id, schema_hash,
keep_files);
- }
- TabletSharedPtr to_drop_tablet = to_tablet(to_drop_base_tablet);
-
// Try to get schema change info, we can drop tablet directly if it is not
// in schema-change state.
AlterTabletTaskSharedPtr alter_task = to_drop_tablet->alter_task();
@@ -593,7 +562,7 @@ OLAPStatus TabletManager::drop_tablets_on_error_root_path(
continue;
} else {
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
- for (auto it = tablet_map[tablet_id].table_arr.begin();
+ for (list<TabletSharedPtr>::iterator it =
tablet_map[tablet_id].table_arr.begin();
it != tablet_map[tablet_id].table_arr.end();) {
if ((*it)->equal(tablet_id, schema_hash)) {
// We should first remove tablet from partition_map to
avoid iterator
@@ -610,13 +579,6 @@ OLAPStatus TabletManager::drop_tablets_on_error_root_path(
return res;
}
-BaseTabletSharedPtr TabletManager::get_base_tablet(TTabletId tablet_id,
SchemaHash schema_hash,
- bool include_deleted,
std::string* err) {
- RWMutex& tablet_map_lock = _get_tablet_map_lock(tablet_id);
- ReadLock rlock(&tablet_map_lock);
- return _get_base_tablet_unlocked(tablet_id, schema_hash, include_deleted,
err);
-}
-
TabletSharedPtr TabletManager::get_tablet(TTabletId tablet_id, SchemaHash
schema_hash,
bool include_deleted, string* err) {
RWMutex& tablet_map_lock = _get_tablet_map_lock(tablet_id);
@@ -626,24 +588,8 @@ TabletSharedPtr TabletManager::get_tablet(TTabletId
tablet_id, SchemaHash schema
TabletSharedPtr TabletManager::_get_tablet_unlocked(TTabletId tablet_id,
SchemaHash schema_hash,
bool include_deleted,
string* err) {
- BaseTabletSharedPtr ret = _get_base_tablet_unlocked(tablet_id,
schema_hash, include_deleted,
- err);
- if (ret == nullptr) {
- return TabletSharedPtr();
- }
- if (ret->is_memory()) {
- LOG(FATAL) << "_get_tablet_unlocked get MemTablet";
- return TabletSharedPtr();
- }
- return to_tablet(ret);
-}
-
-BaseTabletSharedPtr TabletManager::_get_base_tablet_unlocked(TTabletId
tablet_id,
- SchemaHash
schema_hash,
- bool
include_deleted,
- string* err) {
- BaseTabletSharedPtr tablet;
- tablet = _get_base_tablet_unlocked(tablet_id, schema_hash);
+ TabletSharedPtr tablet;
+ tablet = _get_tablet_unlocked(tablet_id, schema_hash);
if (tablet == nullptr && include_deleted) {
ReadLock rlock(&_shutdown_tablets_lock);
for (auto& deleted_tablet : _shutdown_tablets) {
@@ -741,12 +687,7 @@ TabletSharedPtr
TabletManager::find_best_tablet_to_compaction(CompactionType com
ReadLock tablet_map_rdlock(&_tablet_map_lock_array[i]);
tablet_map_t& tablet_map = _tablet_map_array[i];
for (tablet_map_t::value_type& table_ins : tablet_map){
- for (BaseTabletSharedPtr& base_tablet_ptr :
table_ins.second.table_arr) {
- if (base_tablet_ptr->is_memory()) {
- // TODO: mem_tablet doesn't do compaction yet
- continue;
- }
- TabletSharedPtr tablet_ptr = to_tablet(base_tablet_ptr);
+ for (TabletSharedPtr& tablet_ptr : table_ins.second.table_arr) {
AlterTabletTaskSharedPtr cur_alter_task =
tablet_ptr->alter_task();
if (cur_alter_task != nullptr
&& cur_alter_task->alter_state() != ALTER_FINISHED
@@ -830,8 +771,7 @@ TabletSharedPtr
TabletManager::find_best_tablet_to_compaction(CompactionType com
}
OLAPStatus TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId
tablet_id,
- TSchemaHash schema_hash, const
string& meta_binary,
- bool update_meta, bool force,
bool restore) {
+ TSchemaHash schema_hash, const string& meta_binary, bool update_meta,
bool force, bool restore) {
RWMutex& tablet_map_lock = _get_tablet_map_lock(tablet_id);
WriteLock wlock(&tablet_map_lock);
TabletMetaSharedPtr tablet_meta(new TabletMeta());
@@ -865,77 +805,37 @@ OLAPStatus TabletManager::load_tablet_from_meta(DataDir*
data_dir, TTabletId tab
tablet_meta->set_tablet_state(TABLET_RUNNING);
}
- if (tablet_meta->tablet_type() == TabletTypePB::TABLET_TYPE_DISK) {
- TabletSharedPtr tablet = Tablet::create_tablet_from_meta(tablet_meta,
data_dir);
- if (tablet == nullptr) {
- LOG(WARNING) << "fail to load tablet. tablet_id=" << tablet_id
- << ", schema_hash:" << schema_hash;
- return OLAP_ERR_TABLE_CREATE_FROM_HEADER_ERROR;
- }
-
- if (tablet_meta->tablet_state() == TABLET_SHUTDOWN) {
- LOG(INFO) << "fail to load tablet because it is to be deleted.
tablet_id=" << tablet_id
- << " schema_hash=" << schema_hash << ", path=" <<
data_dir->path();
- {
- WriteLock shutdown_tablets_wlock(&_shutdown_tablets_lock);
- _shutdown_tablets.push_back(tablet);
- }
- return OLAP_ERR_TABLE_ALREADY_DELETED_ERROR;
- }
- // NOTE: We do not check tablet's initial version here, because if BE
restarts when
- // one tablet is doing schema-change, we may meet empty tablet.
- if (tablet->max_version().first == -1 && tablet->tablet_state() ==
TABLET_RUNNING) {
- LOG(WARNING) << "fail to load tablet. it is in running state but
without delta. "
- << "tablet=" << tablet->full_name() << ", path=" <<
data_dir->path();
- // tablet state is invalid, drop tablet
- return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR;
- }
-
- RETURN_NOT_OK_LOG(tablet->init(), Substitute("tablet init failed.
tablet=$0",
- tablet->full_name()));
- RETURN_NOT_OK_LOG(_add_tablet_unlocked(tablet_id, schema_hash,
-
std::static_pointer_cast<BaseTablet>(tablet),
- update_meta, force),
- Substitute("fail to add tablet. tablet=$0",
tablet->full_name()));
-
- return OLAP_SUCCESS;
- } else {
- memory::MemTabletSharedPtr tablet =
memory::MemTablet::create_tablet_from_meta(
- tablet_meta, data_dir);
- if (tablet == nullptr) {
- LOG(WARNING) << "fail to load tablet. tablet_id=" << tablet_id
- << ", schema_hash:" << schema_hash;
- return OLAP_ERR_TABLE_CREATE_FROM_HEADER_ERROR;
- }
+ TabletSharedPtr tablet = Tablet::create_tablet_from_meta(tablet_meta,
data_dir);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "fail to load tablet. tablet_id=" << tablet_id
+ << ", schema_hash:" << schema_hash;
+ return OLAP_ERR_TABLE_CREATE_FROM_HEADER_ERROR;
+ }
- if (tablet_meta->tablet_state() == TABLET_SHUTDOWN) {
- LOG(INFO) << "fail to load tablet because it is to be deleted.
tablet_id=" << tablet_id
- << " schema_hash=" << schema_hash << ", path=" <<
data_dir->path();
- {
- WriteLock shutdown_tablets_wlock(&_shutdown_tablets_lock);
- _shutdown_tablets.push_back(tablet);
- }
- return OLAP_ERR_TABLE_ALREADY_DELETED_ERROR;
+ if (tablet_meta->tablet_state() == TABLET_SHUTDOWN) {
+ LOG(INFO) << "fail to load tablet because it is to be deleted.
tablet_id=" << tablet_id
+ << " schema_hash=" << schema_hash << ", path=" <<
data_dir->path();
+ {
+ WriteLock shutdown_tablets_wlock(&_shutdown_tablets_lock);
+ _shutdown_tablets.push_back(tablet);
}
- // NOTE: We do not check tablet's initial version here, because if BE
restarts when
- // one tablet is doing schema-change, we may meet empty tablet.
- if (tablet->max_version().first == -1 && tablet->tablet_state() ==
TABLET_RUNNING) {
- LOG(WARNING) << "fail to load tablet. it is in running state but
without delta. "
- << "tablet=" << tablet->full_name() << ", path=" <<
data_dir->path();
- // tablet state is invalid, drop tablet
- return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR;
- }
-
- RETURN_NOT_OK_LOG(tablet->init(), Substitute("tablet init failed.
tablet=$0",
- tablet->full_name()));
- RETURN_NOT_OK_LOG(_add_tablet_unlocked(tablet_id, schema_hash,
-
std::static_pointer_cast<BaseTablet>(tablet),
- update_meta, force),
- Substitute("fail to add tablet. tablet=$0",
tablet->full_name()));
-
- return OLAP_SUCCESS;
+ return OLAP_ERR_TABLE_ALREADY_DELETED_ERROR;
+ }
+ // NOTE: We do not check tablet's initial version here, because if BE
restarts when
+ // one tablet is doing schema-change, we may meet empty tablet.
+ if (tablet->max_version().first == -1 && tablet->tablet_state() ==
TABLET_RUNNING) {
+ LOG(WARNING) << "fail to load tablet. it is in running state but
without delta. "
+ << "tablet=" << tablet->full_name() << ", path=" <<
data_dir->path();
+ // tablet state is invalid, drop tablet
+ return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR;
}
+ RETURN_NOT_OK_LOG(tablet->init(), Substitute("tablet init failed.
tablet=$0",
+ tablet->full_name()));
+ RETURN_NOT_OK_LOG(_add_tablet_unlocked(tablet_id, schema_hash, tablet,
update_meta, force),
+ Substitute("fail to add tablet. tablet=$0",
tablet->full_name()));
+
+ return OLAP_SUCCESS;
}
OLAPStatus TabletManager::load_tablet_from_dir(DataDir* store, TTabletId
tablet_id,
@@ -1005,7 +905,7 @@ OLAPStatus TabletManager::report_tablet_info(TTabletInfo*
tablet_info) {
OLAPStatus res = OLAP_SUCCESS;
- auto tablet = get_base_tablet(tablet_info->tablet_id,
tablet_info->schema_hash);
+ TabletSharedPtr tablet = get_tablet(tablet_info->tablet_id,
tablet_info->schema_hash);
if (tablet == nullptr) {
LOG(WARNING) << "can't find tablet. " << " tablet=" <<
tablet_info->tablet_id
<< " schema_hash=" << tablet_info->schema_hash;
@@ -1038,7 +938,7 @@ OLAPStatus
TabletManager::report_all_tablets_info(std::map<TTabletId, TTablet>*
uint64_t tablet_id = item.first;
TTablet t_tablet;
- for (const auto& tablet_ptr : item.second.table_arr) {
+ for (TabletSharedPtr tablet_ptr : item.second.table_arr) {
TTabletInfo tablet_info;
tablet_ptr->build_tablet_report_info(&tablet_info);
@@ -1064,8 +964,7 @@ OLAPStatus
TabletManager::report_all_tablets_info(std::map<TTabletId, TTablet>*
OLAPStatus TabletManager::start_trash_sweep() {
{
std::vector<int64_t> tablets_to_clean;
- // we use this vector to save all tablet ptr for saving lock time.
- std::vector<TabletSharedPtr> all_tablets;
+ std::vector<TabletSharedPtr> all_tablets; // we use this vector to
save all tablet ptr for saving lock time.
for (int32 i = 0; i < _tablet_map_lock_shard_size; i++) {
tablet_map_t& tablet_map = _tablet_map_array[i];
{
@@ -1075,11 +974,8 @@ OLAPStatus TabletManager::start_trash_sweep() {
if (item.second.table_arr.empty()) {
tablets_to_clean.push_back(item.first);
}
- for (BaseTabletSharedPtr& base_tablet :
item.second.table_arr) {
- // TODO: support MemTablet
- if (!base_tablet->is_memory()) {
- all_tablets.push_back(to_tablet(base_tablet));
- }
+ for (TabletSharedPtr tablet : item.second.table_arr) {
+ all_tablets.push_back(tablet);
}
}
}
@@ -1207,7 +1103,7 @@ bool TabletManager::try_schema_change_lock(TTabletId
tablet_id) {
void TabletManager::update_root_path_info(std::map<string, DataDirInfo>*
path_map,
size_t* tablet_count) {
- DCHECK(tablet_count != nullptr);
+ DCHECK(tablet_count != 0);
*tablet_count = 0;
for (int32 i = 0; i < _tablet_map_lock_shard_size; i++) {
ReadLock rlock(&_tablet_map_lock_array[i]);
@@ -1242,7 +1138,7 @@ void TabletManager::do_tablet_meta_checkpoint(DataDir*
data_dir) {
for (int32 i = 0 ; i < _tablet_map_lock_shard_size; i++) {
ReadLock tablet_map_rdlock(&_tablet_map_lock_array[i]);
for (tablet_map_t::value_type& table_ins : _tablet_map_array[i]){
- for (BaseTabletSharedPtr& tablet_ptr :
table_ins.second.table_arr) {
+ for (TabletSharedPtr& tablet_ptr : table_ins.second.table_arr)
{
if (tablet_ptr->tablet_state() != TABLET_RUNNING) {
continue;
}
@@ -1252,10 +1148,7 @@ void TabletManager::do_tablet_meta_checkpoint(DataDir*
data_dir) {
|| !tablet_ptr->init_succeeded()) {
continue;
}
- // TODO: do MemTablet need checkpoint?
- if (!tablet_ptr->is_memory()) {
- related_tablets.push_back(to_tablet(tablet_ptr));
- }
+ related_tablets.push_back(tablet_ptr);
}
}
}
@@ -1277,7 +1170,7 @@ void TabletManager::_build_tablet_stat() {
TTabletStat stat;
stat.tablet_id = item.first;
- for (BaseTabletSharedPtr tablet : item.second.table_arr) {
+ for (TabletSharedPtr tablet : item.second.table_arr) {
// TODO(lingbin): if it is nullptr, why is it not deleted?
if (tablet == nullptr) {
continue;
@@ -1422,7 +1315,7 @@ OLAPStatus
TabletManager::_create_tablet_meta_unlocked(const TCreateTabletReq& r
OLAPStatus TabletManager::_drop_tablet_directly_unlocked(
TTabletId tablet_id, SchemaHash schema_hash, bool keep_files) {
- BaseTabletSharedPtr dropped_tablet = _get_base_tablet_unlocked(tablet_id,
schema_hash);
+ TabletSharedPtr dropped_tablet = _get_tablet_unlocked(tablet_id,
schema_hash);
if (dropped_tablet == nullptr) {
LOG(WARNING) << "fail to drop tablet because it does not exist. "
<< " tablet_id=" << tablet_id
@@ -1430,16 +1323,16 @@ OLAPStatus
TabletManager::_drop_tablet_directly_unlocked(
return OLAP_ERR_TABLE_NOT_FOUND;
}
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
- auto& candidate_tablets = tablet_map[tablet_id].table_arr;
- auto it = candidate_tablets.begin();
+ list<TabletSharedPtr>& candidate_tablets = tablet_map[tablet_id].table_arr;
+ list<TabletSharedPtr>::iterator it = candidate_tablets.begin();
while (it != candidate_tablets.end()) {
if (!(*it)->equal(tablet_id, schema_hash)) {
++it;
continue;
}
- auto tablet = *it;
- _remove_tablet_from_partition(*tablet);
+ TabletSharedPtr tablet = *it;
+ _remove_tablet_from_partition(*(*it));
it = candidate_tablets.erase(it);
if (!keep_files) {
// drop tablet will update tablet meta, should lock
@@ -1466,13 +1359,12 @@ OLAPStatus
TabletManager::_drop_tablet_directly_unlocked(
return OLAP_SUCCESS;
}
-BaseTabletSharedPtr TabletManager::_get_base_tablet_unlocked(TTabletId
tablet_id,
- SchemaHash
schema_hash) {
+TabletSharedPtr TabletManager::_get_tablet_unlocked(TTabletId tablet_id,
SchemaHash schema_hash) {
VLOG(3) << "begin to get tablet. tablet_id=" << tablet_id << ",
schema_hash=" << schema_hash;
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
tablet_map_t::iterator it = tablet_map.find(tablet_id);
if (it != tablet_map.end()) {
- for (BaseTabletSharedPtr tablet : it->second.table_arr) {
+ for (TabletSharedPtr tablet : it->second.table_arr) {
CHECK(tablet != nullptr) << "tablet is nullptr. tablet_id=" <<
tablet_id;
if (tablet->equal(tablet_id, schema_hash)) {
VLOG(3) << "get tablet success. tablet_id=" << tablet_id
@@ -1484,28 +1376,16 @@ BaseTabletSharedPtr
TabletManager::_get_base_tablet_unlocked(TTabletId tablet_id
VLOG(3) << "fail to get tablet. tablet_id=" << tablet_id << ",
schema_hash=" << schema_hash;
// Return nullptr tablet if fail
- return nullptr;
-}
-
-TabletSharedPtr TabletManager::_get_tablet_unlocked(TTabletId tablet_id,
SchemaHash schema_hash) {
- BaseTabletSharedPtr ret = _get_base_tablet_unlocked(tablet_id,
schema_hash);
- if (ret == nullptr) {
- return TabletSharedPtr();
- }
- if (ret->is_memory()) {
- LOG(FATAL) << "fail to get TabletSharedPtr from MemTabletSharedPtr.
tablet_id=" << tablet_id
- << ", schema_hash=" << schema_hash;
- return TabletSharedPtr();
- }
- return to_tablet(ret);
+ TabletSharedPtr tablet;
+ return tablet;
}
-void TabletManager::_add_tablet_to_partition(const BaseTablet& tablet) {
+void TabletManager::_add_tablet_to_partition(const Tablet& tablet) {
WriteLock wlock(&_partition_tablet_map_lock);
_partition_tablet_map[tablet.partition_id()].insert(tablet.get_tablet_info());
}
-void TabletManager::_remove_tablet_from_partition(const BaseTablet& tablet) {
+void TabletManager::_remove_tablet_from_partition(const Tablet& tablet) {
WriteLock wlock(&_partition_tablet_map_lock);
_partition_tablet_map[tablet.partition_id()].erase(tablet.get_tablet_info());
if (_partition_tablet_map[tablet.partition_id()].empty()) {
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 745d7d6..347fb10 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -71,9 +71,6 @@ public:
TabletSharedPtr find_best_tablet_to_compaction(CompactionType
compaction_type, DataDir* data_dir);
- BaseTabletSharedPtr get_base_tablet(TTabletId tablet_id, SchemaHash
schema_hash,
- bool include_deleted = false,
std::string* err = nullptr);
-
TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash,
bool include_deleted = false, std::string* err
= nullptr);
@@ -143,10 +140,10 @@ private:
// OLAP_ERR_TABLE_INSERT_DUPLICATION_ERROR, if find duplication
// OLAP_ERR_NOT_INITED, if not inited
OLAPStatus _add_tablet_unlocked(TTabletId tablet_id, SchemaHash
schema_hash,
- const BaseTabletSharedPtr& tablet, bool
update_meta, bool force);
+ const TabletSharedPtr& tablet, bool
update_meta, bool force);
OLAPStatus _add_tablet_to_map_unlocked(TTabletId tablet_id, SchemaHash
schema_hash,
- const BaseTabletSharedPtr& tablet,
bool update_meta,
+ const TabletSharedPtr& tablet, bool
update_meta,
bool keep_files, bool drop_old);
bool _check_tablet_id_exist_unlocked(TTabletId tablet_id);
@@ -159,19 +156,16 @@ private:
OLAPStatus _drop_tablet_unlocked(TTabletId tablet_id, SchemaHash
schema_hash, bool keep_files);
- BaseTabletSharedPtr _get_base_tablet_unlocked(TTabletId tablet_id,
SchemaHash schema_hash);
- BaseTabletSharedPtr _get_base_tablet_unlocked(TTabletId tablet_id,
SchemaHash schema_hash,
- bool include_deleted,
std::string* err);
TabletSharedPtr _get_tablet_unlocked(TTabletId tablet_id, SchemaHash
schema_hash);
TabletSharedPtr _get_tablet_unlocked(TTabletId tablet_id, SchemaHash
schema_hash,
bool include_deleted, std::string*
err);
- BaseTabletSharedPtr _internal_create_tablet_unlocked(const AlterTabletType
alter_type,
- const
TCreateTabletReq& request,
- const bool
is_schema_change,
- const Tablet*
base_tablet,
- const
std::vector<DataDir*>& data_dirs);
- BaseTabletSharedPtr _create_tablet_meta_and_dir_unlocked(const
TCreateTabletReq& request,
+ TabletSharedPtr _internal_create_tablet_unlocked(const AlterTabletType
alter_type,
+ const TCreateTabletReq&
request,
+ const bool
is_schema_change,
+ const Tablet* base_tablet,
+ const
std::vector<DataDir*>& data_dirs);
+ TabletSharedPtr _create_tablet_meta_and_dir_unlocked(const
TCreateTabletReq& request,
const bool
is_schema_change,
const Tablet*
base_tablet,
const
std::vector<DataDir*>& data_dirs);
@@ -183,9 +177,9 @@ private:
void _build_tablet_stat();
- void _add_tablet_to_partition(const BaseTablet& tablet);
+ void _add_tablet_to_partition(const Tablet& tablet);
- void _remove_tablet_from_partition(const BaseTablet& tablet);
+ void _remove_tablet_from_partition(const Tablet& tablet);
inline RWMutex& _get_tablet_map_lock(TTabletId tabletId);
@@ -199,7 +193,7 @@ private:
// The first element(i.e. tablet_arr[0]) is the base tablet. When we
add new tablet
// to tablet_arr, we will sort all the elements in create-time
ascending order,
// which will ensure the first one is base-tablet
- std::list<BaseTabletSharedPtr> table_arr;
+ std::list<TabletSharedPtr> table_arr;
};
// tablet_id -> TabletInstances
typedef std::unordered_map<int64_t, TableInstances> tablet_map_t;
@@ -215,7 +209,7 @@ private:
RWMutex _shutdown_tablets_lock;
// partition_id => tablet_info
std::map<int64_t, std::set<TabletInfo>> _partition_tablet_map;
- std::vector<BaseTabletSharedPtr> _shutdown_tablets;
+ std::vector<TabletSharedPtr> _shutdown_tablets;
std::mutex _tablet_stat_mutex;
// cache to save tablets' statistics, such as data-size and row-count
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index a4ef612..4b9a45f 100755
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -91,7 +91,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id,
tablet_meta_pb.set_tablet_state(PB_RUNNING);
*(tablet_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto();
tablet_meta_pb.set_tablet_type(tabletType ==
TTabletType::TABLET_TYPE_MEMORY ?
- TabletTypePB::TABLET_TYPE_MEMORY : TabletTypePB::TABLET_TYPE_DISK);
+ TabletTypePB::TABLET_TYPE_DISK : TabletTypePB::TABLET_TYPE_MEMORY);
TabletSchemaPB* schema = tablet_meta_pb.mutable_schema();
schema->set_num_short_key_columns(tablet_schema.short_key_column_count);
schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block);
diff --git a/be/test/olap/memory/mem_tablet_test.cpp
b/be/test/olap/memory/mem_tablet_test.cpp
index c0c6f9b..979f462 100644
--- a/be/test/olap/memory/mem_tablet_test.cpp
+++ b/be/test/olap/memory/mem_tablet_test.cpp
@@ -70,7 +70,7 @@ TEST(MemTablet, writescan) {
new TabletMeta(1, 1, 1, 1, 1, tschema,
static_cast<uint32_t>(sc->cid_size()),
col_idx_to_unique_id, TabletUid(1, 1),
TTabletType::TABLET_TYPE_MEMORY));
std::shared_ptr<MemTablet> tablet =
MemTablet::create_tablet_from_meta(tablet_meta, nullptr);
- ASSERT_EQ(tablet->init(), OLAP_SUCCESS);
+ ASSERT_TRUE(tablet->init().ok());
uint64_t cur_version = 0;
vector<TData> alldata(num_insert);
diff --git a/gensrc/thrift/MasterService.thrift
b/gensrc/thrift/MasterService.thrift
index 345b6ce..ded383b 100644
--- a/gensrc/thrift/MasterService.thrift
+++ b/gensrc/thrift/MasterService.thrift
@@ -38,7 +38,6 @@ struct TTabletInfo {
12: optional bool used
13: optional Types.TPartitionId partition_id
14: optional bool is_in_memory
- 15: optional AgentService.TTabletType tablet_type
}
struct TFinishTaskRequest {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
