This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-2.1-lakehouse in repository https://gitbox.apache.org/repos/asf/doris.git
commit a9a82a9c934adfccaf36dd6b13486d0deab9647b Author: zy-kkk <[email protected]> AuthorDate: Wed Jan 22 22:43:35 2025 +0800 [opt](jdbc scan) Add more jdbc scan profile items (#46460) ### What problem does this PR solve? Problem Summary: ``` - GetDataTime: 197.0us (Total time to read data) - CastTime: 2.0us (Convert data from JDBC memory format to Doris BE C++ memory format) - FillBlockTime: 6.0us (Put data into BE Block) - ReadAndFillVectorTableTime: 129.0us (Time to read data and convert to vectorTable) - HasNextTime: 35.0us (The time to obtain each batch of data) - JniSetupTime: 0ns (Start JNI time) - PrepareParamsTime: 24.0us (Preparation parameter time) ``` --- be/src/vec/exec/scan/new_jdbc_scanner.cpp | 14 ++++++-- be/src/vec/exec/scan/new_jdbc_scanner.h | 6 +++- be/src/vec/exec/vjdbc_connector.cpp | 57 +++++++++++++++++++++++-------- be/src/vec/exec/vjdbc_connector.h | 6 +++- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.cpp b/be/src/vec/exec/scan/new_jdbc_scanner.cpp index 60e86f06521..830b4e51383 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.cpp +++ b/be/src/vec/exec/scan/new_jdbc_scanner.cpp @@ -185,8 +185,13 @@ void NewJdbcScanner::_init_profile(const std::shared_ptr<RuntimeProfile>& profil _init_connector_timer = ADD_TIMER(profile, "InitConnectorTime"); _check_type_timer = ADD_TIMER(profile, "CheckTypeTime"); _get_data_timer = ADD_TIMER(profile, "GetDataTime"); - _get_block_address_timer = ADD_CHILD_TIMER(profile, "GetBlockAddressTime", "GetDataTime"); + _read_and_fill_vector_table_timer = + ADD_CHILD_TIMER(profile, "ReadAndFillVectorTableTime", "GetDataTime"); + _jni_setup_timer = ADD_CHILD_TIMER(profile, "JniSetupTime", "GetDataTime"); + _has_next_timer = ADD_CHILD_TIMER(profile, "HasNextTime", "GetDataTime"); + _prepare_params_timer = ADD_CHILD_TIMER(profile, "PrepareParamsTime", "GetDataTime"); _fill_block_timer = ADD_CHILD_TIMER(profile, "FillBlockTime", "GetDataTime"); + _cast_timer = ADD_CHILD_TIMER(profile, "CastTime", "GetDataTime"); _execte_read_timer = ADD_TIMER(profile, "ExecteReadTime"); _connector_close_timer = ADD_TIMER(profile, "ConnectorCloseTime"); } @@ -197,8 +202,13 @@ void NewJdbcScanner::_update_profile() { COUNTER_UPDATE(_init_connector_timer, jdbc_statistic._init_connector_timer); COUNTER_UPDATE(_check_type_timer, jdbc_statistic._check_type_timer); COUNTER_UPDATE(_get_data_timer, jdbc_statistic._get_data_timer); - COUNTER_UPDATE(_get_block_address_timer, jdbc_statistic._get_block_address_timer); + COUNTER_UPDATE(_jni_setup_timer, jdbc_statistic._jni_setup_timer); + COUNTER_UPDATE(_has_next_timer, jdbc_statistic._has_next_timer); + COUNTER_UPDATE(_prepare_params_timer, jdbc_statistic._prepare_params_timer); + COUNTER_UPDATE(_read_and_fill_vector_table_timer, + jdbc_statistic._read_and_fill_vector_table_timer); COUNTER_UPDATE(_fill_block_timer, jdbc_statistic._fill_block_timer); + COUNTER_UPDATE(_cast_timer, jdbc_statistic._cast_timer); COUNTER_UPDATE(_execte_read_timer, jdbc_statistic._execte_read_timer); COUNTER_UPDATE(_connector_close_timer, jdbc_statistic._connector_close_timer); } diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.h b/be/src/vec/exec/scan/new_jdbc_scanner.h index 3d60c7a6568..bda6d6b1dc8 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.h +++ b/be/src/vec/exec/scan/new_jdbc_scanner.h @@ -63,7 +63,11 @@ protected: RuntimeProfile::Counter* _load_jar_timer = nullptr; RuntimeProfile::Counter* _init_connector_timer = nullptr; RuntimeProfile::Counter* _get_data_timer = nullptr; - RuntimeProfile::Counter* _get_block_address_timer = nullptr; + RuntimeProfile::Counter* _jni_setup_timer = nullptr; + RuntimeProfile::Counter* _has_next_timer = nullptr; + RuntimeProfile::Counter* _prepare_params_timer = nullptr; + RuntimeProfile::Counter* _cast_timer = nullptr; + RuntimeProfile::Counter* _read_and_fill_vector_table_timer = nullptr; RuntimeProfile::Counter* _fill_block_timer = nullptr; RuntimeProfile::Counter* _check_type_timer = nullptr; RuntimeProfile::Counter* _execte_read_timer = nullptr; diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 0fa33bfaad9..14ce4ab5aca 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -234,14 +234,25 @@ Status JdbcConnector::query() { } Status JdbcConnector::get_next(bool* eos, Block* block, int batch_size) { + SCOPED_RAW_TIMER(&_jdbc_statistic._get_data_timer); // Timer for the entire method + if (!_is_open) { return Status::InternalError("get_next before open of jdbc connector."); } - SCOPED_RAW_TIMER(&_jdbc_statistic._get_data_timer); + JNIEnv* env = nullptr; - RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); - jboolean has_next = - env->CallNonvirtualBooleanMethod(_executor_obj, _executor_clazz, _executor_has_next_id); + { + SCOPED_RAW_TIMER(&_jdbc_statistic._jni_setup_timer); // Timer for setting up JNI environment + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + } // _jni_setup_timer stops when going out of this scope + + jboolean has_next = JNI_FALSE; + { + SCOPED_RAW_TIMER(&_jdbc_statistic._has_next_timer); // Timer for hasNext check + has_next = env->CallNonvirtualBooleanMethod(_executor_obj, _executor_clazz, + _executor_has_next_id); + } // _has_next_timer stops here + if (has_next != JNI_TRUE) { *eos = true; return Status::OK(); @@ -252,10 +263,21 @@ Status JdbcConnector::get_next(bool* eos, Block* block, int batch_size) { auto column_size = _tuple_desc->slots().size(); auto slots = _tuple_desc->slots(); - jobject map = _get_reader_params(block, env, column_size); - SCOPED_RAW_TIMER(&_jdbc_statistic._get_block_address_timer); - long address = - env->CallLongMethod(_executor_obj, _executor_get_block_address_id, batch_size, map); + jobject map; + { + SCOPED_RAW_TIMER(&_jdbc_statistic._prepare_params_timer); // Timer for preparing params + map = _get_reader_params(block, env, column_size); + } // _prepare_params_timer stops here + + long address = 0; + { + SCOPED_RAW_TIMER( + &_jdbc_statistic + ._read_and_fill_vector_table_timer); // Timer for getBlockAddress call + address = + env->CallLongMethod(_executor_obj, _executor_get_block_address_id, batch_size, map); + } // _get_block_address_timer stops here + RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); env->DeleteLocalRef(map); @@ -263,17 +285,22 @@ Status JdbcConnector::get_next(bool* eos, Block* block, int batch_size) { for (size_t i = 0; i < column_size; ++i) { all_columns.push_back(i); } - SCOPED_RAW_TIMER(&_jdbc_statistic._fill_block_timer); - Status fill_block_status = JniConnector::fill_block(block, all_columns, address); + + Status fill_block_status; + { + SCOPED_RAW_TIMER(&_jdbc_statistic._fill_block_timer); // Timer for fill_block + fill_block_status = JniConnector::fill_block(block, all_columns, address); + } // _fill_block_timer stops here + if (!fill_block_status) { return fill_block_status; } - Status cast_status = _cast_string_to_special(block, env, column_size); - - if (!cast_status) { - return cast_status; - } + Status cast_status; + { + SCOPED_RAW_TIMER(&_jdbc_statistic._cast_timer); // Timer for casting process + cast_status = _cast_string_to_special(block, env, column_size); + } // _cast_timer stops here return JniUtil::GetJniExceptionMsg(env); } diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h index 954b0abfa78..941ce4331a5 100644 --- a/be/src/vec/exec/vjdbc_connector.h +++ b/be/src/vec/exec/vjdbc_connector.h @@ -71,8 +71,12 @@ public: int64_t _load_jar_timer = 0; int64_t _init_connector_timer = 0; int64_t _get_data_timer = 0; - int64_t _get_block_address_timer = 0; + int64_t _read_and_fill_vector_table_timer = 0; + int64_t _jni_setup_timer = 0; + int64_t _has_next_timer = 0; + int64_t _prepare_params_timer = 0; int64_t _fill_block_timer = 0; + int64_t _cast_timer = 0; int64_t _check_type_timer = 0; int64_t _execte_read_timer = 0; int64_t _connector_close_timer = 0; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
