This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0523860877 [Enhancement](streamload) print profile for streamload
(#18015)
0523860877 is described below
commit 0523860877a95c728a6de6f35605436a74be88a8
Author: gitccl <[email protected]>
AuthorDate: Fri Mar 24 20:17:33 2023 +0800
[Enhancement](streamload) print profile for streamload (#18015)
When both enable_profile and enable_stream_load_profile_log is true, stream
load profile is printed to the log
---
be/src/common/config.h | 2 ++
be/src/http/action/stream_load.cpp | 8 ++++++++
docs/en/docs/admin-manual/config/be-config.md | 7 +++++++
docs/zh-CN/docs/admin-manual/config/be-config.md | 7 +++++++
.../src/main/java/org/apache/doris/planner/StreamLoadPlanner.java | 2 ++
5 files changed, 26 insertions(+)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2b279432d3..7e3a5d9431 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -418,6 +418,8 @@ CONF_mInt32(stream_load_record_batch_size, "50");
CONF_Int32(stream_load_record_expire_time_secs, "28800");
// time interval to clean expired stream load records
CONF_mInt64(clean_stream_load_record_interval_secs, "1800");
+// Whether to enable stream load profile to be printed to the log, the default
is false.
+CONF_mBool(enable_stream_load_profile_log, "false");
// OlapTableSink sender's send interval, should be less than the real response
time of a tablet writer rpc.
// You may need to lower the speed when the sink receiver bes are too busy.
diff --git a/be/src/http/action/stream_load.cpp
b/be/src/http/action/stream_load.cpp
index c65e554a8b..e69d22fc45 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -28,6 +28,7 @@
#include <rapidjson/prettywriter.h>
#include <thrift/protocol/TDebugProtocol.h>
+#include "common/config.h"
#include "common/consts.h"
#include "common/logging.h"
#include "common/utils.h"
@@ -599,6 +600,13 @@ Status StreamLoadAction::_process_put(HttpRequest*
http_req,
LOG(WARNING) << "plan streaming load failed. errmsg=" << plan_status
<< ctx->brief();
return plan_status;
}
+
+ auto& query_options = ctx->put_result.params.query_options;
+ if (query_options.__isset.is_report_success &&
query_options.is_report_success &&
+ !config::enable_stream_load_profile_log) {
+ query_options.is_report_success = false;
+ }
+
VLOG_NOTICE << "params is " <<
apache::thrift::ThriftDebugString(ctx->put_result.params);
// if we not use streaming, we must download total content before we begin
// to process this load
diff --git a/docs/en/docs/admin-manual/config/be-config.md
b/docs/en/docs/admin-manual/config/be-config.md
index 62091324ac..62904d50f8 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -769,6 +769,13 @@ Metrics:
{"filtered_rows":0,"input_row_num":3346807,"input_rowsets_count":42,"in
* Default value: 100
* Dynamically modifiable: Yes
+#### `enable_stream_load_profile_log`
+
+* Type: bool
+* Description: Whether to enable stream load profile to be printed to the log.
+* Default value: false
+* Dynamically modifiable: Yes
+
### Thread
#### `delete_worker_count`
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md
b/docs/zh-CN/docs/admin-manual/config/be-config.md
index ac3f47043d..f91c63ec8b 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -783,6 +783,13 @@ Metrics:
{"filtered_rows":0,"input_row_num":3346807,"input_rowsets_count":42,"in
* 默认值： 100
* 可动态修改：是
+#### `enable_stream_load_profile_log`
+
+* 类型：bool
+* 描述：是否将 stream load profile 打印到日志。
+* 默认值： false
+* 可动态修改：是
+
### 线程
#### `delete_worker_count`
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
index c3eec8e23c..16b45d683c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
@@ -47,6 +47,7 @@ import org.apache.doris.load.BrokerFileGroup;
import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.load.routineload.RoutineLoadJob;
import org.apache.doris.planner.external.ExternalFileScanNode;
+import org.apache.doris.qe.VariableMgr;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.task.LoadTaskInfo;
import org.apache.doris.thrift.PaloInternalServiceVersion;
@@ -275,6 +276,7 @@ public class StreamLoadPlanner {
queryOptions.setEnableVectorizedEngine(Config.enable_vectorized_load);
queryOptions.setEnablePipelineEngine(Config.enable_pipeline_load);
queryOptions.setBeExecVersion(Config.be_exec_version);
+
queryOptions.setIsReportSuccess(VariableMgr.newSessionVariable().enableProfile());
params.setQueryOptions(queryOptions);
TQueryGlobals queryGlobals = new TQueryGlobals();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
