This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
new 005864e458 [Feature](scan) add num_free_block_in_scan in session
variables (#12128)
005864e458 is described below
commit 005864e4584a1e1cf00c866b623bcc0418eb3f22
Author: HappenLee <[email protected]>
AuthorDate: Mon Aug 29 00:45:15 2022 +0800
[Feature](scan) add num_free_block_in_scan in session variables (#12128)
Only support in dev-1.1.2，to reduce mem use by free blocks in scan node
---
be/src/runtime/runtime_state.h | 1 +
be/src/vec/exec/volap_scan_node.cpp | 3 +--
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java | 7 +++++++
gensrc/thrift/PaloInternalService.thrift | 3 ++-
4 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index a16b49e625..b9edc56ea0 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -102,6 +102,7 @@ public:
const DescriptorTbl& desc_tbl() const { return *_desc_tbl; }
void set_desc_tbl(DescriptorTbl* desc_tbl) { _desc_tbl = desc_tbl; }
int batch_size() const { return _query_options.batch_size; }
+ int num_free_block_in_scan() const { return
_query_options.num_free_block_in_scan; }
bool abort_on_error() const { return _query_options.abort_on_error; }
bool abort_on_default_limit_exceeded() const {
return _query_options.abort_on_default_limit_exceeded;
diff --git a/be/src/vec/exec/volap_scan_node.cpp
b/be/src/vec/exec/volap_scan_node.cpp
index 1ec25dce91..d535c48d61 100644
--- a/be/src/vec/exec/volap_scan_node.cpp
+++ b/be/src/vec/exec/volap_scan_node.cpp
@@ -71,8 +71,7 @@ void VOlapScanNode::transfer_thread(RuntimeState* state) {
auto block_per_scanner = (doris_scanner_row_num + (_block_size - 1)) /
_block_size;
auto pre_block_count =
std::min(_volap_scanners.size(),
-
static_cast<size_t>(config::doris_scanner_thread_pool_thread_num)) *
- block_per_scanner;
+
static_cast<size_t>(_runtime_state->num_free_block_in_scan())) *
block_per_scanner;
for (int i = 0; i < pre_block_count; ++i) {
auto block = new Block(_tuple_desc->slots(), _block_size);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 204ce8dbaa..46017bd24c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -189,6 +189,8 @@ public class SessionVariable implements Serializable,
Writable {
static final String SESSION_CONTEXT = "session_context";
+ public static final String NUM_FREE_BLOCK_IN_SCAN =
"num_free_block_in_scan";
+
// session origin value
public Map<Field, String> sessionOriginValue = new HashMap<Field,
String>();
// check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -462,6 +464,10 @@ public class SessionVariable implements Serializable,
Writable {
*/
@VariableMgr.VarAttr(name = SESSION_CONTEXT, needForward = true)
public String sessionContext = "";
+
+ @VariableMgr.VarAttr(name = NUM_FREE_BLOCK_IN_SCAN)
+ public int numFreeBlockInScan = 12;
+
public String getBlockEncryptionMode() {
return blockEncryptionMode;
@@ -950,6 +956,7 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setBatchSize(batchSize);
tResult.setDisableStreamPreaggregations(disableStreamPreaggregations);
tResult.setLoadMemLimit(loadMemLimit);
+ tResult.setNumFreeBlockInScan(numFreeBlockInScan);
if (maxScanKeyNum > -1) {
tResult.setMaxScanKeyNum(maxScanKeyNum);
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index aa1576b95a..689cf07919 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -163,8 +163,9 @@ struct TQueryOptions {
// trim tailing spaces while querying external table and stream load
44: optional bool trim_tailing_spaces_for_external_table_query = false
+
+ 47: optional i32 num_free_block_in_scan
}
-
// A scan range plus the parameters needed to execute that scan.
struct TScanRangeParams {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
