This is an automated email from the ASF dual-hosted git repository.
jacktengg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d9a53860d2c [refactor](fe) Remove FE session variables sent to BE but
unused by BE (#64285)
d9a53860d2c is described below
commit d9a53860d2c22575fe8e351e3ab954991ab1f154
Author: TengJianPing <[email protected]>
AuthorDate: Thu Jun 11 22:32:36 2026 +0800
[refactor](fe) Remove FE session variables sent to BE but unused by BE
(#64285)
Problem Summary:
Several FE session variables were serialized into TQueryOptions in
SessionVariable.toThrift() and shipped to BE on every query, but BE
never reads the corresponding fields. This is dead RPC payload and dead
FE configuration surface.
After auditing every field written by SessionVariable.toThrift() against
all BE/cloud usage, the following 7 TQueryOptions fields are set by FE
but never consumed by BE:
- enable_common_expr_pushdown
- enable_common_expr_pushdown_for_inverted_index (already
thrift-deprecated)
- enable_phrase_query_sequential_opt
- enable_rust_lance_reader (BE selects the Lance reader by file format,
not by this option)
- shuffled_agg_ids
- resource_limit (TResourceLimit.cpu_limit)
- skip_bad_tablet
For the first five, the backing session variable is also unused anywhere
else in FE, so the variable is fully removed (field, @VarAttr, name
constant, getter/setter, fuzzy assignment and the toThrift plumbing). To
preserve compatibility with existing SET/JDBC init statements, the
removed variable names are added to
VariableMgr.REMOVED_SESSION_VAR_NAMES, which makes them silently no-op
instead of raising ERR_UNKNOWN_SYSTEM_VARIABLE.
cpuResourceLimit and skipBadTablet are still used by other FE logic
(user properties / statistics, and OlapScanNode / CacheAnalyzer
respectively), so only their dead TQueryOptions plumbing is removed; the
variables themselves are kept. The thrift definitions are left untouched
for BE compatibility.
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/runtime/runtime_state.h | 4 -
.../java/org/apache/doris/qe/SessionVariable.java | 99 ----------------------
.../main/java/org/apache/doris/qe/VariableMgr.java | 8 +-
.../org/apache/doris/qe/SessionVariablesTest.java | 12 ---
gensrc/thrift/PaloInternalService.thrift | 6 +-
5 files changed, 10 insertions(+), 119 deletions(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 619b842eef7..ee1ab4cf882 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -213,7 +213,6 @@ public:
TimezoneUtils::find_cctz_time_zone(_timezone, _timezone_obj);
}
const std::string& lc_time_names() const { return _lc_time_names; }
- const std::string& user() const { return _user; }
const TUniqueId& query_id() const { return _query_id; }
const TUniqueId& fragment_instance_id() const { return
_fragment_instance_id; }
// should only be called in pipeline engine
@@ -893,9 +892,6 @@ private:
// _error_log[_unreported_error_idx+] has been not reported to the
coordinator.
int _unreported_error_idx;
- // Username of user that is executing the query to which this RuntimeState
belongs.
- std::string _user;
-
//Query-global timestamp_ms
int64_t _timestamp_ms;
int32_t _nano_seconds;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a36a3c7e79f..f537912c09d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -42,7 +42,6 @@ import org.apache.doris.qe.VarAttrDef.VarAttr;
import org.apache.doris.thrift.TGroupCommitMode;
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
import org.apache.doris.thrift.TQueryOptions;
-import org.apache.doris.thrift.TResourceLimit;
import org.apache.doris.thrift.TRuntimeFilterType;
import org.apache.doris.thrift.TSerdeDialect;
@@ -317,8 +316,6 @@ public class SessionVariable implements Serializable,
Writable {
public static final long DEFAULT_INSERT_VISIBLE_TIMEOUT_MS = 60_000;
- public static final String ENABLE_VECTORIZED_ENGINE =
"enable_vectorized_engine";
-
// If user set a very small value, use this value instead.
public static final long MIN_INSERT_VISIBLE_TIMEOUT_MS = 1000;
@@ -446,16 +443,12 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_SINGLE_REPLICA_INSERT =
"enable_single_replica_insert";
- public static final String SHUFFLED_AGG_NODE_IDS = "shuffled_agg_node_ids";
-
public static final String ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES =
"enable_fast_analyze_into_values";
public static final String ENABLE_FUNCTION_PUSHDOWN =
"enable_function_pushdown";
public static final String ENABLE_EXT_FUNC_PRED_PUSHDOWN =
"enable_ext_func_pred_pushdown";
- public static final String ENABLE_COMMON_EXPR_PUSHDOWN =
"enable_common_expr_pushdown";
-
public static final String ENABLE_SEGMENT_LIMIT_PUSHDOWN =
"enable_segment_limit_pushdown";
public static final String FRAGMENT_TRANSMISSION_COMPRESSION_CODEC =
"fragment_transmission_compression_codec";
@@ -507,9 +500,6 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_INVERTED_INDEX_QUERY =
"enable_inverted_index_query";
- public static final String ENABLE_COMMON_EXPR_PUSHDOWN_FOR_INVERTED_INDEX
- = "enable_common_expr_pushdown_for_inverted_index";
-
public static final String ENABLE_PUSHDOWN_COUNT_ON_INDEX =
"enable_count_on_index_pushdown";
public static final String ENABLE_NO_NEED_READ_DATA_OPT =
"enable_no_need_read_data_opt";
@@ -773,8 +763,6 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_PAIMON_CPP_READER =
"enable_paimon_cpp_reader";
- public static final String ENABLE_RUST_LANCE_READER =
"enable_rust_lance_reader";
-
public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE =
"enable_count_push_down_for_external_table";
public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE =
"fetch_all_fe_for_system_table";
@@ -856,8 +844,6 @@ public class SessionVariable implements Serializable,
Writable {
public static final String MINIMUM_OPERATOR_MEMORY_REQUIRED_KB =
"minimum_operator_memory_required_kb";
- public static final String ENABLE_PHRASE_QUERY_SEQUENYIAL_OPT =
"enable_phrase_query_sequential_opt";
-
public static final String ENABLE_COOLDOWN_REPLICA_AFFINITY =
"enable_cooldown_replica_affinity";
@@ -1566,9 +1552,6 @@ public class SessionVariable implements Serializable,
Writable {
@VarAttrDef.VarAttr(name = ENABLE_STRICT_CONSISTENCY_DML, needForward =
true)
public boolean enableStrictConsistencyDml = true;
- @VarAttrDef.VarAttr(name = ENABLE_VECTORIZED_ENGINE, varType =
VariableAnnotation.REMOVED)
- public boolean enableVectorizedEngine = true;
-
@VarAttrDef.VarAttr(name = ENABLE_PIPELINE_ENGINE, fuzzy = false,
needForward = true,
varType = VariableAnnotation.REMOVED, setter =
"setEnablePipelineEngine")
private boolean enablePipelineEngine = true;
@@ -2084,10 +2067,6 @@ public class SessionVariable implements Serializable,
Writable {
needForward = true, varType = VariableAnnotation.EXPERIMENTAL)
public boolean enableSingleReplicaInsert = false;
- @VarAttrDef.VarAttr(name = SHUFFLED_AGG_NODE_IDS,
- needForward = true, varType = VariableAnnotation.EXPERIMENTAL)
- public String shuffledAggNodeIds = "";
-
@VarAttrDef.VarAttr(
name = ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES, fuzzy = true,
description = {
@@ -2109,10 +2088,6 @@ public class SessionVariable implements Serializable,
Writable {
@VarAttrDef.VarAttr(name = FORBID_UNKNOWN_COLUMN_STATS)
public boolean forbidUnknownColStats = false;
- // Legacy session variable. BE treats common expr pushdown as enabled in
this branch.
- @VarAttrDef.VarAttr(name = ENABLE_COMMON_EXPR_PUSHDOWN, fuzzy = true)
- public boolean enableCommonExprPushdown = true;
-
@VarAttrDef.VarAttr(name = ENABLE_SEGMENT_LIMIT_PUSHDOWN, fuzzy = true,
needForward = true,
description = {"是否启用 SegmentIterator 层 LIMIT 下推。",
"Set whether to push down LIMIT into SegmentIterator."})
@@ -2272,11 +2247,6 @@ public class SessionVariable implements Serializable,
Writable {
"是否启用 inverted index query。", "Set whether to use inverted index
query."})
public boolean enableInvertedIndexQuery = true;
- // Whether enable query expr with inverted index.
- @VarAttrDef.VarAttr(name = ENABLE_COMMON_EXPR_PUSHDOWN_FOR_INVERTED_INDEX,
fuzzy = true, needForward = true,
- description = {"是否启用表达式上使用 inverted index。", "Set whether to use
inverted index query for expr."})
- public boolean enableCommonExpPushDownForInvertedIndex = true;
-
// Whether enable pushdown count agg to scan node when using inverted
index match.
@VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_COUNT_ON_INDEX, needForward =
true, description = {
"是否启用 count_on_index pushdown。", "Set whether to pushdown
count_on_index."})
@@ -2886,12 +2856,6 @@ public class SessionVariable implements Serializable,
Writable {
description = {"Paimon 非原生文件读取使用 paimon-cpp", "Use paimon-cpp for
non-native Paimon reads"})
private boolean enablePaimonCppReader = false;
- @VarAttrDef.VarAttr(name = ENABLE_RUST_LANCE_READER,
- fuzzy = true,
- description = {"使用 Rust Lance 读取器读取 Lance 格式数据",
- "Use Rust-based Lance reader for Lance format data"})
- private boolean enableRustLanceReader = false;
-
@VarAttrDef.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE,
fuzzy = true,
description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown
optimization for external table"})
@@ -3098,36 +3062,6 @@ public class SessionVariable implements Serializable,
Writable {
return ids;
}
- public List<Integer> getShuffledAggNodeIds() {
- List<Integer> ids = Lists.newLinkedList();
- if (shuffledAggNodeIds.isEmpty()) {
- return ImmutableList.of();
- }
- for (String v : shuffledAggNodeIds.split(",[\\s]*")) {
- int res = -1;
- if (!v.isEmpty()) {
- boolean isNumber = true;
- for (int i = 0; i < v.length(); ++i) {
- char c = v.charAt(i);
- if (c < '0' || c > '9') {
- isNumber = false;
- break;
- }
- }
- if (isNumber) {
- try {
- res = Integer.parseInt(v);
- } catch (Throwable t) {
- // ignore
- }
- }
-
- }
- ids.add(res);
- }
- return ids;
- }
-
public void setIgnoreRuntimeFilterIds(String ignoreRuntimeFilterIds) {
this.ignoreRuntimeFilterIds = ignoreRuntimeFilterIds;
}
@@ -3423,12 +3357,6 @@ public class SessionVariable implements Serializable,
Writable {
})
public int adaptivePipelineTaskSerialReadOnLimit = 10000;
- @VarAttrDef.VarAttr(name = ENABLE_PHRASE_QUERY_SEQUENYIAL_OPT, needForward
= true, description = {
- "开启顺序短语查询对连词的优化",
- "enable optimization for conjunctions in sequential phrase queries"
- })
- public boolean enablePhraseQuerySequentialOpt = true;
-
@VarAttrDef.VarAttr(name = "enable_adjust_conjunct_order_by_cost",
needForward = true)
public boolean enableAdjustConjunctOrderByCost = true;
@@ -3706,14 +3634,11 @@ public class SessionVariable implements Serializable,
Writable {
this.enableConditionCache = Config.pull_request_id % 2 == 0;
this.parallelPipelineTaskNum = random.nextInt(8);
this.parallelPrepareThreshold = random.nextInt(32) + 1;
- this.enableCommonExprPushdown = random.nextBoolean();
// enable fuzzy after we clean all case of
- //
enable_common_expr_pushdown/enable_common_exp_pushdown_for_inverted_index
// this.enableSegmentLimitPushdown = random.nextBoolean();
this.enableLocalExchange = random.nextBoolean();
this.enableSharedExchangeSinkBuffer = random.nextBoolean();
this.useSerialExchange = random.nextBoolean();
- this.enableCommonExpPushDownForInvertedIndex = random.nextBoolean();
this.disableStreamPreaggregations = random.nextBoolean();
this.enableStreamingAggHashJoinForcePassthrough = random.nextBoolean();
this.enableLocalExchangeBeforeAgg = random.nextBoolean();
@@ -4077,10 +4002,6 @@ public class SessionVariable implements Serializable,
Writable {
return transactionIsolation;
}
- public String getTxIsolation() {
- return txIsolation;
- }
-
public String getCharsetClient() {
return charsetClient;
}
@@ -5325,14 +5246,6 @@ public class SessionVariable implements Serializable,
Writable {
this.enableInvertedIndexQuery = enableInvertedIndexQuery;
}
- public boolean isEnableCommonExprPushdownForInvertedIndex() {
- return enableCommonExpPushDownForInvertedIndex;
- }
-
- public void setEnableCommonExprPushdownForInvertedIndex(boolean
enableCommonExpPushDownForInvertedIndex) {
- this.enableCommonExpPushDownForInvertedIndex =
enableCommonExpPushDownForInvertedIndex;
- }
-
public boolean isEnablePushDownCountOnIndex() {
return enablePushDownCountOnIndex;
}
@@ -5433,7 +5346,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setQueryTimeout(queryTimeoutS);
tResult.setEnableProfile(enableProfile);
tResult.setRpcVerboseProfileMaxInstanceCount(rpcVerboseProfileMaxInstanceCount);
- tResult.setShuffledAggIds(getShuffledAggNodeIds());
if (enableProfile) {
// If enable profile == true, then also set report success to true
// be need report success to start report thread. But it is very
tricky
@@ -5482,14 +5394,7 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setRuntimeFilterWaitInfinitely(runtimeFilterWaitInfinitely);
tResult.setEnableFuzzyBlockableTask(enableFuzzyBlockableTask);
- if (cpuResourceLimit > 0) {
- TResourceLimit resourceLimit = new TResourceLimit();
- resourceLimit.setCpuLimit(cpuResourceLimit);
- tResult.setResourceLimit(resourceLimit);
- }
-
tResult.setEnableFunctionPushdown(enableFunctionPushdown);
- tResult.setEnableCommonExprPushdown(enableCommonExprPushdown);
tResult.setEnableSegmentLimitPushdown(enableSegmentLimitPushdown);
tResult.setCheckOverflowForDecimal(checkOverflowForDecimal);
tResult.setFragmentTransmissionCompressionCodec(fragmentTransmissionCompressionCodec.trim().toLowerCase());
@@ -5511,7 +5416,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setFileCacheBasePath(fileCacheBasePath);
tResult.setEnableInvertedIndexQuery(enableInvertedIndexQuery);
-
tResult.setEnableCommonExprPushdownForInvertedIndex(enableCommonExpPushDownForInvertedIndex);
tResult.setEnableNoNeedReadDataOpt(enableNoNeedReadDataOpt);
if (dryRunQuery) {
@@ -5527,7 +5431,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax);
tResult.setEnablePaimonCppReader(enablePaimonCppReader);
tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds);
- tResult.setEnableRustLanceReader(enableRustLanceReader);
tResult.setEmbedMaxBatchSize(embedMaxBatchSize);
tResult.setAiContextWindowSize(aiContextWindowSize);
tResult.setCheckOrcInitSargsSuccess(checkOrcInitSargsSuccess);
@@ -5553,7 +5456,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setParallelScanMaxScannersCount(parallelScanMaxScannersCount);
tResult.setParallelScanMinRowsPerScanner(parallelScanMinRowsPerScanner);
tResult.setOptimizeIndexScanParallelism(optimizeIndexScanParallelism);
- tResult.setSkipBadTablet(skipBadTablet);
tResult.setDisableFileCache(disableFileCache);
tResult.setEnablePreferCachedRowset(getEnablePreferCachedRowset());
@@ -5604,7 +5506,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setEnableAdaptivePipelineTaskSerialReadOnLimit(enableAdaptivePipelineTaskSerialReadOnLimit);
tResult.setAdaptivePipelineTaskSerialReadOnLimit(adaptivePipelineTaskSerialReadOnLimit);
tResult.setInListValueCountThreshold(inListValueCountThreshold);
-
tResult.setEnablePhraseQuerySequentialOpt(enablePhraseQuerySequentialOpt);
tResult.setEnableAutoCreateWhenOverwrite(enableAutoCreateWhenOverwrite);
tResult.setOrcTinyStripeThresholdBytes(orcTinyStripeThresholdBytes);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
index dc27b11de42..5f6a60a2d16 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -148,7 +148,13 @@ public class VariableMgr {
"enable_eliminate_sort_node",
"drop_table_if_ctas_failed",
"trace_nereids",
- "enable_sync_mv_cost_based_rewrite");
+ "enable_sync_mv_cost_based_rewrite",
+ "enable_vectorized_engine",
+ "enable_common_expr_pushdown",
+ "enable_common_expr_pushdown_for_inverted_index",
+ "enable_phrase_query_sequential_opt",
+ "enable_rust_lance_reader",
+ "shuffled_agg_node_ids");
private static boolean isRemovedSessionVar(String varName) {
return varName != null &&
REMOVED_SESSION_VAR_NAMES.contains(varName.toLowerCase());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
index 746c357ef60..fb9c482ca1c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
@@ -77,18 +77,6 @@ public class SessionVariablesTest extends TestWithFeService {
sessionVariable.getInsertVisibleTimeoutReturnModeEnum());
}
- @Test
- public void testCloneSessionVariablesWithSessionOriginValueNotEmpty()
throws NoSuchFieldException {
- Field txIsolation = SessionVariable.class.getField("txIsolation");
- SessionVariableField txIsolationSessionVariableField = new
SessionVariableField(txIsolation);
- sessionVariable.addSessionOriginValue(txIsolationSessionVariableField,
"test");
-
- SessionVariable sessionVariableClone =
VariableMgr.cloneSessionVariable(sessionVariable);
-
- Assertions.assertEquals("test",
-
sessionVariableClone.getSessionOriginValue().get(txIsolationSessionVariableField));
- }
-
@Test
public void testInsertVisibleTimeoutReturnMode() throws Exception {
connectContext.setThreadLocalInfo();
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 211f38dccb5..3271cb01e88 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -209,7 +209,7 @@ struct TQueryOptions {
64: optional bool dry_run_query = false
- 65: optional bool enable_common_expr_pushdown = false;
+ 65: optional bool enable_common_expr_pushdown = false; // deprecated
66: optional i32 parallel_instance = 1
// Indicate where useServerPrepStmts enabled
@@ -352,7 +352,7 @@ struct TQueryOptions {
135: optional bool enable_parallel_outfile = false;
- 136: optional bool enable_phrase_query_sequential_opt = true;
+ 136: optional bool enable_phrase_query_sequential_opt = true; // deprecated
137: optional bool enable_auto_create_when_overwrite = false;
@@ -482,7 +482,7 @@ struct TQueryOptions {
215: optional i64 ai_context_window_size = 131072;
// Use Rust-based Lance reader for FORMAT_LANCE scan ranges
- 216: optional bool enable_rust_lance_reader = false;
+ 216: optional bool enable_rust_lance_reader = false; // deprecated
217: optional bool new_version_percentile = false
// Adaptive batch size: target output block size in bytes. Valid range [1MB,
512MB].
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
