This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit d658a44cefbe9a02c80e6a8bd79e1e718e251269 Author: zy-kkk <[email protected]> AuthorDate: Tue Jan 16 21:07:59 2024 +0800 [improvement](catalog) Change the push-down parameters of the predicate function of the table query SQL into variables (#30028) In this PR, we will control whether the external data source query is a push-down function parameter in the filter condition, changing the enable_fun_pushdown of fe conf to the enable_ext_func_pred_pushdown of the variable --- docs/en/docs/lakehouse/multi-catalog/jdbc.md | 2 +- docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md | 2 +- .../src/main/java/org/apache/doris/common/Config.java | 7 ------- .../org/apache/doris/planner/external/jdbc/JdbcScanNode.java | 8 ++++++-- .../org/apache/doris/planner/external/odbc/OdbcScanNode.java | 7 +++++-- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 12 +++++++++++- .../external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy | 11 ++++++++--- 7 files changed, 32 insertions(+), 17 deletions(-) diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index 662fcc81118..1a401146df9 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -114,7 +114,7 @@ In some cases, the keywords in the database might be used as the field names. Fo 1. When executing a query like `where dt = '2022-01-01'`, Doris can push down these filtering conditions to the external data source, thereby directly excluding data that does not meet the conditions at the data source level, reducing the number of unqualified Necessary data acquisition and transfer. This greatly improves query performance while also reducing the load on external data sources. -2. When `enable_func_pushdown` is set to true, the function conditions after where will also be pushed down to the external data source. Currently, only MySQL, ClickHouse, and Oracle are supported. If you encounter functions that are not supported by MySQL, ClickHouse, and Oracle, you can use this The parameter is set to false. At present, Doris will automatically identify some functions that are not supported by MySQL and functions supported by CLickHouse and Oracle for push-down condit [...] +2. When variable `enable_ext_func_pred_pushdown` is set to true, the function conditions after where will also be pushed down to the external data source. Currently, only MySQL, ClickHouse, and Oracle are supported. If you encounter functions that are not supported by MySQL, ClickHouse, and Oracle, you can use this The parameter is set to false. At present, Doris will automatically identify some functions that are not supported by MySQL and functions supported by CLickHouse and Oracle fo [...] Functions that are currently not pushed down include: diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index e3eb90d7f40..a86ab26c33d 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -114,7 +114,7 @@ select * from mysql_catalog.mysql_database.mysql_table where k1 > 1000 and k3 =' 1. 当执行类似于 `where dt = '2022-01-01'` 这样的查询时，Doris 能够将这些过滤条件下推到外部数据源，从而直接在数据源层面排除不符合条件的数据，减少了不必要的数据获取和传输。这大大提高了查询性能，同时也降低了对外部数据源的负载。 -2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL、ClickHouse、Oracle，如遇到 MySQL、ClickHouse、Oracle 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数以及 CLickHouse、Oracle 支持的函数进行下推条件过滤，可通过 explain sql 查看。 +2. 当变量 `enable_ext_func_pred_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL、ClickHouse、Oracle，如遇到 MySQL、ClickHouse、Oracle 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数以及 CLickHouse、Oracle 支持的函数进行下推条件过滤，可通过 explain sql 查看。 目前不会下推的函数有： diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index d3211c5b397..f5865e1b7ba 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1921,13 +1921,6 @@ public class Config extends ConfigBase { @ConfField(mutable = false, varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_fqdn_mode = false; - /** - * This is used whether to push down function to MYSQL in external Table with query sql - * like odbc, jdbc for mysql table - */ - @ConfField(mutable = true) - public static boolean enable_func_pushdown = true; - /** * If set to true, doris will try to parse the ddl of a hive view and try to execute the query * otherwise it will throw an AnalysisException. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java index 774d4284bce..d2731d33e13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java @@ -37,7 +37,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.external.JdbcExternalTable; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; @@ -223,6 +222,7 @@ public class JdbcScanNode extends ExternalScanNode { } if (jdbcType == TOdbcTableType.CLICKHOUSE + && ConnectContext.get() != null && ConnectContext.get().getSessionVariable().jdbcClickhouseQueryFinal) { sql.append(" SETTINGS final = 1"); } @@ -313,7 +313,11 @@ public class JdbcScanNode extends ExternalScanNode { if (containsFunctionCallExpr(expr)) { if (tableType.equals(TOdbcTableType.MYSQL) || tableType.equals(TOdbcTableType.CLICKHOUSE) || tableType.equals(TOdbcTableType.ORACLE)) { - return Config.enable_func_pushdown; + if (ConnectContext.get() != null) { + return ConnectContext.get().getSessionVariable().enableExtFuncPredPushdown; + } else { + return true; + } } else { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java index f3ba2a5add8..891cd8240b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java @@ -30,7 +30,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.JdbcTable; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; @@ -267,6 +266,10 @@ public class OdbcScanNode extends ExternalScanNode { return false; } } - return Config.enable_func_pushdown; + if (ConnectContext.get() != null) { + return ConnectContext.get().getSessionVariable().enableExtFuncPredPushdown; + } else { + return true; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index e87647bfb6c..978171001f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -302,6 +302,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_FUNCTION_PUSHDOWN = "enable_function_pushdown"; + public static final String ENABLE_EXT_FUNC_PRED_PUSHDOWN = "enable_ext_func_pred_pushdown"; + public static final String ENABLE_COMMON_EXPR_PUSHDOWN = "enable_common_expr_pushdown"; public static final String FRAGMENT_TRANSMISSION_COMPRESSION_CODEC = "fragment_transmission_compression_codec"; @@ -524,7 +526,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = EXPAND_RUNTIME_FILTER_BY_INNER_JION) public boolean expandRuntimeFilterByInnerJoin = true; - @VariableMgr.VarAttr(name = JDBC_CLICKHOUSE_QUERY_FINAL) + @VariableMgr.VarAttr(name = JDBC_CLICKHOUSE_QUERY_FINAL, needForward = true, + description = {"是否在查询 ClickHouse JDBC 外部表时，对查询 SQL 添加 FINAL 关键字。", + "Whether to add the FINAL keyword to the query SQL when querying ClickHouse JDBC external tables."}) public boolean jdbcClickhouseQueryFinal = false; @VariableMgr.VarAttr(name = ROUND_PRECISE_DECIMALV2_VALUE) @@ -1090,6 +1094,12 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN, fuzzy = true) public boolean enableFunctionPushdown = false; + @VariableMgr.VarAttr(name = ENABLE_EXT_FUNC_PRED_PUSHDOWN, needForward = true, + description = {"启用外部表（如通过ODBC或JDBC访问的表）查询中谓词的函数下推", + "Enable function pushdown for predicates in queries to external tables " + + "(such as tables accessed via ODBC or JDBC)"}) + public boolean enableExtFuncPredPushdown = true; + @VariableMgr.VarAttr(name = FORBID_UNKNOWN_COLUMN_STATS) public boolean forbidUnknownColStats = false; diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index fdc1f84e382..86afd749e56 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -186,6 +186,11 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc contains """ SELECT `datetime` FROM `doris_test`.`all_types` WHERE (date(`datetime`) = '2012-10-25')""" } + explain { + sql("select /*+ SET_VAR(enable_ext_func_pred_pushdown = false) */ `datetime` from all_types where to_date(`datetime`) = '2012-10-25';") + contains """SELECT `datetime` FROM `doris_test`.`all_types`""" + } + // test insert String uuid1 = UUID.randomUUID().toString(); connect(user=user, password="${pwd}", url=url) { @@ -378,7 +383,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc } try { sql """ use ${ex_db_name}""" - sql """ admin set frontend config ("enable_func_pushdown" = "true"); """ + sql """ set enable_ext_func_pred_pushdown = "true"; """ order_qt_filter1 """select * from ${ex_tb17} where id = 1; """ order_qt_filter2 """select * from ${ex_tb17} where 1=1 order by 1; """ order_qt_filter3 """select * from ${ex_tb17} where id = 1 and 1 = 1; """ @@ -409,13 +414,13 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc contains "QUERY: SELECT `k6`, `k8` FROM `doris_test`.`test1` WHERE (ifnull(ifnull(`k6`, NULL), NULL) = 1)" } - sql """ admin set frontend config ("enable_func_pushdown" = "false"); """ + sql """ set enable_ext_func_pred_pushdown = "false"; """ explain { sql ("select k6, k8 from test1 where nvl(k6, null) = 1 and k8 = 1;") contains "QUERY: SELECT `k6`, `k8` FROM `doris_test`.`test1` WHERE (`k8` = 1)" } - sql """ admin set frontend config ("enable_func_pushdown" = "true"); """ + sql """ set enable_ext_func_pred_pushdown = "true"; """ } finally { res_dbs_log = sql "show databases;" for(int i = 0;i < res_dbs_log.size();i++) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
