This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from e546d62f58 change branch-2.0 version to 2.0.0-dev
new 3ee83c77b0 [fix](merge-on-write) fix be core and delete unused pending
publish info for async publish when tablet dropped (#21793)
new 26497c49a4 [Fix](MoW) Fix bug about caculating all committed rowsets
delete bitmaps when do comapction (#21760)
new 9eedda343b [feature](hudi) support hudi time travel in external table
(#21739)
new 1e9e6afdeb [Fix](parquet-reader) Fix parquet string column min max
statistics issue which caused query result incorrectly. (#21675)
new 4d595d548f [Agg](exec) support aggregation_node limit short circuit
(#21767)
new 9b61fcbf22 [FIX](map) fix map key-column nullable for arrow serde
#21762
new c3c87ab3db [Improvement](multi catalog)Cache file system to improve
list remote files performance (#21700)
new 580ddd8c1d [feature](table-value-functions)add catalogs
table-value-function (#21790)
new 0456342faa [improve](nereids)inner join estimation: assume children
output at least one tuple #21792
new 46ab7587e2 [imporve](udaf) refactor java-udaf executor by using for
loop (#21713)
new 466b7b854d [fix](Nereids) use groupExpr's children to make logicalPlan
(#21794)
new d7960b8053 [enhance](Nereids) Pushdown Project Through OuterJoin.
(#21730)
new 5170c50ca9 [enhancement](multi-table) enable mullti table routine load
on pipeline engine (#21729)
new 26f5bea217 [feature](nereids) adjust min/max of column stats for cast
function (#21772)
new 7e0c9f551e [Enhancement] (binlog) TBinlog and BinlogManager V2 (#21674)
new dcbdb9cb1e [Feature](Nereids) support udf for Nereids (#18257)
new a51741cba6 [regression] add order by in test case for stable output
(#21815)
new 976a9e0197 [imporve](bloomfilter) refactor runtime_filter_mgr with
bloomfilter and fix bug in change_to_bloom_filter (#21783)
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exprs/runtime_filter.cpp | 5 +-
be/src/io/file_factory.cpp | 27 +-
be/src/io/file_factory.h | 2 +-
be/src/io/fs/multi_table_pipe.cpp | 85 ++++--
be/src/io/fs/multi_table_pipe.h | 3 +
be/src/io/fs/stream_load_pipe.cpp | 7 +
be/src/io/fs/stream_load_pipe.h | 3 +
be/src/olap/compaction.cpp | 46 +++-
be/src/olap/compaction.h | 3 +
be/src/olap/olap_server.cpp | 6 +-
be/src/olap/storage_engine.cpp | 27 ++
be/src/olap/storage_engine.h | 2 +
be/src/pipeline/pipeline_fragment_context.cpp | 6 +-
be/src/runtime/runtime_filter_mgr.cpp | 38 +--
be/src/runtime/runtime_filter_mgr.h | 13 +-
be/src/runtime/runtime_state.cpp | 6 +-
be/src/runtime/runtime_state.h | 9 +-
.../aggregate_function_java_udaf.h | 119 ++++----
.../vec/data_types/serde/data_type_map_serde.cpp | 15 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 3 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 3 +-
be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 14 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 15 +-
be/src/vec/exec/scan/vmeta_scanner.cpp | 18 ++
be/src/vec/exec/scan/vmeta_scanner.h | 3 +-
be/src/vec/exec/vaggregation_node.cpp | 4 +
be/src/vec/exec/vaggregation_node.h | 5 +
.../serde/data_type_serde_arrow_test.cpp | 84 +++++-
docs/en/docs/lakehouse/multi-catalog/hudi.md | 22 +-
.../sql-functions/table-functions/catalogs.md | 91 +++++++
docs/sidebars.json | 3 +-
docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md | 22 +-
.../sql-functions/table-functions/catalogs.md | 92 +++++++
fe/be-java-extensions/hudi-scanner/pom.xml | 3 +-
.../org/apache/doris/hudi/BaseSplitReader.scala | 8 +-
fe/be-java-extensions/java-udf/pom.xml | 6 +
.../java/org/apache/doris/udf/BaseExecutor.java | 181 +++++++++++++
.../java/org/apache/doris/udf/UdafExecutor.java | 89 +++++-
.../main/java/org/apache/doris/udf/UdfConvert.java | 262 +++++++++---------
.../java/org/apache/doris/udf/UdfExecutor.java | 165 +-----------
.../main/java/org/apache/doris/common/Config.java | 4 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 7 +-
.../java/org/apache/doris/analysis/SlotRef.java | 5 +
.../java/org/apache/doris/analysis/TableRef.java | 32 ++-
.../org/apache/doris/analysis/TableSnapshot.java | 6 +-
.../org/apache/doris/binlog/BinlogComparator.java | 24 ++
.../java/org/apache/doris/binlog/BinlogGcer.java | 5 +-
.../org/apache/doris/binlog/BinlogManager.java | 120 +++++----
.../org/apache/doris/binlog/BinlogTombstone.java | 8 +-
.../java/org/apache/doris/binlog/BinlogUtils.java | 53 ++++
.../java/org/apache/doris/binlog/DBBinlog.java | 298 +++++++++++++++------
.../java/org/apache/doris/binlog/TableBinlog.java | 167 ++++++++----
.../java/org/apache/doris/catalog/Database.java | 4 +
.../org/apache/doris/catalog/FunctionRegistry.java | 89 +++++-
.../org/apache/doris/catalog/FunctionUtil.java | 33 +++
.../apache/doris/catalog/GlobalFunctionMgr.java | 11 +
.../doris/catalog/HiveMetaStoreClientHelper.java | 13 +-
.../java/org/apache/doris/catalog/HudiUtils.java | 34 +++
.../java/org/apache/doris/common/ErrorCode.java | 2 +-
.../org/apache/doris/datasource/CatalogMgr.java | 2 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 29 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 24 +-
.../java/org/apache/doris/fs/FileSystemCache.java | 91 +++++++
.../org/apache/doris/fs/FileSystemFactory.java | 43 ++-
.../java/org/apache/doris/fs/FileSystemType.java | 25 ++
.../org/apache/doris/nereids/NereidsPlanner.java | 4 +
.../doris/nereids/analyzer/UnboundFunction.java | 19 +-
.../glue/translator/ExpressionTranslator.java | 18 ++
.../hypergraph/receiver/PlanReceiver.java | 6 +-
.../java/org/apache/doris/nereids/memo/Memo.java | 11 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 11 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +-
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
...a => PushdownProjectThroughInnerOuterJoin.java} | 37 ++-
.../rules/expression/rules/FunctionBinder.java | 14 +-
.../doris/nereids/stats/ExpressionEstimation.java | 31 ++-
.../apache/doris/nereids/stats/JoinEstimation.java | 2 +-
.../nereids/trees/expressions/functions/Udf.java | 49 ++++
.../trees/expressions/functions/udf/AliasUdf.java | 141 ++++++++++
.../expressions/functions/udf/AliasUdfBuilder.java | 108 ++++++++
.../trees/expressions/functions/udf/JavaUdaf.java | 201 ++++++++++++++
.../expressions/functions/udf/JavaUdafBuilder.java | 72 +++++
.../trees/expressions/functions/udf/JavaUdf.java | 167 ++++++++++++
.../expressions/functions/udf/JavaUdfBuilder.java | 82 ++++++
.../expressions/functions/udf/UdfBuilder.java | 30 +++
.../visitor/AggregateFunctionVisitor.java | 5 +
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../trees/plans/commands/ExplainCommand.java | 1 +
.../doris/planner/external/FileQueryScanNode.java | 7 +-
.../doris/planner/external/HiveScanNode.java | 2 +-
.../planner/external/TablePartitionValues.java | 255 ++++++++++++++++++
.../hudi/HudiCachedPartitionProcessor.java | 131 +++++++++
.../planner/external/hudi/HudiPartitionMgr.java | 86 ++++++
.../external/hudi/HudiPartitionProcessor.java | 124 +++++++++
.../doris/planner/external/hudi/HudiScanNode.java | 84 +++++-
.../apache/doris/statistics/ColumnStatistic.java | 4 +-
.../doris/statistics/util/StatisticsUtil.java | 3 +-
.../tablefunction/CatalogsTableValuedFunction.java | 88 ++++++
.../doris/tablefunction/MetadataGenerator.java | 37 +++
.../tablefunction/MetadataTableValuedFunction.java | 2 +
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
.../apache/doris/catalog/CreateFunctionTest.java | 7 +-
.../rules/analysis/FunctionRegistryTest.java | 2 +-
... PushdownProjectThroughInnerOuterJoinTest.java} | 10 +-
.../doris/nereids/trees/expressions/UdfTest.java | 174 ++++++++++++
gensrc/thrift/FrontendService.thrift | 5 +-
gensrc/thrift/Types.thrift | 3 +-
.../table_valued_function/test_catalogs_tvf.out | 10 +
.../hive/test_multi_langs.out | 148 ++++++++++
.../data/index_p0/test_ngram_bloomfilter_index.out | 8 +-
.../javaudf}/collect/test_javaudf_groupcount.out | 0
.../javaudf}/collect/test_javaudf_murmurhash3.out | 0
.../javaudf}/collect/test_javaudf_sessionize.out | 0
.../javaudf}/date/test_javaudf_adddays.out | 0
.../javaudf}/date/test_javaudf_addisoperioud.out | 0
.../javaudf}/date/test_javaudf_daydiff.out | 0
.../json/test_javaudf_convertfromcamelcase.out | 0
.../json/test_javaudf_converttocamelcase.out | 0
.../javaudf}/sanity/test_javaudf_assertequal.out | 0
.../sanity/test_javaudf_assertlessthan.out | 0
.../javaudf}/sanity/test_javaudf_assertudf.out | 0
.../javaudf}/sketch/test_javaudf_md5.out | 0
.../javaudf}/test_javaudaf_my_date_datetime.out | 0
.../test_javaudaf_mygroupconcat_string.out | 0
.../javaudf/test_javaudaf_mysum_array.out | 87 ++++++
.../javaudf}/test_javaudaf_mysum_decimal.out | 0
.../javaudf}/test_javaudaf_mysum_double.out | 0
.../javaudf}/test_javaudaf_mysum_float_double.out | 0
.../nereids_p0/javaudf/test_javaudaf_mysum_int.out | 33 +++
.../javaudf}/test_javaudaf_null_test.out | 0
.../data/nereids_p0/javaudf/test_javaudf_array.out | 133 +++++++++
.../javaudf}/test_javaudf_boolean.out | 0
.../javaudf}/test_javaudf_case.out | 0
.../javaudf}/test_javaudf_date.out | 0
.../javaudf}/test_javaudf_decimal.out | 0
.../javaudf}/test_javaudf_float.out | 0
.../javaudf}/test_javaudf_int.out | 0
.../javaudf}/test_javaudf_null.out | 0
.../javaudf}/test_javaudf_string.out | 0
.../table_valued_function/test_catalogs_tvf.groovy | 71 +++++
.../hive/test_multi_langs.groovy | 61 +++++
.../index_p0/test_ngram_bloomfilter_index.groovy | 26 +-
.../javaudf/collect/test_javaudf_groupcount.groovy | 61 +++++
.../collect/test_javaudf_murmurhash3.groovy | 62 +++++
.../javaudf/collect/test_javaudf_sessionize.groovy | 65 +++++
.../javaudf/date/test_javaudf_adddays.groovy | 62 +++++
.../javaudf/date/test_javaudf_addisoperioud.groovy | 62 +++++
.../javaudf/date/test_javaudf_daydiff.groovy | 63 +++++
.../json/test_javaudf_convertfromcamelcase.groovy | 62 +++++
.../json/test_javaudf_converttocamelcase.groovy | 62 +++++
.../javaudf/sanity/test_javaudf_assertequal.groovy | 64 +++++
.../sanity/test_javaudf_assertlessthan.groovy | 64 +++++
.../javaudf/sanity/test_javaudf_assertudf.groovy | 54 ++++
.../javaudf/sketch/test_javaudf_md5.groovy | 62 +++++
.../nereids_p0/javaudf/test_alias_function.groovy | 78 ++++++
.../javaudf/test_javaudaf_my_date_datetime.groovy | 127 +++++++++
.../test_javaudaf_mygroupconcat_string.groovy | 78 ++++++
.../javaudf/test_javaudaf_mysum_array.groovy | 127 +++++++++
.../javaudf/test_javaudaf_mysum_decimal.groovy | 74 +++++
.../test_javaudaf_mysum_float_double.groovy | 94 +++++++
.../javaudf/test_javaudaf_mysum_int.groovy | 84 ++++++
.../javaudf/test_javaudaf_null_test.groovy | 81 ++++++
.../nereids_p0/javaudf/test_javaudf_array.groovy | 133 +++++++++
.../nereids_p0/javaudf/test_javaudf_boolean.groovy | 75 ++++++
.../nereids_p0/javaudf/test_javaudf_case.groovy | 73 +++++
.../nereids_p0/javaudf/test_javaudf_date.groovy | 211 +++++++++++++++
.../nereids_p0/javaudf/test_javaudf_decimal.groovy | 86 ++++++
.../nereids_p0/javaudf/test_javaudf_float.groovy | 93 +++++++
.../nereids_p0/javaudf/test_javaudf_int.groovy | 127 +++++++++
.../nereids_p0/javaudf/test_javaudf_null.groovy | 74 +++++
.../nereids_p0/javaudf/test_javaudf_string.groovy | 79 ++++++
172 files changed, 6951 insertions(+), 759 deletions(-)
create mode 100644
docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md
create mode 100644
docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogComparator.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemType.java
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/{PushdownProjectThroughInnerJoin.java
=> PushdownProjectThroughInnerOuterJoin.java} (79%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfBuilder.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/external/TablePartitionValues.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionMgr.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionProcessor.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java
rename
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/{PushdownProjectThroughInnerJoinTest.java
=> PushdownProjectThroughInnerOuterJoinTest.java} (94%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
create mode 100644
regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out
create mode 100644
regression-test/data/external_table_emr_p2/hive/test_multi_langs.out
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/collect/test_javaudf_groupcount.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/collect/test_javaudf_murmurhash3.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/collect/test_javaudf_sessionize.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/date/test_javaudf_adddays.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/date/test_javaudf_addisoperioud.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/date/test_javaudf_daydiff.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/json/test_javaudf_convertfromcamelcase.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/json/test_javaudf_converttocamelcase.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/sanity/test_javaudf_assertequal.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/sanity/test_javaudf_assertlessthan.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/sanity/test_javaudf_assertudf.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/sketch/test_javaudf_md5.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudaf_my_date_datetime.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudaf_mygroupconcat_string.out (100%)
create mode 100644
regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_array.out
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudaf_mysum_decimal.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudaf_mysum_double.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudaf_mysum_float_double.out (100%)
create mode 100644
regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_int.out
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudaf_null_test.out (100%)
create mode 100644
regression-test/data/nereids_p0/javaudf/test_javaudf_array.out
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_boolean.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_case.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_date.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_decimal.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_float.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_int.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_null.out (100%)
copy regression-test/data/{javaudf_p0 =>
nereids_p0/javaudf}/test_javaudf_string.out (100%)
create mode 100644
regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy
create mode 100644
regression-test/suites/external_table_emr_p2/hive/test_multi_langs.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_groupcount.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_sessionize.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/date/test_javaudf_adddays.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/date/test_javaudf_addisoperioud.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/date/test_javaudf_daydiff.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertequal.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertudf.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/sketch/test_javaudf_md5.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_alias_function.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_my_date_datetime.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_array.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_decimal.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_float_double.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_int.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudaf_null_test.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_array.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_boolean.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_case.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_date.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_decimal.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_float.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_int.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_null.groovy
create mode 100644
regression-test/suites/nereids_p0/javaudf/test_javaudf_string.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
