This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 6551dfd04b [fix](sink) fix pipeline load stuck (#216361)
new 50f2c89b64 [fix](jdbc) fix presto jdbc catalog pushDown and nameFormat
(#21447)
new 041deafbea [typo](docs) fix presto jdbc catalog docs (#21445)
new 9a3f54c139 [refactor](Nereids): remove withLogicalProperties & check
children size (#21563)
new ef5ed80fe3 [improvement](multi catalog)Use getPartitionsByNames to
retrieve hive partitions (#21562)
new c84fa82aab [Refactor](multi-catalog) Refactor to process splitted
conjuncts for dict filter. (#21459)
new f17a82b176 (chore) Disable ssl connection to FE by default for
compatibility reason (#20230)
new 79191bcd5a [fix](test) build internal table for TPCHTest to fix
testRank (#21566)
new 07aa865016 [fix](partial update) correct col_nums when init agg state
in memtable (#21592)
new d5155ae6e2 [fix](kerberos) should renew the kerberos ticket each half
of ticket lifetime (#21546)
new cc54beef4e [fix](Nereids) union distinct should not prune any column
(#21610)
new 33629b0129 [Improve](mysql)ensure constant time for computing hash
value (#21569)
new 687eb44b97 [enhancement](merge-on-write) add more version and txn
information for mow publish (#21257)
new 8b48adcf67 [fix](nereids) Agg on unknown-stats column (#21428)
new 352e7f173d [fix](catalog) wrong required slot info causing BE crash
(#21598)
new 4f31a371c2 [Fix](Planner) change non boolean return type to boolean
(#20599)
new dfe6d6969e [Enhancement](planner&Nereids) support sql_select_limit for
master (#21138)
new b1e39ad785 [opt](Nereids) forbid some bad case on agg plans (#21565)
new aa4ad7b8ff [fix](nereids) don't build cte producer if the consumer is
empty relation (#21317)
new 1d6122bd46 [fix](Nereids) Add alias name for system variable (#21615)
new aeedb0d1db fix pick pr conflict
new 319ac5bacc [Feature] (binlog) Add getBinlogLag (#21637)
new 745e330763 [fix][improvement](fs) add HdfsIO profile and modification
time (#21638)
new 0eb73c8218 [FIX](serde) fix map/struct/array support arrow #21628
new 54cbe46461 [opt](Nereids) enable parallel scan for local phase agg
(#21642)
new 3eaaad5b80 [refactor](planner) refactor automatically set instance_num
(#21640)
new fa94f6d3a9 [Fix](storage engine) shutdown cooldown and cold data
compaction thread when engine stop (#21639)
new 6a95750303 [opt](regression case) add more index change regression
case (#21633)
new 2b5163d3c3 [Enhancement](binlog) Add create/drop table, add/drop
paritition && alter job, modify columns binlog support (#21544)
new 2fa898e9d9 [pipeline](exec) disable pipeline load in now version
(#21632)
The 29 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/agent/task_worker_pool.cpp | 7 +-
be/src/io/file_factory.cpp | 54 ++--
be/src/io/file_factory.h | 42 +--
be/src/io/fs/benchmark/hdfs_benchmark.hpp | 12 +-
be/src/io/fs/benchmark/s3_benchmark.hpp | 10 +-
be/src/io/fs/broker_file_system.cpp | 27 +-
be/src/io/fs/broker_file_system.h | 3 +-
be/src/io/fs/buffered_reader.cpp | 4 +-
be/src/io/fs/buffered_reader.h | 7 +-
be/src/io/fs/file_reader_options.h | 4 -
be/src/io/fs/file_system.cpp | 6 +-
be/src/io/fs/file_system.h | 13 +-
be/src/io/fs/fs_utils.cpp | 46 ---
be/src/io/fs/fs_utils.h | 106 +------
be/src/io/fs/hdfs_file_reader.cpp | 40 ++-
be/src/io/fs/hdfs_file_reader.h | 16 +-
be/src/io/fs/hdfs_file_system.cpp | 33 +-
be/src/io/fs/hdfs_file_system.h | 12 +-
be/src/io/fs/local_file_system.cpp | 30 +-
be/src/io/fs/local_file_system.h | 7 +-
be/src/io/fs/remote_file_system.cpp | 11 +-
be/src/io/fs/remote_file_system.h | 5 +-
be/src/io/fs/s3_file_system.cpp | 8 +-
be/src/io/fs/s3_file_system.h | 3 +-
be/src/olap/data_dir.cpp | 3 +-
be/src/olap/data_dir.h | 2 -
be/src/olap/memtable.cpp | 2 +-
be/src/olap/rowset/segment_v2/segment.cpp | 9 +-
be/src/olap/storage_engine.cpp | 5 +
be/src/olap/task/engine_publish_version_task.cpp | 28 +-
be/src/olap/txn_manager.cpp | 36 +++
be/src/olap/txn_manager.h | 5 +
be/src/runtime/snapshot_loader.cpp | 2 +-
be/src/util/arrow/row_batch.cpp | 4 +-
be/src/util/os_util.cpp | 6 +-
be/src/util/path_util.cpp | 7 -
be/src/util/path_util.h | 6 -
be/src/vec/core/block_spill_reader.cpp | 8 +-
.../vec/data_types/serde/data_type_array_serde.cpp | 9 +-
.../vec/data_types/serde/data_type_array_serde.h | 2 +-
.../vec/data_types/serde/data_type_bitmap_serde.h | 2 +-
.../data_types/serde/data_type_date64_serde.cpp | 5 +-
.../vec/data_types/serde/data_type_date64_serde.h | 2 +-
.../serde/data_type_datetimev2_serde.cpp | 4 +-
.../data_types/serde/data_type_datetimev2_serde.h | 2 +-
.../data_types/serde/data_type_datev2_serde.cpp | 6 +-
.../vec/data_types/serde/data_type_datev2_serde.h | 2 +-
.../data_types/serde/data_type_decimal_serde.cpp | 10 +-
.../vec/data_types/serde/data_type_decimal_serde.h | 2 +-
.../serde/data_type_fixedlengthobject_serde.h | 2 +-
.../vec/data_types/serde/data_type_hll_serde.cpp | 4 +-
be/src/vec/data_types/serde/data_type_hll_serde.h | 2 +-
.../vec/data_types/serde/data_type_jsonb_serde.cpp | 21 ++
.../vec/data_types/serde/data_type_jsonb_serde.h | 3 +
.../vec/data_types/serde/data_type_map_serde.cpp | 41 ++-
be/src/vec/data_types/serde/data_type_map_serde.h | 2 +-
.../data_types/serde/data_type_nullable_serde.cpp | 15 +-
.../data_types/serde/data_type_nullable_serde.h | 2 +-
.../data_types/serde/data_type_number_serde.cpp | 21 +-
.../vec/data_types/serde/data_type_number_serde.h | 2 +-
.../vec/data_types/serde/data_type_object_serde.h | 2 +-
.../serde/data_type_quantilestate_serde.h | 2 +-
be/src/vec/data_types/serde/data_type_serde.h | 18 +-
.../data_types/serde/data_type_string_serde.cpp | 15 +-
.../vec/data_types/serde/data_type_string_serde.h | 2 +-
.../data_types/serde/data_type_struct_serde.cpp | 27 +-
.../vec/data_types/serde/data_type_struct_serde.h | 2 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 12 +-
be/src/vec/exec/format/csv/csv_reader.h | 4 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 7 +-
be/src/vec/exec/format/json/new_json_reader.h | 4 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 6 +-
be/src/vec/exec/format/orc/vorc_reader.h | 4 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 6 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 4 +-
be/src/vec/exec/scan/vfile_scanner.cpp | 61 ++--
be/src/vec/exec/scan/vfile_scanner.h | 4 +-
be/src/vec/runtime/vfile_result_writer.cpp | 2 +-
be/test/olap/primary_key_index_test.cpp | 1 -
be/test/olap/tablet_cooldown_test.cpp | 6 +-
be/test/olap/txn_manager_test.cpp | 20 ++
.../serde/data_type_serde_arrow_test.cpp | 131 +++++++-
docs/en/docs/lakehouse/multi-catalog/jdbc.md | 2 +-
docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md | 2 +-
.../main/java/org/apache/doris/common/Config.java | 4 +-
.../main/java/org/apache/doris/alter/Alter.java | 2 +-
.../java/org/apache/doris/alter/AlterHandler.java | 11 +-
.../java/org/apache/doris/alter/AlterJobV2.java | 10 +-
.../doris/alter/MaterializedViewHandler.java | 26 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 19 +-
.../apache/doris/alter/SchemaChangeHandler.java | 31 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 9 +-
.../java/org/apache/doris/alter/SystemHandler.java | 5 +-
.../apache/doris/analysis/AbstractBackupStmt.java | 8 +
.../java/org/apache/doris/analysis/IndexDef.java | 26 +-
.../org/apache/doris/analysis/RestoreStmt.java | 4 +
.../java/org/apache/doris/analysis/SelectStmt.java | 9 +-
.../java/org/apache/doris/analysis/TableRef.java | 4 +
.../org/apache/doris/binlog/BinlogManager.java | 74 ++++-
.../java/org/apache/doris/binlog/BinlogUtils.java | 29 +-
.../org/apache/doris/binlog/CreateTableRecord.java | 102 +++++++
.../java/org/apache/doris/binlog/DBBinlog.java | 38 ++-
.../org/apache/doris/binlog/DropTableRecord.java | 59 ++++
.../java/org/apache/doris/binlog/TableBinlog.java | 29 +-
.../java/org/apache/doris/catalog/JdbcTable.java | 1 +
.../doris/datasource/hive/HiveMetaStoreCache.java | 49 ++-
.../datasource/hive/PooledHiveMetaStoreClient.java | 14 +
.../org/apache/doris/master/ReportHandler.java | 41 ++-
.../java/org/apache/doris/mysql/MysqlPassword.java | 11 +-
.../nereids/analyzer/UnboundOlapTableSink.java | 6 -
.../nereids/analyzer/UnboundOneRowRelation.java | 5 -
.../doris/nereids/analyzer/UnboundRelation.java | 6 -
.../doris/nereids/analyzer/UnboundTVFRelation.java | 5 -
.../glue/translator/PhysicalPlanTranslator.java | 7 +-
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 2 +-
.../properties/ChildrenPropertiesRegulator.java | 53 +++-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/analysis/AddDefaultLimit.java | 96 ++++++
.../rules/implementation/AggregateStrategies.java | 333 ++++++++++++---------
.../rewrite/BuildCTEAnchorAndCTEProducer.java | 23 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 5 +
.../doris/nereids/rules/rewrite/MultiJoin.java | 5 -
.../mv/AbstractSelectMaterializedIndexRule.java | 3 +-
.../doris/nereids/stats/StatsCalculator.java | 10 +-
.../expressions/functions/agg/AggregateParam.java | 31 +-
.../functions/agg/MultiDistinctCount.java | 2 +-
.../functions/agg/MultiDistinctGroupConcat.java | 2 +-
.../functions/agg/MultiDistinctSum.java | 4 +-
.../functions/agg/MultiDistinction.java} | 16 +-
.../apache/doris/nereids/trees/plans/FakePlan.java | 5 -
.../doris/nereids/trees/plans/GroupPlan.java | 10 -
.../org/apache/doris/nereids/trees/plans/Plan.java | 6 -
.../nereids/trees/plans/algebra/Aggregate.java | 6 +-
.../nereids/trees/plans/commands/Command.java | 5 -
.../trees/plans/logical/LogicalAggregate.java | 8 +-
.../nereids/trees/plans/logical/LogicalApply.java | 8 +-
.../trees/plans/logical/LogicalAssertNumRows.java | 9 +-
.../nereids/trees/plans/logical/LogicalCTE.java | 7 +-
.../trees/plans/logical/LogicalCTEAnchor.java | 5 -
.../trees/plans/logical/LogicalCTEConsumer.java | 8 -
.../trees/plans/logical/LogicalCTEProducer.java | 6 -
.../trees/plans/logical/LogicalCheckPolicy.java | 6 +-
.../trees/plans/logical/LogicalEmptyRelation.java | 5 -
.../nereids/trees/plans/logical/LogicalEsScan.java | 6 -
.../nereids/trees/plans/logical/LogicalExcept.java | 6 -
.../trees/plans/logical/LogicalFileScan.java | 6 -
.../nereids/trees/plans/logical/LogicalFilter.java | 6 +-
.../trees/plans/logical/LogicalGenerate.java | 7 +-
.../nereids/trees/plans/logical/LogicalHaving.java | 6 +-
.../trees/plans/logical/LogicalIntersect.java | 6 -
.../trees/plans/logical/LogicalJdbcScan.java | 6 -
.../nereids/trees/plans/logical/LogicalJoin.java | 7 +-
.../nereids/trees/plans/logical/LogicalLimit.java | 6 +-
.../trees/plans/logical/LogicalOlapScan.java | 8 -
.../trees/plans/logical/LogicalOlapTableSink.java | 6 -
.../trees/plans/logical/LogicalOneRowRelation.java | 5 -
.../trees/plans/logical/LogicalPartitionTopN.java | 7 +-
.../trees/plans/logical/LogicalProject.java | 7 +-
.../nereids/trees/plans/logical/LogicalRepeat.java | 7 +-
.../trees/plans/logical/LogicalSchemaScan.java | 5 -
.../trees/plans/logical/LogicalSelectHint.java | 13 +-
.../nereids/trees/plans/logical/LogicalSort.java | 6 +-
.../trees/plans/logical/LogicalSubQueryAlias.java | 7 +-
.../trees/plans/logical/LogicalTVFRelation.java | 5 -
.../nereids/trees/plans/logical/LogicalTopN.java | 6 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 6 -
.../nereids/trees/plans/logical/LogicalWindow.java | 7 +-
.../nereids/trees/plans/logical/UsingJoin.java | 6 -
.../plans/physical/PhysicalAssertNumRows.java | 7 +-
.../trees/plans/physical/PhysicalCTEAnchor.java | 7 +-
.../trees/plans/physical/PhysicalCTEConsumer.java | 6 -
.../trees/plans/physical/PhysicalCTEProducer.java | 5 -
.../trees/plans/physical/PhysicalDistribute.java | 7 +-
.../plans/physical/PhysicalEmptyRelation.java | 6 -
.../trees/plans/physical/PhysicalEsScan.java | 5 -
.../trees/plans/physical/PhysicalExcept.java | 6 -
.../trees/plans/physical/PhysicalFileScan.java | 6 -
.../trees/plans/physical/PhysicalFilter.java | 6 +-
.../trees/plans/physical/PhysicalGenerate.java | 7 +-
.../plans/physical/PhysicalHashAggregate.java | 8 +-
.../trees/plans/physical/PhysicalHashJoin.java | 8 +-
.../trees/plans/physical/PhysicalIntersect.java | 6 -
.../trees/plans/physical/PhysicalJdbcScan.java | 5 -
.../trees/plans/physical/PhysicalLimit.java | 6 +-
.../plans/physical/PhysicalNestedLoopJoin.java | 9 +-
.../trees/plans/physical/PhysicalOlapScan.java | 7 -
.../plans/physical/PhysicalOlapTableSink.java | 6 -
.../plans/physical/PhysicalOneRowRelation.java | 6 -
.../plans/physical/PhysicalPartitionTopN.java | 6 -
.../trees/plans/physical/PhysicalProject.java | 12 +-
.../trees/plans/physical/PhysicalQuickSort.java | 6 +-
.../trees/plans/physical/PhysicalRepeat.java | 7 +-
.../trees/plans/physical/PhysicalSchemaScan.java | 6 -
.../physical/PhysicalStorageLayerAggregate.java | 20 +-
.../trees/plans/physical/PhysicalTVFRelation.java | 6 -
.../nereids/trees/plans/physical/PhysicalTopN.java | 6 +-
.../trees/plans/physical/PhysicalUnion.java | 6 -
.../trees/plans/physical/PhysicalWindow.java | 7 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 5 +
.../java/org/apache/doris/persist/EditLog.java | 22 +-
.../doris/persist/TableAddOrDropColumnsInfo.java | 9 +-
.../persist/TableAddOrDropInvertedIndicesInfo.java | 5 +-
.../org/apache/doris/planner/JdbcScanNode.java | 5 +-
.../apache/doris/planner/SingleNodePlanner.java | 51 ++--
.../doris/planner/external/FileQueryScanNode.java | 8 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 4 +-
.../java/org/apache/doris/qe/SessionVariable.java | 6 +-
.../apache/doris/service/FrontendServiceImpl.java | 98 ++++++
.../main/java/org/apache/doris/system/Backend.java | 98 +++---
.../org/apache/doris/system/SystemInfoService.java | 22 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 18 +-
.../org/apache/doris/alter/RollupJobV2Test.java | 2 +-
.../apache/doris/alter/SchemaChangeJobV2Test.java | 2 +-
.../doris/nereids/datasets/tpch/TPCHTestBase.java | 9 +
.../doris/nereids/jobs/RewriteTopDownJobTest.java | 5 -
.../nereids/jobs/cascades/DeriveStatsJobTest.java | 5 +-
.../rules/rewrite/AggregateStrategiesTest.java | 2 +
.../doris/nereids/stats/StatsCalculatorTest.java | 5 +-
.../doris/nereids/trees/plans/PlanOutputTest.java | 19 --
.../persist/TableAddOrDropColumnsInfoTest.java | 2 +-
gensrc/thrift/FrontendService.thrift | 11 +
gensrc/thrift/MasterService.thrift | 1 +
.../hive/test_external_catalog_hive.out | 3 +
.../index_change/test_index_change_1.out | 26 ++
.../index_change/test_index_change_2.out | 43 +++
.../index_change/test_index_change_3.out | 43 +++
.../index_change/test_index_change_4.out | 43 +++
.../index_change/test_index_change_5.out | 43 +++
.../index_change/test_index_change_6.out | 43 +++
.../data/nereids_syntax_p0/set_operation.out | 3 +
.../nereids_tpcds_shape_sf100_p0/shape/query94.out | 4 +-
.../nereids_tpcds_shape_sf100_p0/shape/query95.out | 4 +-
.../nereids_tpch_shape_sf1000_p0/shape/q16.out | 4 +-
.../data/nereids_tpch_shape_sf500_p0/shape/q16.out | 4 +-
.../partial_update/basic_with_duplicate.csv | 3 +
.../partial_update/basic_with_duplicate2.csv | 6 +
.../partial_update/test_partial_update.out | 8 +
.../hive/test_external_catalog_hive.groovy | 2 +
.../index_change/test_index_change_1.groovy | 105 +++++++
.../index_change/test_index_change_2.groovy | 115 +++++++
.../index_change/test_index_change_3.groovy | 117 ++++++++
.../index_change/test_index_change_4.groovy | 116 +++++++
.../index_change/test_index_change_5.groovy | 109 +++++++
.../index_change/test_index_change_6.groovy | 121 ++++++++
.../suites/nereids_p0/join/test_join.groovy | 9 -
.../session_variable/test_default_limit.groovy | 253 ++++++++++++++++
.../suites/nereids_syntax_p0/agg_4_phase.groovy | 12 +-
.../nereids_syntax_p0/aggregate_strategies.groovy | 7 +-
.../suites/nereids_syntax_p0/cte.groovy | 5 +
.../suites/nereids_syntax_p0/group_concat.groovy | 4 +-
.../suites/nereids_syntax_p0/set_operation.groovy | 5 +
.../suites/query_p0/cast/test_cast.groovy | 58 +++-
.../session_variable/test_default_limit.groovy | 251 ++++++++++++++++
.../partial_update/test_partial_update.groovy | 37 +++
255 files changed, 3480 insertions(+), 1374 deletions(-)
delete mode 100644 be/src/io/fs/fs_utils.cpp
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AddDefaultLimit.java
copy
fe/fe-core/src/{test/java/org/apache/doris/nereids/datasets/tpch/TPCHTestBase.java
=>
main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinction.java}
(71%)
create mode 100644
regression-test/data/inverted_index_p0/index_change/test_index_change_1.out
create mode 100644
regression-test/data/inverted_index_p0/index_change/test_index_change_2.out
create mode 100644
regression-test/data/inverted_index_p0/index_change/test_index_change_3.out
create mode 100644
regression-test/data/inverted_index_p0/index_change/test_index_change_4.out
create mode 100644
regression-test/data/inverted_index_p0/index_change/test_index_change_5.out
create mode 100644
regression-test/data/inverted_index_p0/index_change/test_index_change_6.out
create mode 100644
regression-test/data/unique_with_mow_p0/partial_update/basic_with_duplicate.csv
create mode 100644
regression-test/data/unique_with_mow_p0/partial_update/basic_with_duplicate2.csv
create mode 100644
regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy
create mode 100644
regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy
create mode 100644
regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy
create mode 100644
regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy
create mode 100644
regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy
create mode 100644
regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy
create mode 100644
regression-test/suites/nereids_p0/session_variable/test_default_limit.groovy
create mode 100644
regression-test/suites/query_p0/session_variable/test_default_limit.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
