This is an automated email from the ASF dual-hosted git repository.
BiteTheDDDDt pushed a change to branch dev_runtime_partition_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
discard ef2529ca3c3 [fix](fe) Reject non-first range RF pruning targets
discard c090ab64403 [improvement](be) Validate RF partition pruning metadata
discard ddbff8e7f3b [improvement](be) Use per-partition RF monotonicity
discard 572ab7793a8 [improvement](be) Unify RF partition pruning target
handling
discard b49d2be67c6 [fix](be) Preserve open RF pruning boundaries
discard bb97e1c8d1f [refactor](be) Simplify runtime filter partition pruner
state
discard 5bfeb4d44af [test](runtime-filter) Add partition pruner type coverage
discard ca30c183fba update
discard 2337ac55b65 update
discard 625640063ec fix
discard ee7a2e9039a update
discard 694b618c5ba update
discard 2578c6d5941 [fix](fe) Centralize RF partition pruning classification
discard 30a5870347c [chore](runtime-filter) Remove unused partition tablet
mapping
discard 2a7507820c4 [fix](runtime-filter) Fix projection cache lifetime
discard b3ef512c6c2 [feature](runtime-filter) Add runtime filter partition
pruning
add 9445bfa9da4 add TPartitionBoundary thrift definition
add 7e5c941524b [refactor](storage) replace TypeInfo polymorphic hierarchy
with FieldType (#63153)
add 85ede0bafba [opt](nereids) Optimize I/O operations for the IS NULL
predicate (#62304)
add 2cfa2426e41 [Fix](pyudf) Fix python udf error propagation (#62613)
add a8216b532fe [fix](fe) Fix Type.exceedsMaxNestingDepth skipping MAP
keyType recursion (#63201)
add c02bd657868 [fix](fe) Fix deep nested complex type subtype validation
bypass (#63208)
add 886b0bb0e44 [fix](memory-tracker) Avoid nested task attach in cloud
snapshot manager (#63189)
add 63e90d34e4b [fix](compaction) Fix incorrect memory availability check
in RowSourceBuffer during vertical compaction (#63152)
add 090e8fabeda [feat] Ann Index result cache (#60850)
add b82e18f23b4 [refine](column) replace misused check_and_get_column with
assert_cast (#63164)
add 8b3912260d4 [Fix](arrow flight) Fix arrow::Status inline static empty
msg core (#63191)
add c5f2693d147 [Fix](p0) rename duplicate pyudf across p0 suites (#63214)
add d441f1c434e [fix](be) Support TIMESTAMPTZ in TopN runtime predicate
(#63220)
add b30fbf99499 [fix](iow) force drop partition in INSERT OVERWRITE
(#62510)
add 12d1694c9cf [fix](statistics) Enhance OOM error message for statistics
analyze tasks (#63172)
add 5b0355545f6 [fix](fe) Fix simple aggregate cache after partition
recycle (#63175)
add beb52c117e0 [feature](RoutineLoad) Support the Amazon Kinesis (#61325)
add bb3890ac6c3 [Fix](ttz) Fix TIMESTAMPTZ elapsed-time semantics to use
UTC (#63161)
add fb27aebed63 [fix](partition_prune) Move the pruning of predicates that
are always true after partition pruning into the PlanPostProcessor (#63111)
add cf7a5880147 [fix](streaming-job) recompute derived fields after replay
and ALTER (#62936)
add a789c9941e8 [fix](job-manager) cancelTaskById should not be blocked by
unrelated streaming jobs (#62940)
add 4e25c9f7070 [fix](streaming-job) restore split-bound Java types when
reading FE-persisted CDC offset (#63219)
add 1d1846591f7 [fix](insert overwrite) delay overwrite partition routing
until incremental open (#63209)
add 73b3f5eba42 [fix](streamingjob) Persist cdc_stream TVF offset across
FE checkpoint (#62902)
add 5be6557aa56 [improve](streaming-job) add per-job lag metric to
streaming insert jobs (#63194)
add 10790e214e9 [fix](recycler) Recycler/Ms http api result msg should set
ok (#63051)
add 9e23a2fd1cd [feat](snapshot) Support storage vault for clone instance
(#63217)
add aec566e8294 [fix](fe) Define default argLine for FE unit tests (#63129)
add 8115d05b1f6 [Exec](colocate) disable colocate group execution in
bucket shuffle join (#63200)
add db0fd32c430 [chore](dep) pin esdk-obs-java-bundle to 3.21.11 to fix
version range resolution failure (#63278)
add a8fd0a117ab [fix](be) Move partitioned agg shared cleanup to shared
state (#63253)
add d80ead89ae1 [fix](fe) Reject JSONB and variant distribution columns
(#63211)
add 60b0d46dbe3 [fix](function) fix map_contains_entry runtime error when
TIMESTAMPTZ is map key or value (#63124)
add 66dbb85fe3d [feature](fe) Show compute group for MTMV refresh task
(#63206)
add a2daa074d3f [fix](be) Report string column overflow on checked append
paths (#63183)
add b9f5281b566 [fix](timestamptz) Report TIMESTAMPTZ as string to MySQL
clients (#63292)
add 170084d61e1 [fix](thrift) Reserve frontend service field ids (#63239)
add c5109e709ee [fix](be) Fix json contains duplicate array candidates
(#63301)
add 22c9ed691d5 [fix](fe) Reject lone UTF-16 surrogates in JSONB literals
(RFC 8259 ยง8.2) (#63255)
add ee0dcfe5a77 [fix](config) Remove fuzzy of string_overflow_size (#63341)
add 69aa8764dfe [fix](fe) Fix alias function with cast outermost expr and
reject illegal expressions (#63254)
add 76bbe581b96 [fix](fe) Add null reject compensation for join rewrite
(#63268)
add 451c70888d8 [fix](storage) Fix typo 'occured' in user-facing error
messages (#62521)
add 84f8107d878 [chore](window function) remove useless cast in
WindowFunctionChecker (#63284)
add ab1a4ddb592 [refine](exec) replace std::shared_mutex/std::shared_lock
with annotated wrappers for thread safety analysis (#63109)
add b6d50d4756c [doc](AGENTS) Update agents instructions for commit
messages and regressions (#63351)
add d77856dc715 [improvement](statistics) Collect hot value during full
statistics analysis (#62435)
add 92c9c21f2cf [fix](mtmv) support qualified materialized view rename
target (#63216)
add ce61c8dfe99 [Chore](be) Avoid BE crash on exception (#63312)
add 21b9fdfbe9f [fix](be) Reject super wildcard path in json keys (#63300)
add 1e1f1a4c451 [Feature] support binlog<row> read and write (row type)
(2/3) (#63110)
add 8a4f118979f [improvement](iceberg) Reconstruct partition spec in SHOW
CREATE TABLE for Iceberg tables (#63240)
add 65136532975 [fix](parquet) Fix wrong encoding for parquet page v2
(#63305)
add ea933186522 [fix](testcase) Fix wrong test case (#63354)
add 95752b78090 [refine](column) enforce nullable nested types for array
(#63088)
add c75eba5488f [fix](fe) Prune empty and cascading CTE plans (#62828)
add cbffd4f290f [test](regression) Move Iceberg REST HDFS case to P2
(#63367)
add 53132a19a96 [fix](be) Fix macOS BE build (#63230)
add f9fb157ad85 [fix](publish) Avoid NPE for force-finished publish task
(#63069)
add 87d949800b0 [fix](fe) Avoid OFFSET path and NULL path for complex
datatype when its children are accessed. (#63229)
add bd21de83599 [fix](exchange) Drain exchange receiver queues outside
lock (#63331)
add 1faf1d9fda6 [improvement](cloud) Propagate compute group id to BE
thrift payloads (#62487)
add cfe94e17d04 [Enhancement](udf) Support volatility property for scalar
UDF (#62698)
add 8f696e0ecaf [fix](be) Preserve collect aggregate limit during merge
(#63361)
add 8d3bd8a037e [fix](be) Rebind storage common expr slots (#63279)
add be49e67d7b0 [improve](streaming-job) async chunk splitting for cdc
source job (#63079)
add 4483daf9f03 [fix](be) Clean up aggregate states and use Doris hash
containers (#63174)
add 2d9ec456a7a [improvement](cloud) Add enable_recycler config to skip
recycler dynamically (#63286)
add 1891f71d5aa [fix](fe) add --drop_backends param to start_fe.sh (#63306)
add 55107434cf4 [enhancement](agent) Add more agent behavior calibration
instructions (#63395)
add f318357c820 [Fix](topn) Reject non-positive topn count argument
(#63350)
add 7ac81600cde [fix](be) Use debug point for string overflow conversion
(#63392)
add 0338e0efee8 [fix](docker) allow overriding fdb image (#63393)
add fc9dde0b263 [fix](case)Fix test_orc_lazy_mat_profile failure caused by
condition cache. (#63382)
add 2a2856189cc [fix](fe) Skip broken external tables in show tables
(#63407)
add fe71e9b7899 Merge upstream/master into dev_runtime_partition_prune
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ef2529ca3c3)
\
N -- N -- N refs/heads/dev_runtime_partition_prune (fe71e9b7899)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.claude/skills/be-code-style/SKILL.md | 9 +
AGENTS.md | 10 +-
be/src/cloud/cloud_rowset_writer.cpp | 2 +
be/src/cloud/cloud_snapshot_mgr.cpp | 6 +-
be/src/cloud/cloud_tablets_channel.cpp | 4 +
be/src/cloud/pb_convert.cpp | 32 +-
be/src/common/config.cpp | 9 +-
be/src/common/config.h | 8 +-
be/src/common/metrics/doris_metrics.cpp | 12 +
be/src/common/metrics/doris_metrics.h | 5 +
be/src/common/multi_version.h | 8 +
be/src/common/thread_safety_annotations.h | 48 +
be/src/core/column/column_string.cpp | 14 +-
be/src/core/column/column_string.h | 6 +-
be/src/core/column/column_variant.cpp | 2 +-
be/src/core/data_type/data_type_array.cpp | 17 +-
be/src/core/data_type/data_type_array.h | 11 +-
be/src/core/data_type/primitive_type.h | 2 +
.../core/data_type_serde/data_type_array_serde.cpp | 4 +-
.../core/data_type_serde/data_type_array_serde.h | 10 +-
.../core/data_type_serde/data_type_decimal_serde.h | 2 +-
.../core/data_type_serde/data_type_map_serde.cpp | 2 +-
.../data_type_serde/data_type_nullable_serde.cpp | 3 +-
.../data_type_serde/data_type_nullable_serde.h | 3 +
.../core/data_type_serde/data_type_number_serde.h | 2 +-
.../data_type_serde/data_type_variant_serde.cpp | 6 +-
be/src/core/decimal_comparison.h | 28 +-
be/src/core/value/timestamptz_value.h | 6 +
be/src/exec/common/columns_hashing.h | 3 +-
be/src/exec/common/memory.cpp | 2 +-
be/src/exec/common/variant_util.cpp | 2 +-
be/src/exec/exchange/vdata_stream_mgr.cpp | 28 +-
be/src/exec/exchange/vdata_stream_mgr.h | 14 +-
be/src/exec/exchange/vdata_stream_recvr.cpp | 55 +-
be/src/exec/exchange/vdata_stream_recvr.h | 3 +
be/src/exec/operator/join_probe_operator.cpp | 3 +-
be/src/exec/operator/olap_scan_operator.cpp | 96 +-
be/src/exec/operator/olap_scan_operator.h | 15 +-
be/src/exec/operator/operator.h | 10 -
.../partitioned_aggregation_source_operator.cpp | 7 -
be/src/exec/operator/scan_operator.cpp | 49 -
be/src/exec/operator/scan_operator.h | 38 +-
be/src/exec/pipeline/dependency.cpp | 5 +
be/src/exec/pipeline/dependency.h | 6 +-
be/src/exec/pipeline/pipeline_fragment_context.cpp | 2 +-
be/src/exec/runtime_filter/runtime_filter_mgr.cpp | 20 +-
be/src/exec/runtime_filter/runtime_filter_mgr.h | 9 +-
.../runtime_filter_partition_pruner.cpp | 875 ------------
.../runtime_filter_partition_pruner.h | 163 ---
be/src/exec/scan/olap_scanner.cpp | 64 +-
be/src/exec/scan/olap_scanner.h | 9 +-
be/src/exec/scan/parallel_scanner_builder.cpp | 6 +-
be/src/exec/scan/scanner.h | 4 -
be/src/exec/scan/scanner_scheduler.cpp | 3 -
be/src/exec/scan/scanner_scheduler.h | 2 +-
.../time_sharing/prioritized_split_runner.cpp | 26 +-
be/src/exec/sink/autoinc_buffer.cpp | 2 +
be/src/exec/sink/autoinc_buffer.h | 8 +
be/src/exec/sink/vrow_distribution.cpp | 19 +-
.../sink/writer/iceberg/partition_transformers.h | 2 +-
.../sink/writer/iceberg/viceberg_table_writer.cpp | 2 +-
be/src/exec/sink/writer/vhive_table_writer.cpp | 2 +-
be/src/exprs/aggregate/aggregate_function.h | 54 +-
.../exprs/aggregate/aggregate_function_collect.h | 13 +-
.../exprs/aggregate/aggregate_function_distinct.h | 8 +-
be/src/exprs/aggregate/aggregate_function_map.h | 2 +-
be/src/exprs/aggregate/aggregate_function_map_v2.h | 2 +-
.../aggregate_function_sequence_match.cpp | 3 +
.../aggregate/aggregate_function_sequence_match.h | 7 +-
.../aggregate/aggregate_function_window_funnel.cpp | 7 +-
.../aggregate/aggregate_function_window_funnel.h | 34 +-
.../aggregate_function_window_funnel_v2.cpp | 7 +-
.../aggregate_function_window_funnel_v2.h | 27 +-
.../function/array/function_array_aggregation.cpp | 4 +-
.../exprs/function/array/function_array_apply.cpp | 2 +-
.../exprs/function/array/function_array_concat.cpp | 4 +-
.../function/array/function_array_cum_sum.cpp | 3 +-
.../exprs/function/array/function_array_distinct.h | 2 +-
.../exprs/function/array/function_array_element.h | 2 +-
.../function/array/function_array_enumerate.cpp | 3 +-
.../array/function_array_enumerate_uniq.cpp | 5 +-
.../exprs/function/array/function_array_mapped.h | 4 +-
be/src/exprs/function/cast/cast_base.cpp | 4 +-
be/src/exprs/function/cast/cast_base.h | 1 +
be/src/exprs/function/cast/cast_to_array.h | 2 +-
.../function/cast/cast_to_basic_number_common.h | 2 +-
be/src/exprs/function/cast/cast_to_boolean.h | 6 +-
be/src/exprs/function/cast/cast_to_date.h | 6 +-
be/src/exprs/function/cast/cast_to_decimal.h | 2 +-
be/src/exprs/function/cast/cast_to_ip.h | 4 +-
.../exprs/function/function_always_not_nullable.h | 10 +-
be/src/exprs/function/function_collection_in.h | 4 +-
.../function_date_or_datetime_computation.cpp | 6 +-
.../function_date_or_datetime_computation.h | 16 +-
.../function_date_or_datetime_to_string.cpp | 4 +-
.../function/function_datetime_floor_ceil.cpp | 19 +-
be/src/exprs/function/function_hex.cpp | 6 +-
be/src/exprs/function/function_hll.cpp | 6 +-
be/src/exprs/function/function_jsonb.cpp | 11 +-
be/src/exprs/function/function_map.cpp | 7 +-
.../exprs/function/function_needs_to_handle_null.h | 4 +-
be/src/exprs/function/function_quantile_state.cpp | 17 +-
be/src/exprs/function/function_variant_element.cpp | 2 +-
.../function/functions_multi_string_position.cpp | 9 +-
.../function/functions_multi_string_search.cpp | 2 +-
be/src/exprs/function/in.h | 2 +-
be/src/exprs/function/round.h | 6 +-
be/src/exprs/vectorized_fn_call.cpp | 4 +-
be/src/exprs/vectorized_fn_call.h | 3 +-
be/src/exprs/vexpr.cpp | 2 +-
be/src/exprs/vexpr.h | 3 +-
be/src/exprs/vexpr_context.cpp | 5 +-
be/src/exprs/vexpr_context.h | 3 +-
be/src/exprs/virtual_slot_ref.cpp | 5 +-
be/src/exprs/virtual_slot_ref.h | 7 +-
be/src/exprs/vslot_ref.h | 2 +-
be/src/format/parquet/vparquet_group_reader.cpp | 3 +-
.../information_schema/schema_rowsets_scanner.cpp | 9 +-
be/src/io/fs/kafka_consumer_pipe.h | 15 -
.../src/io/fs/kinesis_consumer_pipe.h | 20 +-
be/src/io/fs/stream_load_pipe.h | 9 +
be/src/io/io_common.h | 4 +-
be/src/load/channel/tablets_channel.cpp | 38 +-
be/src/load/delta_writer/delta_writer.cpp | 14 +-
be/src/load/delta_writer/delta_writer.h | 3 +
be/src/load/delta_writer/delta_writer_context.h | 11 +-
be/src/load/delta_writer/delta_writer_v2.cpp | 2 +
be/src/load/memtable/memtable_flush_executor.cpp | 293 +++-
be/src/load/memtable/memtable_flush_executor.h | 57 +-
be/src/load/memtable/memtable_writer.cpp | 5 +-
be/src/load/routine_load/consumer_group_helpers.h | 96 ++
be/src/load/routine_load/consumer_helpers.cpp | 61 +
be/src/load/routine_load/consumer_helpers.h | 119 ++
be/src/load/routine_load/data_consumer.cpp | 578 +++++++-
be/src/load/routine_load/data_consumer.h | 114 ++
be/src/load/routine_load/data_consumer_group.cpp | 344 +++--
be/src/load/routine_load/data_consumer_group.h | 65 +-
be/src/load/routine_load/data_consumer_pool.cpp | 41 +-
be/src/load/routine_load/kinesis_conf.cpp | 169 +++
be/src/load/routine_load/kinesis_conf.h | 89 ++
.../routine_load/routine_load_task_executor.cpp | 82 +-
.../load/routine_load/routine_load_task_executor.h | 8 +
be/src/load/stream_load/stream_load_context.h | 37 +
be/src/load/stream_load/stream_load_executor.cpp | 23 +
be/src/runtime/exec_env.h | 4 +
be/src/runtime/exec_env_init.cpp | 11 +
be/src/runtime/memory/cache_policy.h | 6 +-
be/src/runtime/memory/lru_cache_policy.h | 3 +-
be/src/runtime/runtime_predicate.cpp | 2 +-
be/src/service/internal_service.cpp | 13 +
be/src/storage/binlog.h | 146 ++
be/src/storage/binlog_config.h | 4 +-
be/src/storage/compaction/compaction.cpp | 13 +
be/src/storage/data_dir.cpp | 129 +-
be/src/storage/delete/delete_bitmap_calculator.cpp | 6 +-
be/src/storage/field.h | 11 +-
be/src/storage/index/ann/CMakeLists.txt | 7 +-
be/src/storage/index/ann/ann_index_reader.cpp | 303 ++--
be/src/storage/index/ann/ann_index_reader.h | 9 +-
.../ann_index_result_cache.cpp | 298 ++++
.../ann_index_result_cache.h | 69 +
.../ann_index_result_cache_handle.cpp | 14 +-
.../ann_index_result_cache_handle.h | 26 +-
.../storage/index/ann/ann_range_search_runtime.cpp | 3 +-
be/src/storage/index/ann/ann_search_params.cpp | 49 +
be/src/storage/index/ann/ann_search_params.h | 45 +
be/src/storage/index/ann/ann_topn_runtime.cpp | 2 +
be/src/storage/index/ann/ann_topn_runtime.h | 5 +-
.../bloom_filter/bloom_filter_index_reader.cpp | 3 +-
.../index/bloom_filter/bloom_filter_index_reader.h | 6 +-
.../bloom_filter/bloom_filter_index_writer.cpp | 34 +-
.../index/bloom_filter/bloom_filter_index_writer.h | 12 +-
be/src/storage/index/index_writer.cpp | 13 +-
be/src/storage/index/indexed_column_reader.cpp | 8 +-
be/src/storage/index/indexed_column_reader.h | 5 +-
be/src/storage/index/indexed_column_writer.cpp | 12 +-
be/src/storage/index/indexed_column_writer.h | 8 +-
.../index/inverted/inverted_index_reader.cpp | 44 +-
.../storage/index/inverted/inverted_index_reader.h | 3 +-
be/src/storage/index/primary_key_index.cpp | 8 +-
be/src/storage/index/primary_key_index.h | 5 +-
be/src/storage/index/zone_map/zone_map_index.cpp | 6 +-
be/src/storage/iterator/vcollect_iterator.cpp | 4 +
.../storage/iterator/vertical_merge_iterator.cpp | 10 +-
be/src/storage/iterators.h | 1 +
be/src/storage/merger.cpp | 2 +-
be/src/storage/olap_common.h | 12 +
be/src/storage/partial_update_info.cpp | 49 +-
be/src/storage/partial_update_info.h | 24 +-
be/src/storage/predicate/comparison_predicate.h | 33 +-
be/src/storage/predicate/in_list_predicate.h | 10 +-
be/src/storage/predicate/like_column_predicate.cpp | 11 +-
be/src/storage/predicate/like_column_predicate.h | 6 +-
be/src/storage/rowset/beta_rowset_reader.cpp | 1 +
be/src/storage/rowset/beta_rowset_writer.cpp | 11 +-
be/src/storage/rowset/beta_rowset_writer.h | 11 +
be/src/storage/rowset/beta_rowset_writer_v2.h | 4 +
be/src/storage/rowset/group_rowset_writer.cpp | 43 +-
be/src/storage/rowset/group_rowset_writer.h | 13 +
be/src/storage/rowset/rowset.h | 4 +-
be/src/storage/rowset/rowset_factory.cpp | 4 +-
be/src/storage/rowset/rowset_meta.h | 21 +-
be/src/storage/rowset/rowset_meta_manager.cpp | 157 ++-
be/src/storage/rowset/rowset_meta_manager.h | 40 +-
be/src/storage/rowset/rowset_writer.h | 9 +
be/src/storage/rowset/rowset_writer_context.h | 48 +-
be/src/storage/rowset/segment_creator.cpp | 18 +-
be/src/storage/rowset/segment_creator.h | 3 +
.../storage/rowset/vertical_beta_rowset_writer.cpp | 1 +
be/src/storage/rowset_builder.cpp | 115 +-
be/src/storage/rowset_builder.h | 28 +-
be/src/storage/rowset_version_mgr.cpp | 25 +-
be/src/storage/schema.cpp | 8 +
be/src/storage/schema.h | 17 +
be/src/storage/schema_change/schema_change.cpp | 19 +-
be/src/storage/segment/column_reader.cpp | 47 +-
be/src/storage/segment/column_reader.h | 17 +-
be/src/storage/segment/column_writer.cpp | 18 +-
be/src/storage/segment/encoding_info.h | 1 -
.../storage/segment/historical_row_retriever.cpp | 288 ++++
be/src/storage/segment/historical_row_retriever.h | 134 ++
.../storage/segment/row_binlog_segment_writer.cpp | 522 +++++++
be/src/storage/segment/row_binlog_segment_writer.h | 111 ++
be/src/storage/segment/segment.cpp | 17 +-
be/src/storage/segment/segment_iterator.cpp | 272 +++-
be/src/storage/segment/segment_iterator.h | 2 +
be/src/storage/segment/segment_writer.cpp | 123 +-
be/src/storage/segment/segment_writer.h | 14 +-
.../segment/variant/hierarchical_data_iterator.cpp | 4 +-
.../segment/variant/variant_ext_meta_writer.cpp | 4 +-
be/src/storage/segment/vertical_segment_writer.cpp | 19 +-
be/src/storage/segment/virtual_column_iterator.h | 2 +
be/src/storage/snapshot/snapshot_manager.cpp | 4 +-
be/src/storage/storage_engine.cpp | 54 +-
be/src/storage/tablet/base_tablet.cpp | 164 ++-
be/src/storage/tablet/base_tablet.h | 9 +-
be/src/storage/tablet/tablet.cpp | 143 +-
be/src/storage/tablet/tablet.h | 14 +-
be/src/storage/tablet/tablet_manager.cpp | 12 +
be/src/storage/tablet/tablet_meta_manager.cpp | 46 +-
be/src/storage/tablet/tablet_meta_manager.h | 6 +
be/src/storage/tablet/tablet_reader.cpp | 6 +-
be/src/storage/tablet/tablet_schema.cpp | 2 +-
be/src/storage/tablet/tablet_schema.h | 9 +
be/src/storage/tablet_info.cpp | 49 +
be/src/storage/tablet_info.h | 3 +
.../storage/task/engine_publish_version_task.cpp | 46 +-
be/src/storage/task/engine_publish_version_task.h | 4 +-
be/src/storage/task/index_builder.cpp | 12 +-
be/src/storage/txn/txn_manager.cpp | 108 +-
be/src/storage/txn/txn_manager.h | 17 +-
be/src/storage/types.cpp | 203 ---
be/src/storage/types.h | 286 +---
be/src/udf/python/python_server.py | 25 +-
be/src/udf/python/python_udaf_client.cpp | 82 +-
be/src/udf/python/python_udaf_client.h | 10 +
be/src/util/jsonb_document.h | 48 +-
be/src/util/s3_util.h | 5 +-
be/src/util/thrift_util.cpp | 13 +
be/src/util/thrift_util.h | 1 +
be/test/core/block/block_test.cpp | 6 +-
be/test/core/column/column_array_test.cpp | 2 +-
be/test/core/column/column_string_test.cpp | 22 +-
.../data_type_serde_get_name_test.cpp | 4 +-
.../data_type_serde_string_test.cpp | 8 +-
be/test/core/jsonb/serialize_test.cpp | 6 +-
be/test/core/value/merge_partitioner_test.cpp | 4 +-
.../data/vec/native/all_types_single_row.native | Bin 1124 -> 1140 bytes
be/test/exec/common/schema_util_test.cpp | 4 +-
.../time_sharing_task_executor_test.cpp | 43 +
.../exec/operator/table_function_operator_test.cpp | 55 +-
.../pipeline/partitioned_agg_shared_state_test.cpp | 4 +-
be/test/exec/pipeline/vdata_stream_recvr_test.cpp | 111 +-
.../runtime_filter_partition_pruner_test.cpp | 451 ------
be/test/exec/sink/vrow_distribution_test.cpp | 45 +-
be/test/exprs/aggregate/agg_array_agg_test.cpp | 70 -
be/test/exprs/aggregate/agg_collect_test.cpp | 72 +-
be/test/exprs/aggregate/agg_replace_test.cpp | 6 +-
.../aggregate_function_exception_test.cpp | 162 +++
be/test/exprs/function/function_jsonb_test.cpp | 74 +-
be/test/exprs/hybrid_set_test.cpp | 17 +-
be/test/exprs/minmax_predicate_test.cpp | 19 +-
be/test/format/orc/orc_reader_fill_data_test.cpp | 48 +-
be/test/format/parquet/parquet_thrift_test.cpp | 31 +
.../load/memtable/memtable_flush_executor_test.cpp | 334 ++++-
be/test/olap/rowset/group_rowset_builder_test.cpp | 112 +-
be/test/olap/rowset/group_rowset_writer_test.cpp | 225 +++
.../compaction/vertical_compaction_test.cpp | 77 +
.../index/ann/ann_index_result_cache_test.cpp | 769 ++++++++++
be/test/storage/index/ann/ann_index_smoke_test.cpp | 4 -
.../storage/index/ann/ann_range_search_test.cpp | 6 +-
.../storage/index/ann/ann_topn_descriptor_test.cpp | 2 +-
.../index/ann/ann_topn_runtime_negative_test.cpp | 2 +-
.../compaction/util/index_compaction_utils.cpp | 6 +-
be/test/storage/index/primary_key_index_test.cpp | 4 +-
.../storage/rowset/rowset_meta_manager_test.cpp | 139 ++
.../bloom_filter_index_reader_writer_test.cpp | 30 +-
be/test/storage/segment/encoding_info_test.cpp | 36 +-
.../segment/inverted_index_file_writer_test.cpp | 1 +
be/test/storage/storage_types_test.cpp | 12 +-
be/test/storage/txn/txn_manager_test.cpp | 119 +-
be/test/testutil/creators.h | 148 +-
be/test/udf/python/python_udaf_client_test.cpp | 155 ++
be/test/util/jsonb_contains_test.cpp | 64 +
bin/start_fe.sh | 14 +-
cloud/src/common/config.h | 2 +
cloud/src/common/http_helper.cpp | 8 +-
cloud/src/meta-service/meta_service_http.cpp | 10 +-
cloud/src/meta-service/meta_service_http.h | 6 +-
cloud/src/meta-service/meta_service_resource.cpp | 520 +++++--
cloud/src/recycler/recycler.cpp | 45 +-
cloud/test/meta_service_http_test.cpp | 6 +-
cloud/test/recycler_test.cpp | 81 ++
cloud/test/resource_test.cpp | 166 +++
docker/runtime/doris-compose/command.py | 10 +-
docker/runtime/doris-compose/resource/init_fe.sh | 2 +-
.../postgresql/postgresql-14.yaml.tpl | 4 +-
.../org/apache/doris/analysis/JsonLiteral.java | 39 +-
.../org/apache/doris/analysis/SlotDescriptor.java | 4 +
.../main/java/org/apache/doris/catalog/Column.java | 1 +
.../java/org/apache/doris/catalog/Function.java | 11 +
.../apache/doris/catalog/FunctionVolatility.java} | 30 +-
.../main/java/org/apache/doris/common/Config.java | 5 +-
.../job/cdc/request/FetchTableSplitsRequest.java | 12 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 5 +-
.../src/main/java/org/apache/doris/DorisFE.java | 5 +
.../apache/doris/alter/SchemaChangeHandler.java | 6 +
.../doris/analysis/HashDistributionDesc.java | 14 +-
.../doris/catalog/AutoIncrementGenerator.java | 4 +
.../doris/catalog/BuiltinTableValuedFunctions.java | 2 +
.../main/java/org/apache/doris/catalog/Env.java | 34 +
.../org/apache/doris/catalog/FunctionGenTable.java | 8 +
.../doris/catalog/FunctionToSqlConverter.java | 67 +-
.../apache/doris/catalog/HashDistributionInfo.java | 23 +
.../java/org/apache/doris/catalog/OlapTable.java | 28 +-
.../org/apache/doris/catalog/OlapTableWrapper.java | 2 +
.../doris/catalog/RowBinlogTableWrapper.java | 18 +-
.../java/org/apache/doris/catalog/SchemaTable.java | 2 +-
.../org/apache/doris/catalog/TableAttributes.java | 4 +-
.../apache/doris/cloud/transaction/TxnUtil.java | 2 +-
.../java/org/apache/doris/common/FeConstants.java | 1 +
.../apache/doris/datasource/ExternalDatabase.java | 11 +-
.../apache/doris/datasource/InternalCatalog.java | 13 +-
.../doris/datasource/doris/FeServiceClient.java | 4 +-
.../datasource/iceberg/IcebergExternalTable.java | 50 +
.../doris/datasource/kinesis/KinesisUtil.java | 165 +++
.../AbstractInsertOverwriteManager.java | 2 +-
.../insertoverwrite/InsertOverwriteManager.java | 4 +-
.../RemoteInsertOverwriteManager.java | 4 +-
.../insert/streaming/StreamingInsertJob.java | 77 +-
.../streaming/StreamingJobSchedulerTask.java | 7 +
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 31 +-
.../doris/job/extensions/mtmv/MTMVTaskContext.java | 12 +-
.../org/apache/doris/job/manager/JobManager.java | 6 +-
.../doris/job/offset/SourceOffsetProvider.java | 24 +-
.../job/offset/jdbc/JdbcSourceOffsetProvider.java | 503 +++++--
.../offset/jdbc/JdbcTvfSourceOffsetProvider.java | 131 +-
.../apache/doris/job/util/StreamingJobUtils.java | 72 +-
.../java/org/apache/doris/load/DeleteHandler.java | 7 +
.../doris/load/routineload/LoadDataSourceType.java | 3 +-
.../routineload/RLTaskTxnCommitAttachment.java | 5 +
.../RoutineLoadDataSourcePropertyFactory.java | 9 +-
.../doris/load/routineload/RoutineLoadJob.java | 8 +-
.../doris/load/routineload/RoutineLoadManager.java | 5 +
.../load/routineload/RoutineLoadProgress.java | 2 +-
.../load/routineload/RoutineLoadTaskInfo.java | 6 +-
.../kafka/KafkaDataSourceProperties.java | 1 -
.../routineload/{ => kafka}/KafkaProgress.java | 5 +-
.../{ => kafka}/KafkaRoutineLoadJob.java | 10 +-
.../routineload/{ => kafka}/KafkaTaskInfo.java | 7 +-
.../routineload/kinesis/KinesisConfiguration.java | 169 +++
.../kinesis/KinesisDataSourceProperties.java | 411 ++++++
.../load/routineload/kinesis/KinesisProgress.java | 334 +++++
.../routineload/kinesis/KinesisRoutineLoadJob.java | 893 ++++++++++++
.../KinesisTaskInfo.java} | 170 ++-
.../java/org/apache/doris/metric/MetricRepo.java | 14 +
.../java/org/apache/doris/mtmv/MTMVJobManager.java | 25 +-
.../org/apache/doris/mysql/MysqlSerializer.java | 7 +-
.../org/apache/doris/nereids/CascadesContext.java | 9 +
.../org/apache/doris/nereids/StatementContext.java | 11 +
.../RuntimeFilterPartitionPruneClassifier.java | 295 ----
.../glue/translator/RuntimeFilterTranslator.java | 54 +-
.../doris/nereids/jobs/executor/Optimizer.java | 78 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 7 +-
.../nereids/processor/post/PlanPostProcessors.java | 1 +
.../processor/post/PrunePartitionPredicate.java | 146 ++
.../post/PushDownFilterThroughProject.java | 2 +-
.../nereids/rules/analysis/BindExpression.java | 10 +-
.../rules/analysis/WindowFunctionChecker.java | 28 +-
.../mv/AbstractMaterializedViewRule.java | 173 ++-
.../exploration/mv/MaterializedViewUtils.java | 11 +
.../nereids/rules/exploration/mv/StructInfo.java | 2 +-
.../exploration/mv/SyncMaterializationContext.java | 11 +-
.../expression/rules/FoldConstantRuleOnBE.java | 4 +-
.../rules/InPredicateExtractNonConstant.java | 2 +-
.../rules/expression/rules/PartitionPruner.java | 23 -
.../expression/rules/PushIntoCaseWhenBranch.java | 4 +-
.../LogicalOlapScanToPhysicalOlapScan.java | 3 +-
.../rewrite/AccessPathExpressionCollector.java | 118 +-
.../nereids/rules/rewrite/AccessPathInfo.java | 3 +
.../rules/rewrite/AccessPathPlanCollector.java | 37 +-
.../rules/rewrite/AddProjectForUniqueFunction.java | 8 +-
.../doris/nereids/rules/rewrite/CTEInliner.java | 86 +-
.../CheckAndStandardizeWindowFunctionAndFrame.java | 5 +-
.../nereids/rules/rewrite/ClearContextStatus.java | 5 +-
.../rules/rewrite/CollectFilterAboveConsumer.java | 2 +-
.../rules/rewrite/JoinExtractOrFromCaseWhen.java | 2 +-
.../nereids/rules/rewrite/NestedColumnPruning.java | 531 ++++++-
.../rules/rewrite/PruneOlapScanPartition.java | 30 +-
.../rewrite/PushDownFilterThroughAggregation.java | 2 +-
.../rewrite/PushDownFilterThroughGenerate.java | 2 +-
.../rules/rewrite/PushDownFilterThroughJoin.java | 4 +-
.../rewrite/PushDownFilterThroughProject.java | 2 +-
.../rules/rewrite/PushFilterInsideJoin.java | 2 +-
.../rules/rewrite/PushProjectIntoUnion.java | 2 +-
.../doris/nereids/rules/rewrite/ReorderJoin.java | 4 +-
.../nereids/rules/rewrite/SlotTypeReplacer.java | 3 +-
.../rewrite/eageraggregation/EagerAggRewriter.java | 2 +-
.../doris/nereids/stats/SimpleAggCacheMgr.java | 12 +
.../trees/expressions/ArrayItemReference.java | 3 +-
.../nereids/trees/expressions/Expression.java | 6 +-
.../trees/expressions/VolatileExpression.java} | 13 +-
.../trees/expressions/VolatileIdentity.java | 90 ++
.../functions/AggCombinerFunctionBuilder.java | 3 +-
.../functions/ComputeSignatureHelper.java | 2 +-
.../expressions/functions/ExpressionTrait.java | 2 +-
.../expressions/functions/agg/SequenceCount.java | 3 +
.../functions/agg/SequenceFunction.java | 2 +-
.../expressions/functions/agg/SequenceMatch.java | 3 +
.../trees/expressions/functions/agg/TopN.java | 14 +
.../trees/expressions/functions/agg/TopNArray.java | 14 +
.../expressions/functions/agg/TopNWeighted.java | 22 +
.../expressions/functions/agg/WindowFunnel.java | 15 +-
.../expressions/functions/agg/WindowFunnelV2.java | 15 +-
.../functions/combinator/ForEachCombinator.java | 2 +-
.../expressions/functions/scalar/ArrayMap.java | 2 +-
.../expressions/functions/scalar/ArraySort.java | 4 +-
.../expressions/functions/scalar/DateDiff.java | 3 +
.../expressions/functions/scalar/DaysDiff.java | 2 +
.../expressions/functions/scalar/HoursDiff.java | 3 +
.../functions/scalar/MicroSecondsDiff.java | 3 +
.../functions/scalar/MilliSecondsDiff.java | 3 +
.../expressions/functions/scalar/MinutesDiff.java | 3 +
.../expressions/functions/scalar/MonthsDiff.java | 7 +-
.../expressions/functions/scalar/QuartersDiff.java | 7 +-
.../expressions/functions/scalar/SecondsDiff.java | 3 +
.../expressions/functions/scalar/TimeDiff.java | 11 +
.../functions/scalar/UniqueFunction.java | 23 +-
.../expressions/functions/scalar/WeeksDiff.java | 7 +-
.../expressions/functions/scalar/YearsDiff.java | 7 +-
.../trees/expressions/functions/table/Binlog.java | 57 +
.../trees/expressions/functions/udf/AliasUdf.java | 17 +-
.../trees/expressions/functions/udf/JavaUdf.java | 65 +-
.../expressions/functions/udf/JavaUdfBuilder.java | 2 +-
.../trees/expressions/functions/udf/PythonUdf.java | 65 +-
.../functions/udf/PythonUdfBuilder.java | 4 +-
.../trees/expressions/literal/JsonLiteral.java | 39 +-
.../visitor/TableValuedFunctionVisitor.java | 5 +
.../trees/plans/PartitionPrunablePredicate.java | 95 ++
.../plans/commands/CreateFunctionCommand.java | 42 +-
.../trees/plans/commands/ShowFunctionsCommand.java | 11 +-
.../plans/commands/info/AlterMTMVRenameInfo.java | 15 +-
.../insert/InsertOverwriteTableCommand.java | 4 +-
.../job/UnassignedScanBucketOlapTableJob.java | 7 +
.../trees/plans/logical/LogicalAggregate.java | 4 +-
.../trees/plans/logical/LogicalLoadProject.java | 2 +-
.../trees/plans/logical/LogicalOlapScan.java | 100 +-
.../trees/plans/logical/LogicalProject.java | 2 +-
.../physical/PhysicalBucketedHashAggregate.java | 4 +-
.../plans/physical/PhysicalHashAggregate.java | 4 +-
.../trees/plans/physical/PhysicalOlapScan.java | 63 +-
.../org/apache/doris/nereids/types/ArrayType.java | 28 +-
.../org/apache/doris/nereids/types/DataType.java | 27 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 11 +-
.../org/apache/doris/nereids/util/PlanUtils.java | 2 +-
.../doris/nereids/util/TypeCoercionUtils.java | 3 +-
.../org/apache/doris/persist/gson/GsonUtils.java | 18 +-
.../org/apache/doris/planner/OlapScanNode.java | 196 +--
.../org/apache/doris/planner/OlapTableSink.java | 17 +
.../org/apache/doris/planner/RuntimeFilter.java | 76 -
.../java/org/apache/doris/qe/SessionVariable.java | 10 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 1 -
.../apache/doris/service/FrontendServiceImpl.java | 8 +-
.../doris/statistics/AnalysisTaskWrapper.java | 16 +-
.../apache/doris/statistics/BaseAnalysisTask.java | 25 +-
.../doris/statistics/ExternalAnalysisTask.java | 6 +-
.../apache/doris/statistics/OlapAnalysisTask.java | 6 +-
.../doris/tablefunction/TableBinlogFunction.java | 211 +++
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
.../apache/doris/task/AgentTaskCleanupDaemon.java | 2 +-
.../org/apache/doris/task/PublishVersionTask.java | 7 +-
.../doris/transaction/DatabaseTransactionMgr.java | 6 +-
.../doris/alter/SchemaChangeHandlerTest.java | 12 +
.../apache/doris/catalog/CreateFunctionTest.java | 23 +
.../org/apache/doris/catalog/CreateTableTest.java | 23 +-
.../doris/catalog/FunctionToSqlConverterTest.java | 136 +-
.../java/org/apache/doris/catalog/TypeTest.java | 46 +-
.../iceberg/IcebergExternalTableTest.java | 160 ++-
.../StreamingInsertJobCheckDataQualityTest.java | 54 +
.../apache/doris/job/manager/JobManagerTest.java | 39 +
.../JdbcSourceOffsetProviderAsyncSplitTest.java | 486 +++++++
.../doris/job/offset/jdbc/SplitProgressTest.java | 60 +
.../load/routineload/KafkaRoutineLoadJobTest.java | 2 +
.../routineload/KinesisRoutineLoadJobTest.java | 339 +++++
.../doris/load/routineload/RoutineLoadJobTest.java | 3 +
.../load/routineload/RoutineLoadManagerTest.java | 1 +
.../load/routineload/RoutineLoadSchedulerTest.java | 2 +
.../routineload/RoutineLoadTaskSchedulerTest.java | 3 +
.../kinesis/KinesisDataSourcePropertiesTest.java | 83 ++
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 29 +-
.../org/apache/doris/mtmv/MTMVJobManagerTest.java | 88 ++
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 88 +-
.../doris/mysql/MysqlSerializerVarbinaryTest.java | 30 +
.../exploration/mv/NullRejectInferenceTest.java | 120 +-
.../doris/nereids/rules/rewrite/CTEInlineTest.java | 38 +
.../rewrite/CollectFilterAboveConsumerTest.java | 2 +-
.../nereids/rules/rewrite/PartitionPrunerTest.java | 256 ----
.../rules/rewrite/PruneNestedColumnTest.java | 510 +++++--
.../nereids/trees/expressions/LiteralTest.java | 12 +-
.../doris/nereids/trees/expressions/UdfTest.java | 31 +
.../functions/scalar/UniqueFunctionTest.java | 2 +-
.../functions/udf/UdfVolatilityTest.java | 91 ++
.../trees/expressions/literal/JsonLiteralTest.java | 124 ++
.../plans/commands/ShowFunctionsCommandTest.java | 50 +
.../commands/info/DistributionDescriptorTest.java | 48 +
.../job/UnassignedScanBucketOlapTableJobTest.java | 117 ++
.../apache/doris/nereids/types/DataTypeTest.java | 54 +
.../apache/doris/resource/ComputeGroupTest.java | 2 +-
.../doris/statistics/AnalysisTaskWrapperTest.java | 84 ++
.../doris/statistics/HMSAnalysisTaskTest.java | 72 +-
.../doris/statistics/OlapAnalysisTaskTest.java | 122 +-
.../apache/doris/task/PublishVersionTaskTest.java | 109 ++
.../CheckReplicaContinuousVersionSuccTest.java | 177 +++
.../transaction/GlobalTransactionMgrTest.java | 7 +-
.../doris/transaction/TransactionStateTest.java | 2 +-
.../java/org/apache/doris/catalog/ArrayType.java | 57 +-
.../org/apache/doris/catalog/PrimitiveType.java | 5 +-
.../main/java/org/apache/doris/catalog/Type.java | 3 +
fe/pom.xml | 3 +-
.../source/reader/AbstractCdcSourceReader.java | 45 +
.../source/reader/JdbcIncrementalSourceReader.java | 255 ++--
.../source/reader/mysql/MySqlSourceReader.java | 270 ++--
.../reader/postgres/PostgresSourceReader.java | 23 +
.../source/reader/AbstractCdcSourceReaderTest.java | 165 +++
gensrc/proto/descriptors.proto | 2 +
gensrc/proto/internal_service.proto | 24 +
gensrc/proto/olap_file.proto | 17 +-
gensrc/thrift/BackendService.thrift | 13 +
gensrc/thrift/Descriptors.thrift | 2 +
gensrc/thrift/FrontendService.thrift | 38 +
gensrc/thrift/HeartbeatService.thrift | 1 +
gensrc/thrift/PaloInternalService.thrift | 1 +
gensrc/thrift/PlanNodes.thrift | 59 +-
gensrc/thrift/Types.thrift | 2 +
.../test_insert_overwrite_multiple.out | 15 -
.../test_insert_overwrite_recover.out | 14 -
.../test_insert_overwrite_recover_multiple.out | 23 -
.../test_insert_overwrite_recover_no_partition.out | 14 -
.../timestamptz/test_cast_timestamptz.out | 4 +-
.../timestamptz/test_timestamptz_cast.out | 12 +-
.../test_timestamptz_map_contains_entry.out | 43 +
.../timestamptz/test_timestamptz_sort.out | 104 ++
.../timestamptz/test_timestamptz_utc_functions.out | 33 +
.../sql-functions/doc_date_functions_test.out | 48 +-
.../paimon/test_paimon_catalog_timestamp_tz.out | 12 +-
...doris_25531_string_overflow_fault_injection.out | 7 +
.../cdc/test_streaming_mysql_job_date_pk.out | 29 +
.../cdc/test_streaming_postgres_job_date_pk.out | 29 +
..._cdc_stream_postgres_checkpoint_restart_fe.out} | 0
.../column_pruning/null_column_pruning.out | 107 ++
.../string_length_column_pruning.out | 9 +
.../distinct_split/disitinct_split.out | 57 +-
.../mv/unsafe_equals/null_un_safe_equals.out | 12 +
.../prune_predicates_mv_test.out} | 19 +-
.../unique_function/agg_with_unique_function.out | 8 +-
.../query_p0/repeat/test_repeat_output_slot.out | 28 +-
.../runtime_filter/rf_partition_pruning.out | 125 --
.../query_p0/system/test_query_sys_rowsets.out | 3 +-
.../system/test_query_sys_scan_rowsets.out | 3 +-
.../test_parquet_dict.out} | 11 +-
.../data/row_binlog_p0/test_row_binlog_basic.out | 53 +
.../test_row_binlog_multi_segment.out | 36 +
.../test_row_binlog_publish_conflict.out | 40 +
.../test_row_binlog_schema_change.out | 20 +
regression-test/framework/pom.xml | 27 +
.../pipeline/nonConcurrent/conf/fe.conf | 2 +
.../ann_index_p0/ann_index_cache_additional.groovy | 248 ++++
.../ann_index_p0/ann_index_result_cache.groovy | 236 ++++
.../ann_index_p0/ann_index_topn_cache.groovy | 185 +++
.../test_insert_overwrite_multiple.groovy | 68 -
.../test_insert_overwrite_recover.groovy | 64 -
.../test_insert_overwrite_recover_multiple.groovy | 87 --
...st_insert_overwrite_recover_no_partition.groovy | 78 --
.../test_complex_disallowed_subtypes.groovy | 77 +
.../test_timestamptz_jdbc_binary_protocol.groovy | 85 ++
.../test_timestamptz_map_contains_entry.groovy | 155 ++
.../timestamptz/test_timestamptz_sort.groovy | 134 ++
.../test_timestamptz_utc_functions.groovy | 183 +++
.../hive/test_orc_lazy_mat_profile.groovy | 1 +
.../tvf/test_hdfs_parquet_group0.groovy | 4 +-
.../iceberg_rest_on_hdfs.groovy | 4 +-
...is_25531_string_overflow_fault_injection.groovy | 62 +
.../test_publish_no_npe_when_be_down.groovy | 142 ++
.../suites/javaudf_p0/test_javaudf_float.groovy | 3 +-
.../test_streaming_mysql_job_async_split.groovy | 160 +++
...t_streaming_mysql_job_async_split_uneven.groovy | 195 +++
.../cdc/test_streaming_mysql_job_date_pk.groovy | 129 ++
.../cdc/test_streaming_mysql_job_metrics.groovy | 13 +-
...ing_postgres_job_async_split_multi_table.groovy | 178 +++
...ng_postgres_job_async_split_pause_resume.groovy | 172 +++
...treaming_postgres_job_async_split_uneven.groovy | 198 +++
...stgres_job_async_split_uneven_restart_fe.groovy | 207 +++
...streaming_postgres_job_binlog_restart_fe.groovy | 191 +++
.../cdc/test_streaming_postgres_job_date_pk.groovy | 131 ++
...st_streaming_postgres_job_special_offset.groovy | 15 +-
...ming_job_cdc_stream_postgres_async_split.groovy | 193 +++
...c_stream_postgres_async_split_restart_fe.groovy | 218 +++
...dc_stream_postgres_checkpoint_restart_fe.groovy | 298 ++++
.../jsonb_p0/test_jsonb_distribution_column.groovy | 46 +
.../jsonb_p0/test_jsonb_keys_invalid_path.groovy | 30 +-
.../test_kinesis_routine_load.groovy | 307 ++++
.../test_kinesis_routine_load_be_restart.groovy | 255 ++++
..._kinesis_routine_load_data_quality_error.groovy | 400 ++++++
.../test_kinesis_routine_load_fe_restart.groovy | 250 ++++
.../test_kinesis_routine_load_pause_resume.groovy | 265 ++++
.../test_kinesis_routine_load_property.groovy | 255 ++++
.../test_kinesis_routine_load_shard_change.groovy | 407 ++++++
.../test_kinesis_show_routine_load.groovy | 318 +++++
.../suites/mtmv_p0/test_expand_star_mtmv.groovy | 3 +-
.../suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy | 14 +-
.../agg_function/topn/topn.groovy | 32 +
.../column_pruning/null_column_pruning.groovy | 532 +++++++
.../string_length_column_pruning.groovy | 89 +-
.../mv/dimension/dimension_self_conn.groovy | 5 +-
.../inner_join_null_reject_compensation.groovy | 217 +++
.../mv/unsafe_equals/null_un_safe_equals.groovy | 17 +-
.../prune_predicates_mv_test.groovy | 182 +++
.../predicate_infer/infer_predicate.groovy | 36 +
.../auto_partition_recycle_count_cache.groovy | 91 ++
.../test_python_raise_error_propagation.groovy | 269 ++++
.../suites/pythonudaf_p0/udaf_scripts/pyudaf.zip | Bin 7858 -> 9153 bytes
.../pythonudaf_p0/udaf_scripts/udaf_errors.py | 29 +-
.../test_python_udf_business_logic.groovy | 8 +-
.../pythonudf_p0/test_pythonudf_aggregate.groovy | 6 +-
.../test_pythonudf_file_protocol.groovy | 11 +-
.../pythonudf_p0/test_pythonudf_float.groovy | 3 +-
.../test_pythonudf_inline_complex.groovy | 10 +-
.../test_pythonudf_mixed_params.groovy | 12 +-
.../pythonudf_p0/test_pythonudf_volatility.groovy | 204 +++
.../pythonudf_p0/udf_scripts/array_int_test.py | 2 +
.../udf_scripts/array_return_array_int_test.py | 2 +
.../udf_scripts/array_return_array_string_test.py | 2 +
.../pythonudf_p0/udf_scripts/array_string_test.py | 2 +
.../suites/pythonudf_p0/udf_scripts/float_test.py | 2 +
.../suites/pythonudf_p0/udf_scripts/int_test.py | 2 +
.../suites/pythonudf_p0/udf_scripts/pyudf.zip | Bin 6086 -> 15967 bytes
.../{array_int_test.py => udf_errors.py} | 10 +-
.../suites/pythonudtf_p0/udtf_scripts/pyudtf.zip | Bin 10216 -> 10924 bytes
.../udtf_scripts/pyudtf_module/exceptions_udtf.py | 7 +
.../query_p0/cte/test_cbo_cte_inline_prune.groovy | 113 ++
.../runtime_filter/rf_partition_pruning.groovy | 1474 --------------------
.../rf_partition_pruning_type_matrix.groovy | 219 ---
.../suites/query_p0/test_parquet_dict.groovy | 28 +-
.../row_binlog_p0/test_row_binlog_basic.groovy | 264 ++++
.../test_row_binlog_multi_segment.groovy | 239 ++++
.../test_row_binlog_publish_conflict.groovy | 303 ++++
.../test_row_binlog_schema_change.groovy | 110 ++
.../statistics/test_full_analyze_hot_value.groovy | 166 +++
.../suites/statistics/test_hot_value.groovy | 20 +-
regression-test/suites/tso_p0/test_tso_api.groovy | 169 +--
.../tso_p0/test_tso_rowset_commit_tso.groovy | 70 +-
.../suites/variant_p0/doc_mode/load.groovy | 2 +-
regression-test/suites/variant_p0/load.groovy | 2 +-
thirdparty/download-thirdparty.sh | 10 +-
...he-arrow-17.0.0-flight-safe-finish-status.patch | 216 ---
...che-arrow-17.0.0-status-inline-static-fix.patch | 58 +
676 files changed, 29742 insertions(+), 8673 deletions(-)
delete mode 100644
be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp
delete mode 100644 be/src/exec/runtime_filter/runtime_filter_partition_pruner.h
copy
fe/fe-core/src/main/java/org/apache/doris/load/routineload/LoadDataSourceType.java
=> be/src/io/fs/kinesis_consumer_pipe.h (66%)
create mode 100644 be/src/load/routine_load/consumer_group_helpers.h
create mode 100644 be/src/load/routine_load/consumer_helpers.cpp
create mode 100644 be/src/load/routine_load/consumer_helpers.h
create mode 100644 be/src/load/routine_load/kinesis_conf.cpp
create mode 100644 be/src/load/routine_load/kinesis_conf.h
create mode 100644
be/src/storage/index/ann/ann_index_result_cache/ann_index_result_cache.cpp
create mode 100644
be/src/storage/index/ann/ann_index_result_cache/ann_index_result_cache.h
copy
fe/fe-core/src/main/java/org/apache/doris/load/routineload/LoadDataSourceType.java
=>
be/src/storage/index/ann/ann_index_result_cache/ann_index_result_cache_handle.cpp
(65%)
copy
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadProgress.java
=>
be/src/storage/index/ann/ann_index_result_cache/ann_index_result_cache_handle.h
(63%)
create mode 100644 be/src/storage/index/ann/ann_search_params.cpp
create mode 100644 be/src/storage/segment/historical_row_retriever.cpp
create mode 100644 be/src/storage/segment/historical_row_retriever.h
create mode 100644 be/src/storage/segment/row_binlog_segment_writer.cpp
create mode 100644 be/src/storage/segment/row_binlog_segment_writer.h
delete mode 100644
be/test/exec/runtime_filter/runtime_filter_partition_pruner_test.cpp
create mode 100644
be/test/exprs/aggregate/aggregate_function_exception_test.cpp
create mode 100644 be/test/olap/rowset/group_rowset_writer_test.cpp
create mode 100644 be/test/storage/index/ann/ann_index_result_cache_test.cpp
create mode 100644 be/test/udf/python/python_udaf_client_test.cpp
create mode 100644 be/test/util/jsonb_contains_test.cpp
copy fe/{fe-core/src/main/java/org/apache/doris/catalog/FunctionGenTable.java
=> fe-catalog/src/main/java/org/apache/doris/catalog/FunctionVolatility.java}
(54%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/kinesis/KinesisUtil.java
rename fe/fe-core/src/main/java/org/apache/doris/load/routineload/{ =>
kafka}/KafkaProgress.java (97%)
rename fe/fe-core/src/main/java/org/apache/doris/load/routineload/{ =>
kafka}/KafkaRoutineLoadJob.java (99%)
copy fe/fe-core/src/main/java/org/apache/doris/load/routineload/{ =>
kafka}/KafkaTaskInfo.java (97%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisConfiguration.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourceProperties.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisProgress.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java
rename
fe/fe-core/src/main/java/org/apache/doris/load/routineload/{KafkaTaskInfo.java
=> kinesis/KinesisTaskInfo.java} (61%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
copy
fe/fe-core/src/main/java/org/apache/doris/{load/routineload/LoadDataSourceType.java
=> nereids/trees/expressions/VolatileExpression.java} (68%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VolatileIdentity.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Binlog.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableBinlogFunction.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderAsyncSplitTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/SplitProgressTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/load/routineload/kinesis/KinesisDataSourcePropertiesTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVJobManagerTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/JsonLiteralTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJobTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskWrapperTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/task/PublishVersionTaskTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/transaction/CheckReplicaContinuousVersionSuccTest.java
create mode 100644
fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/AbstractCdcSourceReaderTest.java
delete mode 100644
regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
delete mode 100644
regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover.out
delete mode 100644
regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
delete mode 100644
regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
create mode 100644
regression-test/data/datatype_p0/timestamptz/test_timestamptz_map_contains_entry.out
create mode 100644
regression-test/data/datatype_p0/timestamptz/test_timestamptz_utc_functions.out
create mode 100644
regression-test/data/fault_injection_p0/test_doris_25531_string_overflow_fault_injection.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_date_pk.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_postgres_job_date_pk.out
copy
regression-test/data/job_p0/streaming_job/cdc/tvf/{test_streaming_job_cdc_stream_postgres_restart_fe.out
=> test_streaming_job_cdc_stream_postgres_checkpoint_restart_fe.out} (100%)
create mode 100644
regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out
copy
regression-test/data/nereids_rules_p0/{mv/unsafe_equals/null_un_safe_equals.out
=> partition_prune/prune_predicates_mv_test.out} (52%)
delete mode 100644
regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out
copy
regression-test/data/{nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out
=> query_p0/test_parquet_dict.out} (52%)
create mode 100644 regression-test/data/row_binlog_p0/test_row_binlog_basic.out
create mode 100644
regression-test/data/row_binlog_p0/test_row_binlog_multi_segment.out
create mode 100644
regression-test/data/row_binlog_p0/test_row_binlog_publish_conflict.out
create mode 100644
regression-test/data/row_binlog_p0/test_row_binlog_schema_change.out
create mode 100644
regression-test/suites/ann_index_p0/ann_index_cache_additional.groovy
create mode 100644
regression-test/suites/ann_index_p0/ann_index_result_cache.groovy
create mode 100644
regression-test/suites/ann_index_p0/ann_index_topn_cache.groovy
delete mode 100644
regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
delete mode 100644
regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
delete mode 100644
regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy
delete mode 100644
regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
create mode 100644
regression-test/suites/datatype_p0/complex_types/test_complex_disallowed_subtypes.groovy
create mode 100644
regression-test/suites/datatype_p0/timestamptz/test_timestamptz_jdbc_binary_protocol.groovy
create mode 100644
regression-test/suites/datatype_p0/timestamptz/test_timestamptz_map_contains_entry.groovy
create mode 100644
regression-test/suites/datatype_p0/timestamptz/test_timestamptz_utc_functions.groovy
rename regression-test/suites/{external_table_p0/refactor_storage_param =>
external_table_p2/refactor_catalog_param}/iceberg_rest_on_hdfs.groovy (99%)
create mode 100644
regression-test/suites/fault_injection_p0/test_doris_25531_string_overflow_fault_injection.groovy
create mode 100644
regression-test/suites/fault_injection_p0/test_publish_no_npe_when_be_down.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_async_split_uneven.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_date_pk.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_multi_table.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_pause_resume.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_async_split_uneven_restart_fe.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_binlog_restart_fe.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_date_pk.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_async_split_restart_fe.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_checkpoint_restart_fe.groovy
create mode 100644
regression-test/suites/jsonb_p0/test_jsonb_distribution_column.groovy
copy fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionGenTable.java
=> regression-test/suites/jsonb_p0/test_jsonb_keys_invalid_path.groovy (59%)
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load_be_restart.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load_data_quality_error.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load_fe_restart.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load_pause_resume.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load_property.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_routine_load_shard_change.groovy
create mode 100644
regression-test/suites/load_p0/kinesis_routine_load/test_kinesis_show_routine_load.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_null_reject_compensation.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/rewrite_simple_agg_to_constant/auto_partition_recycle_count_cache.groovy
create mode 100644
regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy
copy be/src/storage/index/ann/CMakeLists.txt =>
regression-test/suites/pythonudaf_p0/udaf_scripts/udaf_errors.py (59%)
create mode 100644
regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy
copy regression-test/suites/pythonudf_p0/udf_scripts/{array_int_test.py =>
udf_errors.py} (85%)
create mode 100644
regression-test/suites/query_p0/cte/test_cbo_cte_inline_prune.groovy
delete mode 100644
regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy
delete mode 100644
regression-test/suites/query_p0/runtime_filter/rf_partition_pruning_type_matrix.groovy
copy
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadProgress.java
=> regression-test/suites/query_p0/test_parquet_dict.groovy (60%)
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_multi_segment.groovy
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_publish_conflict.groovy
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_schema_change.groovy
create mode 100644
regression-test/suites/statistics/test_full_analyze_hot_value.groovy
delete mode 100644
thirdparty/patches/apache-arrow-17.0.0-flight-safe-finish-status.patch
create mode 100644
thirdparty/patches/apache-arrow-17.0.0-status-inline-static-fix.patch
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
