This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a change to branch fe_local_shuffle_rebase3
in repository https://gitbox.apache.org/repos/asf/doris.git
omit 15d92ba024b [refactor](local shuffle) Move local exchange planning
from BE to FE
add 92c9c21f2cf [fix](mtmv) support qualified materialized view rename
target (#63216)
add ce61c8dfe99 [Chore](be) Avoid BE crash on exception (#63312)
add 21b9fdfbe9f [fix](be) Reject super wildcard path in json keys (#63300)
add 1e1f1a4c451 [Feature] support binlog<row> read and write (row type)
(2/3) (#63110)
add 8a4f118979f [improvement](iceberg) Reconstruct partition spec in SHOW
CREATE TABLE for Iceberg tables (#63240)
add 65136532975 [fix](parquet) Fix wrong encoding for parquet page v2
(#63305)
add ea933186522 [fix](testcase) Fix wrong test case (#63354)
add 95752b78090 [refine](column) enforce nullable nested types for array
(#63088)
add c75eba5488f [fix](fe) Prune empty and cascading CTE plans (#62828)
add cbffd4f290f [test](regression) Move Iceberg REST HDFS case to P2
(#63367)
add 53132a19a96 [fix](be) Fix macOS BE build (#63230)
add f9fb157ad85 [fix](publish) Avoid NPE for force-finished publish task
(#63069)
add 87d949800b0 [fix](fe) Avoid OFFSET path and NULL path for complex
datatype when its children are accessed. (#63229)
add bd21de83599 [fix](exchange) Drain exchange receiver queues outside
lock (#63331)
add 1faf1d9fda6 [improvement](cloud) Propagate compute group id to BE
thrift payloads (#62487)
add f87c73e363d [refactor](local shuffle) Move local exchange planning
from BE to FE
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (15d92ba024b)
\
N -- N -- N refs/heads/fe_local_shuffle_rebase3 (f87c73e363d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
be/src/cloud/cloud_rowset_writer.cpp | 2 +
be/src/cloud/cloud_tablets_channel.cpp | 4 +
be/src/cloud/pb_convert.cpp | 32 +-
be/src/common/multi_version.h | 33 +-
be/src/core/data_type/data_type_array.cpp | 17 +-
be/src/core/data_type/data_type_array.h | 11 +-
be/src/core/data_type/primitive_type.h | 2 +
.../core/data_type_serde/data_type_array_serde.cpp | 4 +-
.../core/data_type_serde/data_type_array_serde.h | 10 +-
.../data_type_serde/data_type_nullable_serde.h | 3 +
be/src/exec/common/memory.cpp | 2 +-
be/src/exec/exchange/vdata_stream_recvr.cpp | 55 +--
be/src/exec/exchange/vdata_stream_recvr.h | 3 +
be/src/exec/operator/olap_scan_operator.cpp | 15 +-
be/src/exec/pipeline/pipeline_fragment_context.cpp | 2 +-
be/src/exec/scan/olap_scanner.cpp | 15 +-
be/src/exec/scan/olap_scanner.h | 1 +
be/src/exec/scan/parallel_scanner_builder.cpp | 6 +-
be/src/exec/scan/scanner_scheduler.h | 2 +-
.../time_sharing/prioritized_split_runner.cpp | 26 +-
be/src/exec/sink/autoinc_buffer.cpp | 2 +
be/src/exec/sink/autoinc_buffer.h | 8 +
be/src/exprs/function/cast/cast_to_array.h | 2 +-
be/src/exprs/function/function_jsonb.cpp | 4 +-
be/src/format/parquet/vparquet_group_reader.cpp | 3 +-
.../information_schema/schema_rowsets_scanner.cpp | 9 +-
be/src/io/io_common.h | 4 +-
be/src/load/channel/tablets_channel.cpp | 38 +-
be/src/load/delta_writer/delta_writer.cpp | 14 +-
be/src/load/delta_writer/delta_writer.h | 3 +
be/src/load/delta_writer/delta_writer_context.h | 11 +-
be/src/load/delta_writer/delta_writer_v2.cpp | 2 +
be/src/load/memtable/memtable_flush_executor.cpp | 293 +++++++++---
be/src/load/memtable/memtable_flush_executor.h | 57 ++-
be/src/load/memtable/memtable_writer.cpp | 5 +-
be/src/storage/binlog.h | 146 ++++++
be/src/storage/binlog_config.h | 4 +-
be/src/storage/compaction/compaction.cpp | 13 +
be/src/storage/data_dir.cpp | 129 ++++-
be/src/storage/iterator/vcollect_iterator.cpp | 4 +
be/src/storage/iterators.h | 1 +
be/src/storage/olap_common.h | 10 +
be/src/storage/partial_update_info.cpp | 49 +-
be/src/storage/partial_update_info.h | 24 +-
be/src/storage/rowset/beta_rowset_reader.cpp | 1 +
be/src/storage/rowset/beta_rowset_writer.cpp | 11 +-
be/src/storage/rowset/beta_rowset_writer.h | 11 +
be/src/storage/rowset/beta_rowset_writer_v2.h | 4 +
be/src/storage/rowset/group_rowset_writer.cpp | 43 +-
be/src/storage/rowset/group_rowset_writer.h | 13 +
be/src/storage/rowset/rowset.h | 4 +-
be/src/storage/rowset/rowset_factory.cpp | 4 +-
be/src/storage/rowset/rowset_meta.h | 21 +-
be/src/storage/rowset/rowset_meta_manager.cpp | 157 ++++++-
be/src/storage/rowset/rowset_meta_manager.h | 40 +-
be/src/storage/rowset/rowset_writer.h | 9 +
be/src/storage/rowset/rowset_writer_context.h | 48 +-
be/src/storage/rowset/segment_creator.cpp | 18 +-
be/src/storage/rowset/segment_creator.h | 3 +
.../storage/rowset/vertical_beta_rowset_writer.cpp | 1 +
be/src/storage/rowset_builder.cpp | 115 +++--
be/src/storage/rowset_builder.h | 28 +-
be/src/storage/rowset_version_mgr.cpp | 25 +-
be/src/storage/schema.cpp | 8 +
be/src/storage/schema.h | 17 +
.../storage/segment/historical_row_retriever.cpp | 288 ++++++++++++
be/src/storage/segment/historical_row_retriever.h | 134 ++++++
.../storage/segment/row_binlog_segment_writer.cpp | 522 +++++++++++++++++++++
be/src/storage/segment/row_binlog_segment_writer.h | 111 +++++
be/src/storage/segment/segment_iterator.cpp | 121 +++++
be/src/storage/segment/segment_iterator.h | 2 +
be/src/storage/segment/segment_writer.cpp | 120 +++--
be/src/storage/segment/segment_writer.h | 14 +-
be/src/storage/segment/vertical_segment_writer.cpp | 16 +-
be/src/storage/storage_engine.cpp | 52 ++
be/src/storage/tablet/base_tablet.cpp | 157 ++++++-
be/src/storage/tablet/base_tablet.h | 9 +-
be/src/storage/tablet/tablet.cpp | 143 +++++-
be/src/storage/tablet/tablet.h | 14 +-
be/src/storage/tablet/tablet_manager.cpp | 12 +
be/src/storage/tablet/tablet_meta_manager.cpp | 46 +-
be/src/storage/tablet/tablet_meta_manager.h | 6 +
be/src/storage/tablet/tablet_reader.cpp | 6 +-
be/src/storage/tablet/tablet_schema.h | 9 +
be/src/storage/tablet_info.cpp | 49 ++
be/src/storage/tablet_info.h | 3 +
.../storage/task/engine_publish_version_task.cpp | 46 +-
be/src/storage/task/engine_publish_version_task.h | 4 +-
be/src/storage/txn/txn_manager.cpp | 108 ++++-
be/src/storage/txn/txn_manager.h | 17 +-
be/src/util/thrift_util.cpp | 13 +
be/src/util/thrift_util.h | 1 +
be/test/core/block/block_test.cpp | 6 +-
.../data_type_serde_get_name_test.cpp | 4 +-
.../data_type_serde_string_test.cpp | 8 +-
be/test/core/jsonb/serialize_test.cpp | 6 +-
.../data/vec/native/all_types_single_row.native | Bin 1124 -> 1140 bytes
be/test/exec/common/schema_util_test.cpp | 4 +-
.../time_sharing_task_executor_test.cpp | 43 ++
.../exec/operator/table_function_operator_test.cpp | 55 ++-
be/test/exec/pipeline/vdata_stream_recvr_test.cpp | 109 +++++
be/test/exprs/aggregate/agg_array_agg_test.cpp | 70 ---
be/test/exprs/aggregate/agg_replace_test.cpp | 6 +-
be/test/exprs/function/function_jsonb_test.cpp | 74 ++-
be/test/format/orc/orc_reader_fill_data_test.cpp | 48 +-
be/test/format/parquet/parquet_thrift_test.cpp | 31 ++
.../load/memtable/memtable_flush_executor_test.cpp | 334 ++++++++++++-
be/test/olap/rowset/group_rowset_builder_test.cpp | 112 +----
be/test/olap/rowset/group_rowset_writer_test.cpp | 225 +++++++++
.../storage/rowset/rowset_meta_manager_test.cpp | 139 ++++++
.../segment/inverted_index_file_writer_test.cpp | 1 +
be/test/storage/txn/txn_manager_test.cpp | 119 ++++-
be/test/testutil/creators.h | 148 +++++-
.../main/java/org/apache/doris/catalog/Column.java | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 5 +-
.../apache/doris/alter/SchemaChangeHandler.java | 6 +
.../doris/catalog/AutoIncrementGenerator.java | 4 +
.../doris/catalog/BuiltinTableValuedFunctions.java | 2 +
.../main/java/org/apache/doris/catalog/Env.java | 12 +
.../org/apache/doris/catalog/FunctionGenTable.java | 8 +
.../java/org/apache/doris/catalog/OlapTable.java | 28 +-
.../org/apache/doris/catalog/OlapTableWrapper.java | 2 +
.../doris/catalog/RowBinlogTableWrapper.java | 18 +-
.../java/org/apache/doris/catalog/SchemaTable.java | 2 +-
.../apache/doris/datasource/InternalCatalog.java | 2 +-
.../datasource/iceberg/IcebergExternalTable.java | 50 ++
.../java/org/apache/doris/load/DeleteHandler.java | 7 +
.../org/apache/doris/nereids/StatementContext.java | 11 +
.../glue/translator/PhysicalPlanTranslator.java | 12 +-
.../doris/nereids/jobs/executor/Optimizer.java | 78 ++-
.../doris/nereids/parser/LogicalPlanBuilder.java | 7 +-
.../expression/rules/FoldConstantRuleOnBE.java | 4 +-
.../rewrite/AccessPathExpressionCollector.java | 19 +
.../doris/nereids/rules/rewrite/CTEInliner.java | 86 +++-
.../nereids/rules/rewrite/ClearContextStatus.java | 5 +-
.../nereids/rules/rewrite/NestedColumnPruning.java | 299 ++++++++++--
.../trees/expressions/ArrayItemReference.java | 3 +-
.../functions/AggCombinerFunctionBuilder.java | 3 +-
.../functions/ComputeSignatureHelper.java | 2 +-
.../functions/combinator/ForEachCombinator.java | 2 +-
.../expressions/functions/scalar/ArrayMap.java | 2 +-
.../expressions/functions/scalar/ArraySort.java | 4 +-
.../functions/table/{Jobs.java => Binlog.java} | 19 +-
.../visitor/TableValuedFunctionVisitor.java | 5 +
.../plans/commands/info/AlterMTMVRenameInfo.java | 15 +-
.../org/apache/doris/nereids/types/ArrayType.java | 28 +-
.../org/apache/doris/nereids/types/DataType.java | 4 +-
.../doris/nereids/util/TypeCoercionUtils.java | 3 +-
.../org/apache/doris/planner/OlapScanNode.java | 33 +-
.../org/apache/doris/planner/OlapTableSink.java | 17 +
.../java/org/apache/doris/qe/SessionVariable.java | 2 +-
.../doris/tablefunction/TableBinlogFunction.java | 211 +++++++++
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
.../apache/doris/task/AgentTaskCleanupDaemon.java | 2 +-
.../org/apache/doris/task/PublishVersionTask.java | 7 +-
.../doris/transaction/DatabaseTransactionMgr.java | 6 +-
.../doris/alter/SchemaChangeHandlerTest.java | 12 +
.../org/apache/doris/catalog/CreateTableTest.java | 23 +-
.../java/org/apache/doris/catalog/TypeTest.java | 6 +-
.../iceberg/IcebergExternalTableTest.java | 160 ++++++-
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 29 +-
.../doris/nereids/rules/rewrite/CTEInlineTest.java | 38 ++
.../rules/rewrite/PruneNestedColumnTest.java | 114 ++++-
.../nereids/trees/expressions/LiteralTest.java | 12 +-
.../apache/doris/task/PublishVersionTaskTest.java | 109 +++++
.../CheckReplicaContinuousVersionSuccTest.java | 177 +++++++
.../java/org/apache/doris/catalog/ArrayType.java | 57 ++-
gensrc/proto/descriptors.proto | 2 +
gensrc/proto/olap_file.proto | 17 +-
gensrc/thrift/BackendService.thrift | 1 +
gensrc/thrift/Descriptors.thrift | 2 +
gensrc/thrift/HeartbeatService.thrift | 1 +
gensrc/thrift/PlanNodes.thrift | 2 +
gensrc/thrift/Types.thrift | 1 +
.../column_pruning/null_column_pruning.out | 6 +
.../string_length_column_pruning.out | 9 +
.../unique_function/agg_with_unique_function.out | 8 +-
.../query_p0/repeat/test_repeat_output_slot.out | 28 +-
.../query_p0/system/test_query_sys_rowsets.out | 3 +-
.../system/test_query_sys_scan_rowsets.out | 3 +-
.../test_parquet_dict.out} | 7 +-
.../data/row_binlog_p0/test_row_binlog_basic.out | 53 +++
.../test_row_binlog_multi_segment.out | 36 ++
.../test_row_binlog_publish_conflict.out | 40 ++
.../test_row_binlog_schema_change.out | 20 +
.../pipeline/nonConcurrent/conf/fe.conf | 2 +
.../tvf/test_hdfs_parquet_group0.groovy | 4 +-
.../iceberg_rest_on_hdfs.groovy | 4 +-
.../test_publish_no_npe_when_be_down.groovy | 142 ++++++
.../test_jsonb_keys_invalid_path.groovy} | 16 +-
.../column_pruning/null_column_pruning.groovy | 67 ++-
.../string_length_column_pruning.groovy | 72 ++-
.../query_p0/cte/test_cbo_cte_inline_prune.groovy | 113 +++++
.../test_parquet_dict.groovy} | 27 +-
.../row_binlog_p0/test_row_binlog_basic.groovy | 264 +++++++++++
.../test_row_binlog_multi_segment.groovy | 239 ++++++++++
.../test_row_binlog_publish_conflict.groovy | 303 ++++++++++++
.../test_row_binlog_schema_change.groovy | 110 +++++
regression-test/suites/tso_p0/test_tso_api.groovy | 169 +++----
.../tso_p0/test_tso_rowset_commit_tso.groovy | 70 ++-
200 files changed, 7756 insertions(+), 1086 deletions(-)
create mode 100644 be/src/storage/segment/historical_row_retriever.cpp
create mode 100644 be/src/storage/segment/historical_row_retriever.h
create mode 100644 be/src/storage/segment/row_binlog_segment_writer.cpp
create mode 100644 be/src/storage/segment/row_binlog_segment_writer.h
create mode 100644 be/test/olap/rowset/group_rowset_writer_test.cpp
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/{Jobs.java
=> Binlog.java} (81%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableBinlogFunction.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/task/PublishVersionTaskTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/transaction/CheckReplicaContinuousVersionSuccTest.java
copy regression-test/data/{ann_index_p0/scalar_quantization.out =>
query_p0/test_parquet_dict.out} (67%)
create mode 100644 regression-test/data/row_binlog_p0/test_row_binlog_basic.out
create mode 100644
regression-test/data/row_binlog_p0/test_row_binlog_multi_segment.out
create mode 100644
regression-test/data/row_binlog_p0/test_row_binlog_publish_conflict.out
create mode 100644
regression-test/data/row_binlog_p0/test_row_binlog_schema_change.out
rename regression-test/suites/{external_table_p0/refactor_storage_param =>
external_table_p2/refactor_catalog_param}/iceberg_rest_on_hdfs.groovy (99%)
create mode 100644
regression-test/suites/fault_injection_p0/test_publish_no_npe_when_be_down.groovy
copy
regression-test/suites/{query_p0/cte/test_cte_with_duplicate_consumer.groovy =>
jsonb_p0/test_jsonb_keys_invalid_path.groovy} (64%)
create mode 100644
regression-test/suites/query_p0/cte/test_cbo_cte_inline_prune.groovy
copy regression-test/suites/{ddl_p0/test_createtable_strlen.groovy =>
query_p0/test_parquet_dict.groovy} (63%)
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_multi_segment.groovy
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_publish_conflict.groovy
create mode 100644
regression-test/suites/row_binlog_p0/test_row_binlog_schema_change.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
