amorynan opened a new pull request, #34076:
URL: https://github.com/apache/doris/pull/34076
FIX:
1. array with empty row which will make doc id is not right
2. array with inverted index with large data set . then compaction action is
triggered , maybe here a core would happened like
```
=================================================================
==1006012==ERROR: AddressSanitizer: heap-buffer-overflow on address
0x7f4170e7f800 at pc 0x5564999e655a bp 0x7f46478a19d0 sp 0x7f46478a19c8
READ of size 1 at 0x7f4170e7f800 thread T1281 (CumuCompactionT)
#0 0x5564999e6559 in
doris::segment_v2::InvertedIndexColumnWriterImpl<(doris::FieldType)5>::add_array_values(unsigned
long, void const*, unsigned char const*, unsigned char const*, unsigned long)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp:418:25
#1 0x55649995254f in
doris::segment_v2::ArrayColumnWriter::append_data(unsigned char const**,
unsigned long)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/rowset/segment_v2/column_writer.cpp:960:13
#2 0x556499952a7d in
doris::segment_v2::ArrayColumnWriter::append_nullable(unsigned char const*,
unsigned char const**, unsigned long)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/rowset/segment_v2/column_writer.cpp:978:5
#3 0x55649994568c in doris::segment_v2::ColumnWriter::append(unsigned
char const*, void const*, unsigned long)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/rowset/segment_v2/column_writer.cpp:419:16
#4 0x5564998fdfc0 in
doris::segment_v2::SegmentWriter::append_block(doris::vectorized::Block const*,
unsigned long, unsigned long)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/rowset/segment_v2/segment_writer.cpp:842:9
#5 0x556499ad46df in
doris::VerticalBetaRowsetWriter<doris::BetaRowsetWriter>::add_columns(doris::vectorized::Block
const*, std::vector<unsigned int, std::allocator<unsigned int>> const&, bool,
unsigned int)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/rowset/vertical_beta_rowset_writer.cpp:112:13
#6 0x556499146a2a in
doris::Merger::vertical_compact_one_group(std::shared_ptr<doris::BaseTablet>,
doris::ReaderType, doris::TabletSchema const&, bool, std::vector<unsigned int,
std::allocator<unsigned int>> const&, doris::vectorized::RowSourcesBuffer*,
std::vector<std::shared_ptr<doris::RowsetReader>,
std::allocator<std::shared_ptr<doris::RowsetReader>>> const&,
doris::RowsetWriter*, long, doris::Merger::Statistics*, std::vector<unsigned
int, std::allocator<unsigned int>>)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/merger.cpp:297:9
#7 0x556499149b6f in
doris::Merger::vertical_merge_rowsets(std::shared_ptr<doris::BaseTablet>,
doris::ReaderType, doris::TabletSchema const&,
std::vector<std::shared_ptr<doris::RowsetReader>,
std::allocator<std::shared_ptr<doris::RowsetReader>>> const&,
doris::RowsetWriter*, long, doris::Merger::Statistics*)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/merger.cpp:406:9
#8 0x5564990accf9 in doris::Compaction::merge_input_rowsets()
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/compaction.cpp:175:19
#9 0x5564990b4cc3 in doris::CompactionMixin::execute_compact_impl(long)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/compaction.cpp:435:5
#10 0x5564990b33ff in doris::CompactionMixin::execute_compact()
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/compaction.cpp:388:17
#11 0x556499df7514 in doris::CumulativeCompaction::execute_compact()
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/cumulative_compaction.cpp:101:5
#12 0x556499d94e2f in
doris::Tablet::execute_compaction(doris::CompactionMixin&)
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/tablet.cpp:1662:29
#13 0x5564990097d8 in
doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType, bool)::$_1::operator()() const
/mnt/disk1/wangqiannan/amory/doris/be/src/olap/olap_server.cpp:1001:25
#14 0x556499009304 in void std::__invoke_impl<void,
doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType, bool)::$_1&>(std::__invoke_other,
doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType, bool)::$_1&)
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14
#15 0x5564990092a4 in std::enable_if<is_invocable_r_v<void,
doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType, bool)::$_1&>, void>::type std::__invoke_r<void,
doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType,
bool)::$_1&>(doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType, bool)::$_1&)
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2
#16 0x5564990090ac in std::_Function_handler<void (),
doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>,
doris::CompactionType, bool)::$_1>::_M_invoke(std::_Any_data const&)
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9
#17 0x556497352fb2 in std::function<void ()>::operator()() const
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560:9
#18 0x55649b1a6018 in doris::FunctionRunnable::run()
/mnt/disk1/wangqiannan/amory/doris/be/src/util/threadpool.cpp:48:27
#19 0x55649b191dcd in doris::ThreadPool::dispatch_thread()
/mnt/disk1/wangqiannan/amory/doris/be/src/util/threadpool.cpp:543:24
#20 0x55649b1b8ef3 in void std::__invoke_impl<void, void
(doris::ThreadPool::*&)(), doris::ThreadPool*&>(std::__invoke_memfun_deref,
void (doris::ThreadPool::*&)(), doris::ThreadPool*&)
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
#21 0x55649b1b8dcc in std::__invoke_result<void
(doris::ThreadPool::*&)(), doris::ThreadPool*&>::type std::__invoke<void
(doris::ThreadPool::*&)(), doris::ThreadPool*&>(void (doris::ThreadPool::*&)(),
doris::ThreadPool*&)
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
#22 0x55649b1b8d54 in void std::_Bind<void (doris::ThreadPool::*
(doris::ThreadPool*))()>::__call<void, 0ul>(std::tuple<>&&,
std::_Index_tuple<0ul>)
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:420:11
#23 0x55649b1b8bfd in void std::_Bind<void (doris::ThreadPool::*
(doris::ThreadPool*))()>::operator()<void>()
/mnt/disk1/wangqiannan/tool/ldb_toolchain_16/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:503:17
```
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
## Further comments
If this is a relatively large or complex change, kick off the discussion at
[[email protected]](mailto:[email protected]) by explaining why you
chose the solution you did and what alternatives you considered, etc...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
