This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 9e9bdfb6721 [fix](inverted index)Add exception check when write bkd
index (#39248) (#39279)
9e9bdfb6721 is described below
commit 9e9bdfb672102a1c90f04dd0db72ecb088a4c276
Author: qiye <[email protected]>
AuthorDate: Wed Aug 14 16:15:04 2024 +0800
[fix](inverted index)Add exception check when write bkd index (#39248)
(#39279)
---
.../rowset/segment_v2/inverted_index_writer.cpp | 40 +++++++++--------
.../test_index_bkd_writer_fault_injection.groovy | 51 ++++++++++++++++++++++
2 files changed, 72 insertions(+), 19 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 9b452b6c765..cdfc0d7fdd4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -395,7 +395,7 @@ public:
_rid++;
}
} else if constexpr (field_is_numeric_type(field_type)) {
- add_numeric_values(values, count);
+ RETURN_IF_ERROR(add_numeric_values(values, count));
}
return Status::OK();
}
@@ -487,11 +487,7 @@ public:
continue;
}
const CppType* p = &reinterpret_cast<const
CppType*>(value_ptr)[j];
- std::string new_value;
- size_t value_length = sizeof(CppType);
-
- _value_key_coder->full_encode_ascending(p, &new_value);
- _bkd_writer->add((const uint8_t*)new_value.c_str(),
value_length, _rid);
+ RETURN_IF_ERROR(add_value(*p));
}
start_off += array_elem_size;
_row_ids_seen_for_bkd++;
@@ -535,11 +531,7 @@ public:
if (values->is_null_at(j)) {
// bkd do not index null values, so we do nothing here.
} else {
- std::string new_value;
- size_t value_length = sizeof(CppType);
-
- _value_key_coder->full_encode_ascending(p, &new_value);
- _bkd_writer->add((const uint8_t*)new_value.c_str(),
value_length, _rid);
+ RETURN_IF_ERROR(add_value(*p));
}
item_data_ptr = (uint8_t*)item_data_ptr + field_size;
}
@@ -551,23 +543,33 @@ public:
return Status::OK();
}
- void add_numeric_values(const void* values, size_t count) {
+ Status add_numeric_values(const void* values, size_t count) {
auto p = reinterpret_cast<const CppType*>(values);
for (size_t i = 0; i < count; ++i) {
- add_value(*p);
+ RETURN_IF_ERROR(add_value(*p));
+ _rid++;
p++;
_row_ids_seen_for_bkd++;
}
+ return Status::OK();
}
- void add_value(const CppType& value) {
- std::string new_value;
- size_t value_length = sizeof(CppType);
+ Status add_value(const CppType& value) {
+ try {
+ std::string new_value;
+ size_t value_length = sizeof(CppType);
- _value_key_coder->full_encode_ascending(&value, &new_value);
- _bkd_writer->add((const uint8_t*)new_value.c_str(), value_length,
_rid);
+
DBUG_EXECUTE_IF("InvertedIndexColumnWriterImpl::add_value_bkd_writer_add_throw_error",
{
+ _CLTHROWA(CL_ERR_IllegalArgument, ("packedValue should be
length=xxx"));
+ });
- _rid++;
+ _value_key_coder->full_encode_ascending(&value, &new_value);
+ _bkd_writer->add((const uint8_t*)new_value.c_str(), value_length,
_rid);
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError add_value: {}", e.what());
+ }
+ return Status::OK();
}
int64_t size() const override {
diff --git
a/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
new file mode 100644
index 00000000000..7df72ebeaf1
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_index_bkd_writer_fault_injection.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_bkd_writer_fault_injection", "nonConcurrent") {
+ def isCloudMode = isCloudMode()
+ def tableName = "test_index_bkd_writer_fault_injection"
+
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
+ sql """
+ CREATE TABLE ${tableName} (
+ `id` int(11) NULL,
+ `name` varchar(255) NULL,
+ `hobbies` text NULL,
+ `score` int(11) NULL,
+ index index_name (name) using inverted,
+ index index_hobbies (hobbies) using inverted
properties("parser"="english"),
+ index index_score (score) using inverted
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" =
"true", "inverted_index_storage_format" = "V1");
+ """
+
+ try {
+
GetDebugPoint().enableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::add_value_bkd_writer_add_throw_error")
+ logger.info("trigger_full_compaction_on_tablets with fault injection:
InvertedIndexColumnWriterImpl::add_value_bkd_writer_add_throw_error")
+ sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple",
100); """
+ } catch (Exception e) {
+ logger.info("error message: ${e.getMessage()}")
+ assert e.getMessage().contains("packedValue should be length=xxx")
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::add_value_bkd_writer_add_throw_error")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
