This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 83a734e762c [fix](split) remove retry when fetch split batch failed
(#37636)
83a734e762c is described below
commit 83a734e762cfa2c6497288d3e5fe96d77c34b455
Author: Ashin Gau <[email protected]>
AuthorDate: Fri Jul 12 22:46:33 2024 +0800
[fix](split) remove retry when fetch split batch failed (#37636)
## Proposed changes
We need to remove the retry logic for failed to fetch split batch.
Originally, this was implemented to handle cases where the cached client
connection might have been lost and needed to be reestablished. However,
this retry mechanism can lead to data loss. For instance, if a batch of
data has already been sent, retrying can cause this batch to be lost
without the receiver being aware of it.
---
be/src/vec/exec/scan/split_source_connector.cpp | 10 ++--------
.../java/org/apache/doris/datasource/SplitSourceManager.java | 7 ++++++-
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/be/src/vec/exec/scan/split_source_connector.cpp
b/be/src/vec/exec/scan/split_source_connector.cpp
index 9bba44b4e76..6533ae2bfe0 100644
--- a/be/src/vec/exec/scan/split_source_connector.cpp
+++ b/be/src/vec/exec/scan/split_source_connector.cpp
@@ -56,14 +56,8 @@ Status RemoteSplitSourceConnector::get_next(bool* has_next,
TFileRangeDesc* rang
TFetchSplitBatchResult result;
try {
coord->fetchSplitBatch(result, request);
- } catch (std::exception& e1) {
- LOG(WARNING) << "Failed to get batch of split source: {}, try to
reopen" << e1.what();
- RETURN_IF_ERROR(coord.reopen());
- try {
- coord->fetchSplitBatch(result, request);
- } catch (std::exception& e2) {
- return Status::IOError("Failed to get batch of split source:
{}", e2.what());
- }
+ } catch (std::exception& e) {
+ return Status::IOError<false>("Failed to get batch of split
source: {}", e.what());
}
_last_batch = result.splits.empty();
_scan_ranges = result.splits;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
index 83a7436df9a..6d4b06e0e7b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
@@ -61,7 +61,12 @@ public class SplitSourceManager extends MasterDaemon {
}
public SplitSource getSplitSource(long uniqueId) {
- return splits.get(uniqueId).get();
+ WeakReference<SplitSource> ref = splits.get(uniqueId);
+ if (ref == null) {
+ return null;
+ } else {
+ return ref.get();
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
