This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d86ad920d37 [fix](http) Clamp range reads to file size (#67634)
d86ad920d37 is described below
commit d86ad920d37c62c0adc494f9bd23c394bbbfb919
Author: Gabriel <[email protected]>
AuthorDate: Wed Sep 9 09:48:47 2026 +0800
[fix](http) Clamp range reads to file size (#67634)
### What problem does this PR solve?
Issue Number: DORIS-28543
Related PR: None
Problem Summary:
`HttpFileReader` expands small reads to a 1 MiB read-ahead request. Near
EOF, the generated byte range can extend beyond the file size. Some
Range-capable object stores respond to that overlong range with HTTP 200
and the complete object, so Doris incorrectly reports that Range support
changed after it was successfully detected during open.
This change clamps speculative read-ahead to the known file boundary. It
also adds a local HTTP server unit test that reproduces the 206-at-open
followed by 200-on-overlong-range behavior.
### Release note
Fix HTTP range reads near EOF for servers that return the complete
object for ranges extending past EOF.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [ ] No.
- [x] Yes. HTTP read-ahead now respects the known EOF boundary.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/src/io/fs/http_file_reader.cpp | 5 ++
be/test/io/fs/http_file_reader_test.cpp | 84 +++++++++++++++++++++++++++++++++
2 files changed, 89 insertions(+)
diff --git a/be/src/io/fs/http_file_reader.cpp
b/be/src/io/fs/http_file_reader.cpp
index dc676e4a9f4..7ae9fc82a3f 100644
--- a/be/src/io/fs/http_file_reader.cpp
+++ b/be/src/io/fs/http_file_reader.cpp
@@ -280,6 +280,11 @@ Status HttpFileReader::read_at_impl(size_t offset, Slice
result, size_t* bytes_r
remaining = std::min<uint64_t>(to_read, left);
}
size_t req_len = (remaining > READ_BUFFER_SIZE) ? remaining :
READ_BUFFER_SIZE;
+ if (_size_known) {
+ // Some servers return the entire object when an otherwise valid range
crosses EOF, so the
+ // advertised file size must remain a hard boundary for speculative
read-ahead.
+ req_len = std::min(req_len, _file_size - offset);
+ }
VLOG(2) << "Issuing HTTP GET request: offset=" << offset << " req_len=" <<
req_len
<< " with_range=" << _range_supported;
diff --git a/be/test/io/fs/http_file_reader_test.cpp
b/be/test/io/fs/http_file_reader_test.cpp
index 08241d5d5c5..1c63ce3fd21 100644
--- a/be/test/io/fs/http_file_reader_test.cpp
+++ b/be/test/io/fs/http_file_reader_test.cpp
@@ -19,9 +19,68 @@
#include <gtest/gtest.h>
+#include <cstdio>
+#include <mutex>
+#include <string>
+#include <utility>
+#include <vector>
+
#include "io/file_factory.h"
+#include "service/http/ev_http_server.h"
+#include "service/http/http_channel.h"
+#include "service/http/http_handler.h"
+#include "service/http/http_headers.h"
+#include "service/http/http_request.h"
namespace doris::io {
+namespace {
+
+class EofSensitiveRangeHandler final : public HttpHandler {
+public:
+ explicit EofSensitiveRangeHandler(std::string content) :
_content(std::move(content)) {}
+
+ void handle(HttpRequest* req) override {
+ if (req->method() == HttpMethod::HEAD) {
+ req->add_output_header(HttpHeaders::CONTENT_LENGTH,
+ std::to_string(_content.size()).c_str());
+ HttpChannel::send_reply(req);
+ return;
+ }
+
+ const std::string range = req->header(HttpHeaders::RANGE);
+ {
+ std::lock_guard<std::mutex> lock(_mutex);
+ _ranges.push_back(range);
+ }
+
+ unsigned long long begin = 0;
+ unsigned long long end = 0;
+ if (std::sscanf(range.c_str(), "bytes=%llu-%llu", &begin, &end) != 2
|| begin > end ||
+ end >= _content.size()) {
+ HttpChannel::send_reply(req, _content);
+ return;
+ }
+
+ const std::string content_range = "bytes " + std::to_string(begin) +
"-" +
+ std::to_string(end) + "/" +
+ std::to_string(_content.size());
+ req->add_output_header(HttpHeaders::CONTENT_RANGE,
content_range.c_str());
+ HttpChannel::send_reply(req, HttpStatus::PARTIAL_CONTENT,
+ _content.substr(begin, end - begin + 1));
+ }
+
+ std::vector<std::string> ranges() {
+ std::lock_guard<std::mutex> lock(_mutex);
+ return _ranges;
+ }
+
+private:
+ std::string _content;
+ std::mutex _mutex;
+ std::vector<std::string> _ranges;
+};
+
+} // namespace
TEST(HttpFileReaderTest, ChunkResponseDisablesFileCache) {
FileSystemProperties properties;
@@ -38,4 +97,29 @@ TEST(HttpFileReaderTest, ChunkResponseDisablesFileCache) {
EXPECT_NE(std::dynamic_pointer_cast<HttpFileReader>(reader.value()),
nullptr);
}
+TEST(HttpFileReaderTest, RangeReadDoesNotCrossKnownEof) {
+ const std::string content =
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+ EofSensitiveRangeHandler handler(content);
+ EvHttpServer server(0);
+ ASSERT_TRUE(server.register_handler(HEAD, "/file", &handler));
+ ASSERT_TRUE(server.register_handler(GET, "/file", &handler));
+ server.start();
+ ASSERT_NE(server.get_real_port(), 0);
+
+ const std::string url = "http://127.0.0.1:"; +
std::to_string(server.get_real_port()) + "/file";
+ auto reader = HttpFileReader::create(url, {}, FileReaderOptions::DEFAULT,
nullptr);
+ ASSERT_TRUE(reader.has_value()) << reader.error();
+
+ constexpr size_t tail_size = 16;
+ char output[tail_size];
+ size_t bytes_read = 0;
+ Status status = reader.value()->read_at(content.size() - tail_size,
Slice(output, tail_size),
+ &bytes_read);
+
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(bytes_read, tail_size);
+ EXPECT_EQ(std::string(output, bytes_read), content.substr(content.size() -
tail_size));
+ EXPECT_EQ(handler.ranges(), (std::vector<std::string> {"bytes=0-0",
"bytes=48-63"}));
+}
+
} // namespace doris::io
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
