This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 728923a5866 [chore](Azure) Print Azure request failed message (#36794)
728923a5866 is described below
commit 728923a5866357ddbd91b9d8ddef8eb198cb6e30
Author: AlexYue <[email protected]>
AuthorDate: Wed Jun 26 13:57:39 2024 +0800
[chore](Azure) Print Azure request failed message (#36794)
Print azure's failed request's http code along with more details.
---
be/src/io/fs/azure_obj_storage_client.cpp | 21 ++++++++++++++-------
be/src/io/fs/s3_file_writer.cpp | 16 ++++++++++++++--
2 files changed, 28 insertions(+), 9 deletions(-)
diff --git a/be/src/io/fs/azure_obj_storage_client.cpp
b/be/src/io/fs/azure_obj_storage_client.cpp
index 4bd0d1b7009..231113350f2 100644
--- a/be/src/io/fs/azure_obj_storage_client.cpp
+++ b/be/src/io/fs/azure_obj_storage_client.cpp
@@ -22,6 +22,7 @@
#include <algorithm>
#include <azure/core/http/http.hpp>
+#include <azure/core/http/http_status_code.hpp>
#include <azure/core/io/body_stream.hpp>
#include <azure/storage/blobs.hpp>
#include <azure/storage/blobs/blob_client.hpp>
@@ -60,8 +61,10 @@ ObjectStorageResponse do_azure_client_call(Func f, const
ObjectStoragePathOption
try {
f();
} catch (Azure::Core::RequestFailedException& e) {
- auto msg = fmt::format("Azure request failed because {}, error msg {},
path msg {}",
- e.what(), e.Message,
wrap_object_storage_path_msg(opts));
+ auto msg = fmt::format(
+ "Azure request failed because {}, error msg {}, http code {},
path msg {}",
+ e.what(), e.Message, static_cast<int>(e.StatusCode),
+ wrap_object_storage_path_msg(opts));
LOG_WARNING(msg);
return {.status =
convert_to_obj_response(Status::InternalError<false>(std::move(msg))),
.http_code = static_cast<int>(e.StatusCode),
@@ -99,10 +102,12 @@ ObjectStorageUploadResponse
AzureObjStorageClient::upload_part(const ObjectStora
Azure::Core::IO::MemoryBodyStream memory_body(
reinterpret_cast<const uint8_t*>(stream.data()),
stream.size());
// The blockId must be base64 encoded
- auto resp = client.StageBlock(base64_encode_part_num(part_num),
memory_body);
+ client.StageBlock(base64_encode_part_num(part_num), memory_body);
} catch (Azure::Core::RequestFailedException& e) {
- auto msg = fmt::format("Azure request failed because {}, error msg {},
path msg {}",
- e.what(), e.Message,
wrap_object_storage_path_msg(opts));
+ auto msg = fmt::format(
+ "Azure request failed because {}, error msg {}, http code {},
path msg {}",
+ e.what(), e.Message, static_cast<int>(e.StatusCode),
+ wrap_object_storage_path_msg(opts));
LOG_WARNING(msg);
// clang-format off
return {
@@ -142,8 +147,10 @@ ObjectStorageHeadResponse
AzureObjStorageClient::head_object(const ObjectStorage
.request_id = std::move(e.RequestId)},
};
}
- auto msg = fmt::format("Failed to head azure blob due to {}, path msg
{}", e.Message,
- wrap_object_storage_path_msg(opts));
+ auto msg = fmt::format(
+ "Azure request failed because {}, error msg {}, http code {},
path msg {}",
+ e.what(), e.Message, static_cast<int>(e.StatusCode),
+ wrap_object_storage_path_msg(opts));
return ObjectStorageHeadResponse {
.resp = {.status = convert_to_obj_response(
Status::InternalError<false>(std::move(msg))),
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index 9af34ea8ca8..20c616ef90a 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -290,15 +290,21 @@ Status S3FileWriter::appendv(const Slice* data, size_t
data_cnt) {
void S3FileWriter::_upload_one_part(int64_t part_num, UploadFileBuffer& buf) {
if (buf.is_cancelled()) {
+ LOG_INFO("file {} skip part {} because previous failure {}",
+ _obj_storage_path_opts.path.native(), part_num, _st);
return;
}
const auto& client = _obj_client->get();
if (nullptr == client) {
+ LOG_WARNING("failed at key: {}, load part {} bacause of invalid obj
client",
+ _obj_storage_path_opts.key, part_num);
buf.set_status(Status::InternalError<false>("invalid obj storage
client"));
return;
}
auto resp = client->upload_part(_obj_storage_path_opts,
buf.get_string_view_data(), part_num);
if (resp.resp.status.code != ErrorCode::OK) {
+ LOG_INFO("failed at key: {}, load part {}, st {}",
_obj_storage_path_opts.key, part_num,
+ resp.resp.status.msg);
buf.set_status(Status(resp.resp.status.code,
std::move(resp.resp.status.msg)));
return;
}
@@ -332,8 +338,10 @@ Status S3FileWriter::_complete() {
if (!_used_by_s3_committer) { // S3 committer will complete multipart
upload file on FE side.
if (_failed || _completed_parts.size() != _cur_part_num) {
_st = Status::InternalError(
- "error status {}, complete parts {}, cur part num {},
whole parts {}", _st,
- _completed_parts.size(), _cur_part_num,
_dump_completed_part());
+ "error status {}, have failed {}, complete parts {}, cur
part num {}, whole "
+ "parts {}, file path {}",
+ _st, _failed, _completed_parts.size(), _cur_part_num,
_dump_completed_part(),
+ _obj_storage_path_opts.path.native());
LOG(WARNING) << _st;
return _st;
}
@@ -343,6 +351,8 @@ Status S3FileWriter::_complete() {
TEST_SYNC_POINT_CALLBACK("S3FileWriter::_complete:2",
&_completed_parts);
auto resp = client->complete_multipart_upload(_obj_storage_path_opts,
_completed_parts);
if (resp.status.code != ErrorCode::OK) {
+ LOG_WARNING("Compltet multi part upload failed because {}, file
path {}",
+ resp.status.msg, _obj_storage_path_opts.path.native());
return {resp.status.code, std::move(resp.status.msg)};
}
}
@@ -378,6 +388,8 @@ void S3FileWriter::_put_object(UploadFileBuffer& buf) {
TEST_SYNC_POINT_RETURN_WITH_VOID("S3FileWriter::_put_object", this, &buf);
auto resp = client->put_object(_obj_storage_path_opts,
buf.get_string_view_data());
if (resp.status.code != ErrorCode::OK) {
+ LOG_WARNING("put object failed because {}, file path {}",
resp.status.msg,
+ _obj_storage_path_opts.path.native());
buf.set_status({resp.status.code, std::move(resp.status.msg)});
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
