morristai commented on code in PR #2499:
URL:
https://github.com/apache/incubator-opendal/pull/2499#discussion_r1235884291
##########
core/src/services/webhdfs/backend.rs:
##########
@@ -489,30 +590,36 @@ impl Accessor for WebhdfsBackend {
if args.delimiter() != "/" {
return Err(Error::new(
ErrorKind::Unsupported,
- "webhdfs only support delimiter `/`",
+ "WebHDFS only support delimiter `/`",
));
}
let path = path.trim_end_matches('/');
- let req = self.webhdfs_list_status_request(path)?;
+ // batch status listing by default, if failed, fallback to non-batch
listing
+ let req = self.webhdfs_list_status_batch_request(path, &None)?;
let resp = self.client.send(req).await?;
- match resp.status() {
- StatusCode::OK => {
- let bs = resp.into_body().bytes().await?;
- let file_statuses =
serde_json::from_slice::<FileStatusesWrapper>(&bs)
- .map_err(new_json_deserialize_error)?
- .file_statuses
- .file_status;
- let objects = WebhdfsPager::new(path, file_statuses);
+ match self.webhdfs_list_status_batch_parse(resp).await {
+ Ok(directory_listing) => {
+ let file_statuses =
directory_listing.partial_listing.file_statuses.file_status;
+ let mut objects = WebhdfsPager::new(self.clone(), path,
file_statuses);
+
objects.set_remaining_entries(directory_listing.remaining_entries);
Ok((RpList::default(), objects))
}
- StatusCode::NOT_FOUND => {
- let objects = WebhdfsPager::new(path, vec![]);
- Ok((RpList::default(), objects))
+ Err(_) => {
+ // API doesn't support batch listing, fallback to non-batch
listing
Review Comment:
added config inside `WebhdfsBuilder` and `WebhdfsBackend`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@opendal.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
