This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new bb8e42f639 Add GetOptions::head (#4931)
bb8e42f639 is described below
commit bb8e42f6392284f4a7a39d3eec74144a603b481c
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Sun Oct 15 11:04:14 2023 +0100
Add GetOptions::head (#4931)
---
object_store/src/aws/client.rs | 9 ++-------
object_store/src/aws/mod.rs | 4 ----
object_store/src/azure/client.rs | 9 ++-------
object_store/src/azure/mod.rs | 4 ----
object_store/src/client/get.rs | 24 +++---------------------
object_store/src/gcp/mod.rs | 13 ++-----------
object_store/src/http/client.rs | 15 +++++----------
object_store/src/http/mod.rs | 4 ----
object_store/src/lib.rs | 12 +++++++++++-
object_store/src/local.rs | 37 ++++---------------------------------
10 files changed, 29 insertions(+), 102 deletions(-)
diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs
index e3ac60eca0..ac07f9ab9a 100644
--- a/object_store/src/aws/client.rs
+++ b/object_store/src/aws/client.rs
@@ -554,15 +554,10 @@ impl GetClient for S3Client {
const STORE: &'static str = STORE;
/// Make an S3 GET request
<https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html>
- async fn get_request(
- &self,
- path: &Path,
- options: GetOptions,
- head: bool,
- ) -> Result<Response> {
+ async fn get_request(&self, path: &Path, options: GetOptions) ->
Result<Response> {
let credential = self.get_credential().await?;
let url = self.config.path_url(path);
- let method = match head {
+ let method = match options.head {
true => Method::HEAD,
false => Method::GET,
};
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index 0028be99fa..285ee2f59d 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -307,10 +307,6 @@ impl ObjectStore for AmazonS3 {
self.client.get_opts(location, options).await
}
- async fn head(&self, location: &Path) -> Result<ObjectMeta> {
- self.client.head(location).await
- }
-
async fn delete(&self, location: &Path) -> Result<()> {
self.client.delete_request(location, &()).await
}
diff --git a/object_store/src/azure/client.rs b/object_store/src/azure/client.rs
index cd1a3a10fc..f65388b61a 100644
--- a/object_store/src/azure/client.rs
+++ b/object_store/src/azure/client.rs
@@ -264,15 +264,10 @@ impl GetClient for AzureClient {
/// Make an Azure GET request
/// <https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob>
///
<https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-properties>
- async fn get_request(
- &self,
- path: &Path,
- options: GetOptions,
- head: bool,
- ) -> Result<Response> {
+ async fn get_request(&self, path: &Path, options: GetOptions) ->
Result<Response> {
let credential = self.get_credential().await?;
let url = self.config.path_url(path);
- let method = match head {
+ let method = match options.head {
true => Method::HEAD,
false => Method::GET,
};
diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs
index b210d486d9..9017634c42 100644
--- a/object_store/src/azure/mod.rs
+++ b/object_store/src/azure/mod.rs
@@ -202,10 +202,6 @@ impl ObjectStore for MicrosoftAzure {
self.client.get_opts(location, options).await
}
- async fn head(&self, location: &Path) -> Result<ObjectMeta> {
- self.client.head(location).await
- }
-
async fn delete(&self, location: &Path) -> Result<()> {
self.client.delete_request(location, &()).await
}
diff --git a/object_store/src/client/get.rs b/object_store/src/client/get.rs
index 333f6fe584..7f68b6d122 100644
--- a/object_store/src/client/get.rs
+++ b/object_store/src/client/get.rs
@@ -17,7 +17,7 @@
use crate::client::header::{header_meta, HeaderConfig};
use crate::path::Path;
-use crate::{Error, GetOptions, GetResult, ObjectMeta};
+use crate::{Error, GetOptions, GetResult};
use crate::{GetResultPayload, Result};
use async_trait::async_trait;
use futures::{StreamExt, TryStreamExt};
@@ -34,27 +34,20 @@ pub trait GetClient: Send + Sync + 'static {
last_modified_required: true,
};
- async fn get_request(
- &self,
- path: &Path,
- options: GetOptions,
- head: bool,
- ) -> Result<Response>;
+ async fn get_request(&self, path: &Path, options: GetOptions) ->
Result<Response>;
}
/// Extension trait for [`GetClient`] that adds common retrieval functionality
#[async_trait]
pub trait GetClientExt {
async fn get_opts(&self, location: &Path, options: GetOptions) ->
Result<GetResult>;
-
- async fn head(&self, location: &Path) -> Result<ObjectMeta>;
}
#[async_trait]
impl<T: GetClient> GetClientExt for T {
async fn get_opts(&self, location: &Path, options: GetOptions) ->
Result<GetResult> {
let range = options.range.clone();
- let response = self.get_request(location, options, false).await?;
+ let response = self.get_request(location, options).await?;
let meta =
header_meta(location, response.headers(),
T::HEADER_CONFIG).map_err(|e| {
Error::Generic {
@@ -77,15 +70,4 @@ impl<T: GetClient> GetClientExt for T {
meta,
})
}
-
- async fn head(&self, location: &Path) -> Result<ObjectMeta> {
- let options = GetOptions::default();
- let response = self.get_request(location, options, true).await?;
- header_meta(location, response.headers(),
T::HEADER_CONFIG).map_err(|e| {
- Error::Generic {
- store: T::STORE,
- source: Box::new(e),
- }
- })
- }
}
diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs
index a0a60f27a6..f80704b917 100644
--- a/object_store/src/gcp/mod.rs
+++ b/object_store/src/gcp/mod.rs
@@ -389,16 +389,11 @@ impl GetClient for GoogleCloudStorageClient {
const STORE: &'static str = STORE;
/// Perform a get request
<https://cloud.google.com/storage/docs/xml-api/get-object-download>
- async fn get_request(
- &self,
- path: &Path,
- options: GetOptions,
- head: bool,
- ) -> Result<Response> {
+ async fn get_request(&self, path: &Path, options: GetOptions) ->
Result<Response> {
let credential = self.get_credential().await?;
let url = self.object_url(path);
- let method = match head {
+ let method = match options.head {
true => Method::HEAD,
false => Method::GET,
};
@@ -604,10 +599,6 @@ impl ObjectStore for GoogleCloudStorage {
self.client.get_opts(location, options).await
}
- async fn head(&self, location: &Path) -> Result<ObjectMeta> {
- self.client.head(location).await
- }
-
async fn delete(&self, location: &Path) -> Result<()> {
self.client.delete_request(location).await
}
diff --git a/object_store/src/http/client.rs b/object_store/src/http/client.rs
index 0bd2e5639c..b2a6ac0aa3 100644
--- a/object_store/src/http/client.rs
+++ b/object_store/src/http/client.rs
@@ -288,14 +288,9 @@ impl GetClient for Client {
last_modified_required: false,
};
- async fn get_request(
- &self,
- location: &Path,
- options: GetOptions,
- head: bool,
- ) -> Result<Response> {
- let url = self.path_url(location);
- let method = match head {
+ async fn get_request(&self, path: &Path, options: GetOptions) ->
Result<Response> {
+ let url = self.path_url(path);
+ let method = match options.head {
true => Method::HEAD,
false => Method::GET,
};
@@ -311,7 +306,7 @@ impl GetClient for Client {
Some(StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED)
=> {
crate::Error::NotFound {
source: Box::new(source),
- path: location.to_string(),
+ path: path.to_string(),
}
}
_ => Error::Request { source }.into(),
@@ -322,7 +317,7 @@ impl GetClient for Client {
if has_range && res.status() != StatusCode::PARTIAL_CONTENT {
return Err(crate::Error::NotSupported {
source: Box::new(Error::RangeNotSupported {
- href: location.to_string(),
+ href: path.to_string(),
}),
});
}
diff --git a/object_store/src/http/mod.rs b/object_store/src/http/mod.rs
index e9ed5902d8..6ffb623589 100644
--- a/object_store/src/http/mod.rs
+++ b/object_store/src/http/mod.rs
@@ -118,10 +118,6 @@ impl ObjectStore for HttpStore {
self.client.get_opts(location, options).await
}
- async fn head(&self, location: &Path) -> Result<ObjectMeta> {
- self.client.head(location).await
- }
-
async fn delete(&self, location: &Path) -> Result<()> {
self.client.delete(location).await
}
diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs
index 68e785b3a3..ff0a46533d 100644
--- a/object_store/src/lib.rs
+++ b/object_store/src/lib.rs
@@ -410,7 +410,13 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync +
Debug + 'static {
}
/// Return the metadata for the specified location
- async fn head(&self, location: &Path) -> Result<ObjectMeta>;
+ async fn head(&self, location: &Path) -> Result<ObjectMeta> {
+ let options = GetOptions {
+ head: true,
+ ..Default::default()
+ };
+ Ok(self.get_opts(location, options).await?.meta)
+ }
/// Delete the object at the specified location.
async fn delete(&self, location: &Path) -> Result<()>;
@@ -716,6 +722,10 @@ pub struct GetOptions {
///
/// <https://datatracker.ietf.org/doc/html/rfc9110#name-range>
pub range: Option<Range<usize>>,
+ /// Request transfer of no content
+ ///
+ /// <https://datatracker.ietf.org/doc/html/rfc9110#name-head>
+ pub head: bool,
}
impl GetOptions {
diff --git a/object_store/src/local.rs b/object_store/src/local.rs
index 69da170b08..3ed63a4108 100644
--- a/object_store/src/local.rs
+++ b/object_store/src/local.rs
@@ -419,35 +419,6 @@ impl ObjectStore for LocalFileSystem {
.await
}
- async fn head(&self, location: &Path) -> Result<ObjectMeta> {
- let path = self.config.path_to_filesystem(location)?;
- let location = location.clone();
-
- maybe_spawn_blocking(move || {
- let metadata = match metadata(&path) {
- Err(e) => Err(match e.kind() {
- ErrorKind::NotFound => Error::NotFound {
- path: path.clone(),
- source: e,
- },
- _ => Error::Metadata {
- source: e.into(),
- path: location.to_string(),
- },
- }),
- Ok(m) => match !m.is_dir() {
- true => Ok(m),
- false => Err(Error::NotFound {
- path,
- source: io::Error::new(ErrorKind::NotFound, "is
directory"),
- }),
- },
- }?;
- convert_metadata(metadata, location)
- })
- .await
- }
-
async fn delete(&self, location: &Path) -> Result<()> {
let path = self.config.path_to_filesystem(location)?;
maybe_spawn_blocking(move || match std::fs::remove_file(&path) {
@@ -1604,15 +1575,15 @@ mod unix_test {
let path = root.path().join(filename);
unistd::mkfifo(&path, stat::Mode::S_IRWXU).unwrap();
- let location = Path::from(filename);
- integration.head(&location).await.unwrap();
-
// Need to open read and write side in parallel
let spawned = tokio::task::spawn_blocking(|| {
- OpenOptions::new().write(true).open(path).unwrap();
+ OpenOptions::new().write(true).open(path).unwrap()
});
+ let location = Path::from(filename);
+ integration.head(&location).await.unwrap();
integration.get(&location).await.unwrap();
+
spawned.await.unwrap();
}
}
