This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1acd6d062 test(huggingface): cover url construction in huggingface
core (#6803)
1acd6d062 is described below
commit 1acd6d0620f45a000d5442e6c91091b51cccc335
Author: Krisztián Szűcs <[email protected]>
AuthorDate: Thu Nov 20 09:43:57 2025 +0100
test(huggingface): cover url construction in huggingface core (#6803)
---
core/src/services/huggingface/core.rs | 236 ++++++++++++++++++++++++++++++++++
1 file changed, 236 insertions(+)
diff --git a/core/src/services/huggingface/core.rs
b/core/src/services/huggingface/core.rs
index e852d0a6c..6aa635c59 100644
--- a/core/src/services/huggingface/core.rs
+++ b/core/src/services/huggingface/core.rs
@@ -253,6 +253,242 @@ mod tests {
use super::*;
use crate::raw::new_json_deserialize_error;
use crate::types::Result;
+ use http::{Request, Response, StatusCode};
+ use std::sync::{Arc, Mutex};
+
+ // Mock HTTP client that captures the request URL and headers
+ #[derive(Clone)]
+ struct MockHttpClient {
+ url: Arc<Mutex<Option<String>>>,
+ headers: Arc<Mutex<Option<http::HeaderMap>>>,
+ }
+
+ impl MockHttpClient {
+ fn new() -> Self {
+ Self {
+ url: Arc::new(Mutex::new(None)),
+ headers: Arc::new(Mutex::new(None)),
+ }
+ }
+
+ fn get_captured_url(&self) -> String {
+ self.url.lock().unwrap().clone().unwrap()
+ }
+
+ fn get_captured_headers(&self) -> http::HeaderMap {
+ self.headers.lock().unwrap().clone().unwrap()
+ }
+ }
+
+ impl HttpFetch for MockHttpClient {
+ async fn fetch(&self, req: Request<Buffer>) ->
Result<Response<HttpBody>> {
+ // Capture the URL and headers
+ *self.url.lock().unwrap() = Some(req.uri().to_string());
+ *self.headers.lock().unwrap() = Some(req.headers().clone());
+
+ // Return a mock response with empty body
+ Ok(Response::builder()
+ .status(StatusCode::OK)
+ .body(HttpBody::new(futures::stream::empty(), Some(0)))
+ .unwrap())
+ }
+ }
+
+ /// Utility function to create HuggingfaceCore with mocked HTTP client
+ fn create_test_core(
+ repo_type: RepoType,
+ repo_id: &str,
+ revision: &str,
+ endpoint: &str,
+ ) -> (HuggingfaceCore, MockHttpClient) {
+ let mock_client = MockHttpClient::new();
+ let http_client = HttpClient::with(mock_client.clone());
+
+ let info = AccessorInfo::default();
+ info.set_scheme("huggingface")
+ .set_native_capability(Capability::default());
+ info.update_http_client(|_| http_client);
+
+ let core = HuggingfaceCore {
+ info: Arc::new(info),
+ repo_type,
+ repo_id: repo_id.to_string(),
+ revision: revision.to_string(),
+ root: "/".to_string(),
+ token: None,
+ endpoint: endpoint.to_string(),
+ };
+
+ (core, mock_client)
+ }
+
+ #[tokio::test]
+ async fn test_hf_path_info_url_model() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Model,
+ "test-user/test-repo",
+ "main",
+ "https://huggingface.co";,
+ );
+
+ core.hf_path_info("test.txt").await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+
"https://huggingface.co/api/models/test-user/test-repo/paths-info/main";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_path_info_url_dataset() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Dataset,
+ "test-org/test-dataset",
+ "v1.0.0",
+ "https://huggingface.co";,
+ );
+
+ core.hf_path_info("data/file.csv").await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+
"https://huggingface.co/api/datasets/test-org/test-dataset/paths-info/v1%2E0%2E0";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_path_info_url_custom_endpoint() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Model,
+ "test-org/test-dataset",
+ "refs/convert/parquet",
+ "https://custom-hf.example.com";,
+ );
+
+ core.hf_path_info("model.bin").await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+
"https://custom-hf.example.com/api/models/test-org/test-dataset/paths-info/refs%2Fconvert%2Fparquet";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_list_url_non_recursive() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Model,
+ "org/model",
+ "main",
+ "https://huggingface.co";,
+ );
+
+ core.hf_list("path1", false).await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+
"https://huggingface.co/api/models/org/model/tree/main/path1?expand=True";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_list_url_recursive() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Model,
+ "org/model",
+ "main",
+ "https://huggingface.co";,
+ );
+
+ core.hf_list("path2", true).await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+
"https://huggingface.co/api/models/org/model/tree/main/path2?expand=True&recursive=True";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_resolve_url_model() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Model,
+ "user/model",
+ "main",
+ "https://huggingface.co";,
+ );
+
+ let args = OpRead::default();
+ core.hf_resolve("config.json", BytesRange::default(), &args)
+ .await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+ "https://huggingface.co/user/model/resolve/main/config.json";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_resolve_url_dataset() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Dataset,
+ "org/data",
+ "v1.0",
+ "https://huggingface.co";,
+ );
+
+ let args = OpRead::default();
+ core.hf_resolve("train.csv", BytesRange::default(), &args)
+ .await?;
+
+ let url = mock_client.get_captured_url();
+ assert_eq!(
+ url,
+ "https://huggingface.co/datasets/org/data/resolve/v1%2E0/train.csv";
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_hf_resolve_with_range() -> Result<()> {
+ let (core, mock_client) = create_test_core(
+ RepoType::Model,
+ "user/model",
+ "main",
+ "https://huggingface.co";,
+ );
+
+ let args = OpRead::default();
+ let range = BytesRange::new(0, Some(1024));
+ core.hf_resolve("large_file.bin", range, &args).await?;
+
+ let url = mock_client.get_captured_url();
+ let headers = mock_client.get_captured_headers();
+ assert_eq!(
+ url,
+ "https://huggingface.co/user/model/resolve/main/large_file.bin";
+ );
+ assert_eq!(headers.get(http::header::RANGE).unwrap(), "bytes=0-1023");
+
+ Ok(())
+ }
#[test]
fn parse_list_response_test() -> Result<()> {
