This is an automated email from the ASF dual-hosted git repository. hoslo pushed a commit to branch add-github-contents in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 1d934217e6bb81ec04d8539db757c92e1ed14c41 Author: hoslo <[email protected]> AuthorDate: Mon Feb 26 17:36:33 2024 +0800 feat(services/github_contents): add github contents support --- core/Cargo.toml | 1 + core/src/services/github_contents/backend.rs | 306 +++++++++++++++++++++++++++ core/src/services/github_contents/core.rs | 289 +++++++++++++++++++++++++ core/src/services/github_contents/docs.md | 54 +++++ core/src/services/github_contents/error.rs | 108 ++++++++++ core/src/services/github_contents/lister.rs | 66 ++++++ core/src/services/github_contents/mod.rs | 25 +++ core/src/services/github_contents/writer.rs | 64 ++++++ core/src/services/mod.rs | 7 + core/src/types/operator/builder.rs | 2 + core/src/types/scheme.rs | 4 + 11 files changed, 926 insertions(+) diff --git a/core/Cargo.toml b/core/Cargo.toml index dbfc4cd309..b8701237b3 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -156,6 +156,7 @@ services-gcs = [ ] services-gdrive = ["internal-path-cache"] services-ghac = [] +services-github-contents = [] services-gridfs = ["dep:mongodb"] services-hdfs = ["dep:hdrs"] services-http = [] diff --git a/core/src/services/github_contents/backend.rs b/core/src/services/github_contents/backend.rs new file mode 100644 index 0000000000..d9c6dee7f7 --- /dev/null +++ b/core/src/services/github_contents/backend.rs @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::sync::Arc; + +use async_trait::async_trait; +use http::StatusCode; +use log::debug; +use serde::Deserialize; + +use super::core::GithubContentsCore; +use super::error::parse_error; +use super::lister::GithubContentsLister; +use super::writer::GithubContentsWriter; +use super::writer::GithubContentsWriters; +use crate::raw::*; +use crate::*; + +/// Config for backblaze GithubContents services support. +#[derive(Default, Deserialize)] +#[serde(default)] +#[non_exhaustive] +pub struct GithubContentsConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option<String>, + /// Github access_token. + /// + /// required. + pub token: String, + /// Github repo owner. + /// + /// required. + pub owner: String, + /// Github repo name. + /// + /// required. + pub repo: String, +} + +impl Debug for GithubContentsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("GithubContentsConfig"); + + d.field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo); + + d.finish_non_exhaustive() + } +} + +/// [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct GithubContentsBuilder { + config: GithubContentsConfig, + + http_client: Option<HttpClient>, +} + +impl Debug for GithubContentsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("GithubContentsBuilder"); + + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + +impl GithubContentsBuilder { + /// Set root of this backend. + /// + /// All operations will happen under this root. + pub fn root(&mut self, root: &str) -> &mut Self { + self.config.root = if root.is_empty() { + None + } else { + Some(root.to_string()) + }; + + self + } + + /// Github access_token. + /// + /// required. + pub fn token(&mut self, token: &str) -> &mut Self { + self.config.token = token.to_string(); + + self + } + + /// Set Github repo owner. + pub fn owner(&mut self, owner: &str) -> &mut Self { + self.config.owner = owner.to_string(); + + self + } + + /// Set Github repo name. + pub fn repo(&mut self, repo: &str) -> &mut Self { + self.config.repo = repo.to_string(); + + self + } + + /// Specify the http client that used by this service. + /// + /// # Notes + /// + /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed + /// during minor updates. + pub fn http_client(&mut self, client: HttpClient) -> &mut Self { + self.http_client = Some(client); + self + } +} + +impl Builder for GithubContentsBuilder { + const SCHEME: Scheme = Scheme::GithubContents; + type Accessor = GithubContentsBackend; + + /// Converts a HashMap into an GithubContentsBuilder instance. + /// + /// # Arguments + /// + /// * `map` - A HashMap containing the configuration values. + /// + /// # Returns + /// + /// Returns an instance of GithubContentsBuilder. + fn from_map(map: HashMap<String, String>) -> Self { + // Deserialize the configuration from the HashMap. + let config = GithubContentsConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); + + // Create an GithubContentsBuilder instance with the deserialized config. + GithubContentsBuilder { + config, + http_client: None, + } + } + + /// Builds the backend and returns the result of GithubContentsBackend. + fn build(&mut self) -> Result<Self::Accessor> { + debug!("backend build started: {:?}", &self); + + let root = normalize_root(&self.config.root.clone().unwrap_or_default()); + debug!("backend use root {}", &root); + + // Handle token. + if self.config.token.is_empty() { + return Err(Error::new(ErrorKind::ConfigInvalid, "token is empty") + .with_operation("Builder::build") + .with_context("service", Scheme::GithubContents)); + } + + // Handle owner. + if self.config.owner.is_empty() { + return Err(Error::new(ErrorKind::ConfigInvalid, "owner is empty") + .with_operation("Builder::build") + .with_context("service", Scheme::GithubContents)); + } + + debug!("backend use owner {}", &self.config.owner); + + // Handle repo. + if self.config.repo.is_empty() { + return Err(Error::new(ErrorKind::ConfigInvalid, "repo is empty") + .with_operation("Builder::build") + .with_context("service", Scheme::GithubContents)); + } + + debug!("backend use repo {}", &self.config.repo); + + let client = if let Some(client) = self.http_client.take() { + client + } else { + HttpClient::new().map_err(|err| { + err.with_operation("Builder::build") + .with_context("service", Scheme::GithubContents) + })? + }; + + Ok(GithubContentsBackend { + core: Arc::new(GithubContentsCore { + root, + token: self.config.token.clone(), + owner: self.config.owner.clone(), + repo: self.config.repo.clone(), + client, + }), + }) + } +} + +/// Backend for GithubContents services. +#[derive(Debug, Clone)] +pub struct GithubContentsBackend { + core: Arc<GithubContentsCore>, +} + +#[async_trait] +impl Accessor for GithubContentsBackend { + type Reader = IncomingAsyncBody; + + type Writer = GithubContentsWriters; + + type Lister = oio::PageLister<GithubContentsLister>; + + type BlockingReader = (); + + type BlockingWriter = (); + + type BlockingLister = (); + + fn info(&self) -> AccessorInfo { + let mut am = AccessorInfo::default(); + am.set_scheme(Scheme::GithubContents) + .set_root(&self.core.root) + .set_native_capability(Capability { + stat: true, + + read: true, + + write: true, + write_can_empty: true, + + delete: true, + + list: true, + + ..Default::default() + }); + + am + } + + async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> { + let resp = self.core.stat(path).await?; + + let status = resp.status(); + + match status { + StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new), + _ => Err(parse_error(resp).await?), + } + } + + async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> { + let resp = self.core.get(path).await?; + + let status = resp.status(); + + match status { + StatusCode::OK => { + let size = parse_content_length(resp.headers())?; + let range = parse_content_range(resp.headers())?; + Ok(( + RpRead::new().with_size(size).with_range(range), + resp.into_body(), + )) + } + _ => Err(parse_error(resp).await?), + } + } + + async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> { + let writer = GithubContentsWriter::new(self.core.clone(), path.to_string()); + + let w = oio::OneShotWriter::new(writer); + + Ok((RpWrite::default(), w)) + } + + async fn delete(&self, path: &str, _: OpDelete) -> Result<RpDelete> { + match self.core.delete(path).await { + Ok(_) => Ok(RpDelete::default()), + Err(err) => Err(err), + } + } + + async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> { + let l = GithubContentsLister::new(self.core.clone(), path); + Ok((RpList::default(), oio::PageLister::new(l))) + } +} diff --git a/core/src/services/github_contents/core.rs b/core/src/services/github_contents/core.rs new file mode 100644 index 0000000000..d6afc3f6ea --- /dev/null +++ b/core/src/services/github_contents/core.rs @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde_json::json; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "OpenDAL"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubContentsCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubContentsCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubContentsCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, constants::USER_AGENT); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubContentsCore { + pub async fn get_file_sha(&self, path: &str) -> Result<String> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(sha.trim_matches('"').to_string()) + } + StatusCode::NOT_FOUND => Err(Error::new(ErrorKind::NotFound, "File not found")), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = json!({ + "message": "opendal upload", + "content": base64::engine::general_purpose::STANDARD.encode(&bs), + }); + + if let Ok(sha) = sha { + req_body["sha"] = serde_json::Value::String(sha); + } + + let req = req + .header("Accept", "application/vnd.github+json") + .body(AsyncBody::Bytes(Bytes::from(req_body.to_string()))) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn delete(&self, path: &str) -> Result<()> { + let sha = self.get_file_sha(path).await; + + match sha { + Ok(sha) => { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::delete(url); + + let req = self.sign(req)?; + + let req_body = json!({ + "message": "opendal delete", + "sha": sha, + }); + + let req = req + .header("Accept", "application/vnd.github.object+json") + .body(AsyncBody::Bytes(Bytes::from(req_body.to_string()))) + .map_err(new_request_build_error)?; + + let resp = self.send(req).await?; + + match resp.status() { + StatusCode::OK => Ok(()), + _ => Err(parse_error(resp).await?), + } + } + Err(err) => { + if err.kind() == ErrorKind::NotFound { + return Ok(()); + } + Err(err) + } + } + } + + pub async fn list(&self, path: &str) -> Result<Vec<Entry>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + let resp = self.send(req).await?; + + match resp.status() { + StatusCode::OK => { + let body = resp.into_body().bytes().await?; + let resp: ListResponse = + serde_json::from_slice(&body).map_err(new_json_deserialize_error)?; + + Ok(resp.entries) + } + _ => Err(parse_error(resp).await?), + } + } +} + +#[derive(Default, Debug, Clone, Deserialize)] +pub struct ListResponse { + pub entries: Vec<Entry>, +} + +#[derive(Default, Debug, Clone, Deserialize)] +pub struct Entry { + pub name: String, + pub path: String, + pub sha: String, + pub size: u64, + pub url: String, + pub html_url: String, + pub git_url: String, + pub download_url: String, + #[serde(rename = "type")] + pub type_field: String, + pub content: Option<String>, + pub encoding: Option<String>, + #[serde(rename = "_links")] + pub links: Links, +} + +#[derive(Default, Debug, Clone, Deserialize)] +pub struct Links { + #[serde(rename = "self")] + pub self_field: String, + pub git: String, + pub html: String, +} diff --git a/core/src/services/github_contents/docs.md b/core/src/services/github_contents/docs.md new file mode 100644 index 0000000000..f5427982eb --- /dev/null +++ b/core/src/services/github_contents/docs.md @@ -0,0 +1,54 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [ ] create_dir +- [x] delete +- [ ] copy +- [ ] rename +- [x] list +- [ ] scan +- [ ] presign +- [ ] blocking + +## Configuration + +- `root`: Set the work directory for backend +- `token`: Github access token +- `owner`: Github owner +- `repo`: Github repository + +You can refer to [`GithubContentsBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::GithubContents; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // create backend builder + let mut builder = GithubContents::default(); + + // set the storage root for OpenDAL + builder.root("/"); + // set the access token for Github API + builder.token("your_access_token"); + // set the owner for Github + builder.owner("your_owner") + // set the repository for Github + builder.repo("your_repo"); + + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} +``` diff --git a/core/src/services/github_contents/error.rs b/core/src/services/github_contents/error.rs new file mode 100644 index 0000000000..61a6c65840 --- /dev/null +++ b/core/src/services/github_contents/error.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Buf; +use http::Response; +use serde::Deserialize; + +use crate::raw::*; +use crate::Error; +use crate::ErrorKind; +use crate::Result; + +#[derive(Default, Debug, Deserialize)] +#[allow(dead_code)] +struct GithubContentsError { + error: GithubContentsSubError, +} + +#[derive(Default, Debug, Deserialize)] +#[allow(dead_code)] +struct GithubContentsSubError { + message: String, + documentation_url: String, +} + +/// Parse error response into Error. +pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> { + let (parts, body) = resp.into_parts(); + let bs = body.bytes().await?; + + let (kind, retryable) = match parts.status.as_u16() { + 401 | 403 => (ErrorKind::PermissionDenied, false), + 404 => (ErrorKind::NotFound, false), + 304 | 412 => (ErrorKind::ConditionNotMatch, false), + // https://github.com/apache/opendal/issues/4146 + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/423 + // We should retry it when we get 423 error. + 423 => (ErrorKind::RateLimited, true), + // Service like Upyun could return 499 error with a message like: + // Client Disconnect, we should retry it. + 499 => (ErrorKind::Unexpected, true), + 500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true), + _ => (ErrorKind::Unexpected, false), + }; + + let (message, _github_content_err) = + serde_json::from_reader::<_, GithubContentsError>(bs.clone().reader()) + .map(|github_content_err| (format!("{github_content_err:?}"), Some(github_content_err))) + .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None)); + + let mut err = Error::new(kind, &message); + + err = with_error_response_context(err, parts); + + if retryable { + err = err.set_temporary(); + } + + Ok(err) +} + +#[cfg(test)] +mod test { + use futures::stream; + use http::StatusCode; + + use super::*; + + #[tokio::test] + async fn test_parse_error() { + let err_res = vec![( + r#"{ + "message": "Not Found", + "documentation_url": "https://docs.github.com/rest/repos/contents#get-repository-content"; + }"#, + ErrorKind::NotFound, + StatusCode::NOT_FOUND, + )]; + + for res in err_res { + let bs = bytes::Bytes::from(res.0); + let body = IncomingAsyncBody::new( + Box::new(oio::into_stream(stream::iter(vec![Ok(bs.clone())]))), + None, + ); + let resp = Response::builder().status(res.2).body(body).unwrap(); + + let err = parse_error(resp).await; + + assert!(err.is_ok()); + assert_eq!(err.unwrap().kind(), res.1); + } + } +} diff --git a/core/src/services/github_contents/lister.rs b/core/src/services/github_contents/lister.rs new file mode 100644 index 0000000000..35ff23bb4e --- /dev/null +++ b/core/src/services/github_contents/lister.rs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; + +use super::core::GithubContentsCore; +use crate::raw::oio::Entry; +use crate::raw::*; +use crate::*; + +pub struct GithubContentsLister { + core: Arc<GithubContentsCore>, + path: String, +} + +impl GithubContentsLister { + pub fn new(core: Arc<GithubContentsCore>, path: &str) -> Self { + Self { + core, + + path: path.to_string(), + } + } +} + +#[async_trait] +impl oio::PageList for GithubContentsLister { + async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> { + let entrys = self.core.list(&self.path).await?; + + for entry in entrys { + let path = build_rel_path(&self.core.root, &entry.path); + let entry = if entry.type_field == "dir" { + let path = format!("{}/", path); + Entry::new(&path, Metadata::new(EntryMode::DIR)) + } else { + let m = Metadata::new(EntryMode::FILE) + .with_content_length(entry.size) + .with_etag(entry.sha); + Entry::new(&path, m) + }; + + ctx.entries.push_back(entry); + } + + ctx.done = true; + + Ok(()) + } +} diff --git a/core/src/services/github_contents/mod.rs b/core/src/services/github_contents/mod.rs new file mode 100644 index 0000000000..e571afca63 --- /dev/null +++ b/core/src/services/github_contents/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod backend; +pub use backend::GithubContentsBuilder as GithubContents; +pub use backend::GithubContentsConfig; + +mod core; +mod error; +mod lister; +mod writer; diff --git a/core/src/services/github_contents/writer.rs b/core/src/services/github_contents/writer.rs new file mode 100644 index 0000000000..d0fd096de1 --- /dev/null +++ b/core/src/services/github_contents/writer.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; +use http::StatusCode; + +use super::core::GithubContentsCore; +use super::error::parse_error; +use crate::raw::*; +use crate::*; + +pub type GithubContentsWriters = oio::OneShotWriter<GithubContentsWriter>; + +pub struct GithubContentsWriter { + core: Arc<GithubContentsCore>, + path: String, +} + +impl GithubContentsWriter { + pub fn new(core: Arc<GithubContentsCore>, path: String) -> Self { + GithubContentsWriter { core, path } + } +} + +#[async_trait] +impl oio::OneShotWrite for GithubContentsWriter { + async fn write_once(&self, bs: &dyn oio::WriteBuf) -> Result<()> { + let bs = bs.bytes(bs.remaining()); + + let path = if self.path.ends_with('/') { + format!("{}{}", self.path, self.path.trim_end_matches('/')) + } else { + self.path.clone() + }; + + let resp = self.core.upload(&path, bs).await?; + + let status = resp.status(); + + match status { + StatusCode::OK | StatusCode::CREATED => { + resp.into_body().consume().await?; + Ok(()) + } + _ => Err(parse_error(resp).await?), + } + } +} diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index de5c0cef02..92d26fc1e4 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -229,6 +229,13 @@ mod gdrive; #[cfg(feature = "services-gdrive")] pub use gdrive::Gdrive; +#[cfg(feature = "services-github-contents")] +mod github_contents; +#[cfg(feature = "services-github-contents")] +pub use github_contents::GithubContents; +#[cfg(feature = "services-github-contents")] +pub use github_contents::GithubContentsConfig; + #[cfg(feature = "services-dropbox")] mod dropbox; #[cfg(feature = "services-dropbox")] diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs index a5767f59e3..6827742101 100644 --- a/core/src/types/operator/builder.rs +++ b/core/src/types/operator/builder.rs @@ -199,6 +199,8 @@ impl Operator { Scheme::Ghac => Self::from_map::<services::Ghac>(map)?.finish(), #[cfg(feature = "services-gridfs")] Scheme::Gridfs => Self::from_map::<services::Gridfs>(map)?.finish(), + #[cfg(feature = "services-github-contents")] + Scheme::GithubContents => Self::from_map::<services::GithubContents>(map)?.finish(), #[cfg(feature = "services-hdfs")] Scheme::Hdfs => Self::from_map::<services::Hdfs>(map)?.finish(), #[cfg(feature = "services-http")] diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 6620fa8145..c55148dcf3 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -151,6 +151,8 @@ pub enum Scheme { Mongodb, /// [gridfs](crate::services::gridfs): MongoDB Gridfs Services Gridfs, + /// [Github Contents][crate::services::GithubContents]: Github contents support. + GithubContents, /// [Native HDFS](crate::services::hdfs_native): Hdfs Native service, using rust hdfs-native client for hdfs HdfsNative, /// Custom that allow users to implement services outside of OpenDAL. @@ -338,6 +340,7 @@ impl FromStr for Scheme { "gdrive" => Ok(Scheme::Gdrive), "ghac" => Ok(Scheme::Ghac), "gridfs" => Ok(Scheme::Gridfs), + "github_contents" => Ok(Scheme::GithubContents), "hdfs" => Ok(Scheme::Hdfs), "http" | "https" => Ok(Scheme::Http), "huggingface" | "hf" => Ok(Scheme::Huggingface), @@ -422,6 +425,7 @@ impl From<Scheme> for &'static str { Scheme::Postgresql => "postgresql", Scheme::Mysql => "mysql", Scheme::Gdrive => "gdrive", + Scheme::GithubContents => "github_contents", Scheme::Dropbox => "dropbox", Scheme::Redis => "redis", Scheme::Rocksdb => "rocksdb",
