Xuanwo commented on code in PR #4281: URL: https://github.com/apache/opendal/pull/4281#discussion_r1505337907
########## core/src/services/github_contents/core.rs: ########## @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde_json::json; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "OpenDAL"; Review Comment: It's better to use `opendal` and include our [VERSION](https://opendal.apache.org/docs/rust/opendal/raw/constant.VERSION.html) ########## core/src/services/github_contents/core.rs: ########## @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde_json::json; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "OpenDAL"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubContentsCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubContentsCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubContentsCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, constants::USER_AGENT); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubContentsCore { + pub async fn get_file_sha(&self, path: &str) -> Result<String> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(sha.trim_matches('"').to_string()) + } + StatusCode::NOT_FOUND => Err(Error::new(ErrorKind::NotFound, "File not found")), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = json!({ + "message": "opendal upload", Review Comment: The message is `commit message` for this operation, right? We should allow user to custom it in the future. But currently, we can add path and current time into the message. ########## core/src/services/github_contents/core.rs: ########## @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde_json::json; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "OpenDAL"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubContentsCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubContentsCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubContentsCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, constants::USER_AGENT); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubContentsCore { + pub async fn get_file_sha(&self, path: &str) -> Result<String> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(sha.trim_matches('"').to_string()) + } + StatusCode::NOT_FOUND => Err(Error::new(ErrorKind::NotFound, "File not found")), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; Review Comment: Please placing sign request as the last operation. ########## core/src/services/github_contents/core.rs: ########## @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde_json::json; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "OpenDAL"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubContentsCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubContentsCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubContentsCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, constants::USER_AGENT); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubContentsCore { + pub async fn get_file_sha(&self, path: &str) -> Result<String> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(sha.trim_matches('"').to_string()) + } + StatusCode::NOT_FOUND => Err(Error::new(ErrorKind::NotFound, "File not found")), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = json!({ Review Comment: It's better to use a struct instead of `jons!()` ########## core/Cargo.toml: ########## @@ -156,6 +156,7 @@ services-gcs = [ ] services-gdrive = ["internal-path-cache"] services-ghac = [] +services-github-contents = [] Review Comment: Maybe use `github` is enough? Will we support other services for github? ########## core/src/services/github_contents/core.rs: ########## @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde_json::json; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "OpenDAL"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubContentsCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubContentsCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubContentsCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, constants::USER_AGENT); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubContentsCore { + pub async fn get_file_sha(&self, path: &str) -> Result<String> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(sha.trim_matches('"').to_string()) + } + StatusCode::NOT_FOUND => Err(Error::new(ErrorKind::NotFound, "File not found")), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = json!({ + "message": "opendal upload", + "content": base64::engine::general_purpose::STANDARD.encode(&bs), + }); + + if let Ok(sha) = sha { Review Comment: We can ignore the sha if fetch failed? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
