This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch add-test-for-r2 in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 6edebd141156ac91325d46ac9d08b90ffd5a38bf Author: Xuanwo <[email protected]> AuthorDate: Tue Jun 6 13:41:24 2023 +0800 ci: Add integration tests for Cloudflare R2 Signed-off-by: Xuanwo <[email protected]> --- .github/workflows/service_test_s3.yml | 32 +++++++++++++++++++++++++++++ core/src/services/s3/backend.rs | 6 +++++- core/src/services/s3/compatible_services.md | 14 +++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 88266f81c..ef3eff066 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -178,3 +178,35 @@ jobs: OPENDAL_S3_BUCKET: test OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"; OPENDAL_S3_ALLOW_ANONYMOUS: on + + r2: + runs-on: ubuntu-latest + if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork + steps: + - uses: actions/checkout@v3 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + need-nextest: true + + - name: Load secret + id: op-load-secret + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + OPENDAL_S3_TEST: op://services/r2/test + OPENDAL_S3_BUCKET: op://services/r2/bucket + OPENDAL_S3_ENDPOINT: op://services/r2/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key + OPENDAL_S3_REGION: auto + # This is the R2's limitation + # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information + OPENDAL_S3_BATCH_MAX_OPERATIONS: 700 + + - name: Test + shell: bash + working-directory: core + run: cargo nextest run s3 diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index be720a8f9..056db3e59 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -560,7 +560,11 @@ impl Builder for S3Builder { .filter(|v| *v == "on" || *v == "true") .map(|_| builder.allow_anonymous()); map.get("default_storage_class") - .map(|v| builder.default_storage_class(v)); + .map(|v: &String| builder.default_storage_class(v)); + map.get("write_min_size") + .map(|v| builder.write_min_size(v.parse().expect("input must be a number"))); + map.get("batch_max_operations") + .map(|v| builder.batch_max_operations(v.parse().expect("input must be a number"))); builder } diff --git a/core/src/services/s3/compatible_services.md b/core/src/services/s3/compatible_services.md index 432ce7c89..d5036cb3d 100644 --- a/core/src/services/s3/compatible_services.md +++ b/core/src/services/s3/compatible_services.md @@ -97,3 +97,17 @@ To connect to wasabi, we need to set: - `bucket`: The bucket name of wasabi. > Refer to [What are the service URLs for Wasabi's different storage > regions?](https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031) > for more details. + +### Cloudflare R2 + +[Cloudflare R2](https://developers.cloudflare.com/r2/) provides s3 compatible API. + +> Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services. + + +To connect to r2, we need to set: + +- `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com` +- `bucket`: The bucket name of r2. +- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region. +- `batch_max_operations`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please set this value `<= 700` to make sure batch delete work as expected.
