This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/release/v1.2/pr-8555-e49f3379f1a5eac75d7fbfbb54a14503f168da87 in repository https://gitbox.apache.org/repos/asf/texera.git
commit ba12d7439249e328bb1b1d758fcc110f046afa73 Author: Xuan Gu <[email protected]> AuthorDate: Fri Sep 18 02:34:24 2026 +0000 feat(deploy, v1.2): replace MinIO with RustFS as the default object store (#8555) ### What changes were proposed in this PR? Backport of #8542 to `release/v1.2`. Fresh 1.2 deployments are broken by the MinIO image removal: the released compose tarball fails at image pull, the K8s lakekeeper-init job dies downloading `mc` from `dl.min.io` (410 Gone), and CI on this branch fails at image pull. Main adaptations for v1.2: - The chart has no `_helpers.tpl` and no external-S3 support, so templates reference the RustFS subchart's service/Secret directly, including two deployment templates that hardcode the MinIO secret on v1.2. - lakekeeper-init creates the bucket from an `rc` init container instead of downloading `mc` at runtime; CI uses the preinstalled AWS CLI. - Test files are adjusted to v1.2's package layout; the `StagedFileCleanupJob` change does not apply here. Migration note: volume/claim names change (`minio_data` -> `rustfs_data`, `minio-data-pvc` -> `rustfs-data-pvc`); no in-place data migration for now. ### Any related issues, documentation, discussions? Discussion in #3998. Original change: #8542. ### How was this PR tested? Compose and Helm render cleanly with no minio left. Tested the single-node deployment. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Fable 5 (Claude Code) --------- Co-authored-by: Ian Maxon <[email protected]> Co-authored-by: Claude Fable 5 <[email protected]> Co-authored-by: Xinyuan Lin <[email protected]> Co-authored-by: Meng Wang <[email protected]> --- .github/workflows/build.yml | 28 +++++--- amber/LICENSE-binary-python | 3 +- .../texera/web/service/WorkflowService.scala | 2 +- .../packaging/test_state_materialization_e2e.py | 2 +- .../core/storage/iceberg/test_iceberg_document.py | 4 +- .../pytexera/storage/test_large_binary_manager.py | 4 +- bin/bootstrap-lakekeeper.sh | 2 +- bin/k8s/Chart.yaml | 6 +- bin/k8s/templates/external-names.yaml | 6 +- bin/k8s/templates/file-service-deployment.yaml | 10 +-- bin/k8s/templates/gateway-routes.yaml | 10 +-- bin/k8s/templates/gateway.yaml | 12 ++-- bin/k8s/templates/lakekeeper-init-job.yaml | 63 ++++++++++++----- ...io-persistence.yaml => rustfs-persistence.yaml} | 8 +-- ...workflow-computing-unit-manager-deployment.yaml | 10 +-- bin/k8s/values-development.yaml | 52 +++++++++----- bin/k8s/values.yaml | 67 +++++++++++++----- bin/local-dev/README.md | 2 +- bin/single-node/.env | 10 +-- bin/single-node/README.md | 4 +- bin/single-node/docker-compose.yml | 65 ++++++++++-------- build.sbt | 1 + common/config/src/main/resources/storage.conf | 2 +- common/workflow-core/build.sbt | 5 +- .../texera/service/util/S3StorageClient.scala | 8 +-- .../texera/service/util/RustFSContainer.scala | 80 ++++++++++++++++++++++ .../texera/service/util/S3StorageTestBase.scala | 21 +++--- .../guide-for-developers.md | 6 +- docs/getting-started/installing-using-docker.md | 4 +- docs/getting-started/run-on-kubernetes.md | 4 +- docs/overview.md | 2 +- docs/tutorials/guide-to-launch-lakekeeper.md | 6 +- file-service/build.sbt | 3 +- file-service/src/main/resources/docker-compose.yml | 25 ++++--- .../{minio-config.yml => rustfs-config.yml} | 23 ++++--- .../org/apache/texera/service/MockLakeFS.scala | 60 ++++++++-------- 36 files changed, 401 insertions(+), 219 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e3ada2c31..2d468dac51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -399,17 +399,19 @@ jobs: run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases -f sql/texera_ddl.sql env: PGPASSWORD: postgres - - name: Start MinIO + - name: Start RustFS run: | - docker run -d --name minio --network host \ - -e MINIO_ROOT_USER=texera_minio \ - -e MINIO_ROOT_PASSWORD=password \ - minio/minio:RELEASE.2025-02-28T09-55-16Z server /data - for i in $(seq 1 3); do - curl -sf http://localhost:9000/minio/health/live && break - echo "Waiting for MinIO... (attempt $i)" + docker run -d --name rustfs --network host \ + -e RUSTFS_ACCESS_KEY=texera_rustfs \ + -e RUSTFS_SECRET_KEY=password \ + -e RUSTFS_REGION=us-west-2 \ + rustfs/rustfs:1.0.0-rc.6 + for i in $(seq 1 15); do + curl -sf http://localhost:9000/health && break + echo "Waiting for RustFS... (attempt $i)" sleep 1 done + curl -sf http://localhost:9000/health - name: Start Lakekeeper env: LAKEKEEPER__PG_DATABASE_URL_READ: postgres://postgres:postgres@localhost:5432/texera_lakekeeper @@ -461,9 +463,13 @@ jobs: LAKEKEEPER_BASE=${REST_URI%/catalog} LAKEKEEPER_BASE=${LAKEKEEPER_BASE%/} - docker run --rm --network host --entrypoint sh minio/mc -c \ - "mc alias set minio $S3_ENDPOINT $S3_USERNAME $S3_PASSWORD && \ - mc mb --ignore-existing minio/$S3_BUCKET" + # Bucket creation goes through the AWS CLI, which is preinstalled on + # the runner image. CreateBucket on an existing bucket is a no-op on + # RustFS, which keeps re-runs idempotent. + AWS_ACCESS_KEY_ID="$S3_USERNAME" \ + AWS_SECRET_ACCESS_KEY="$S3_PASSWORD" \ + AWS_DEFAULT_REGION="$S3_REGION" \ + aws s3api create-bucket --bucket "$S3_BUCKET" --endpoint-url "$S3_ENDPOINT" curl -sf -X POST -H 'Content-Type: application/json' \ -d '{"project-id":"00000000-0000-0000-0000-000000000000","project-name":"default"}' \ "$LAKEKEEPER_BASE/management/v1/project" || true diff --git a/amber/LICENSE-binary-python b/amber/LICENSE-binary-python index a571897d52..11fd33977a 100644 --- a/amber/LICENSE-binary-python +++ b/amber/LICENSE-binary-python @@ -300,6 +300,7 @@ Dependencies under the BSD 3-Clause License Python packages: - cached-property==1.5.2 - click==8.4.2 + - cloudpickle==3.1.2 - contourpy==1.3.3 - cycler==0.12.1 - fsspec==2025.9.0 @@ -365,7 +366,7 @@ Dependencies under the Python Software Foundation License Python packages: - aiohappyeyeballs==2.7.1 - matplotlib==3.11.1 - - typing-extensions==4.14.1 + - typing-extensions==4.16.0 -------------------------------------------------------------------------------- Dependencies under the MIT-CMU License diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index f6c67159ae..b1cc506b5d 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -349,7 +349,7 @@ class WorkflowService( * 2. Clears URI references from the execution registry * 3. Safely clears all result and console message documents * 4. Expires Iceberg snapshots for runtime statistics - * 5. Deletes this execution's large binaries from MinIO + * 5. Deletes this execution's large binaries from the object store * * @param eid The execution identity to clean up resources for */ diff --git a/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py b/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py index cfc4f7f676..93508b6e9a 100644 --- a/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py +++ b/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py @@ -80,7 +80,7 @@ _WAREHOUSE_DIR = tempfile.mkdtemp(prefix="texera-state-e2e-warehouse-") @pytest.fixture(scope="module", autouse=True) def sqlite_iceberg_catalog(): """Inject a sqlite-backed SqlCatalog so the test runs without external - iceberg infra (postgres/minio). + iceberg infra (postgres/rustfs). Module-scoped so all tests in this file share one warehouse, and so namespace creation only happens once. We save/restore the original diff --git a/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py b/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py index 5e61b4ed68..0b264b4508 100644 --- a/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py +++ b/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py @@ -55,8 +55,8 @@ StorageConfig.initialize( commit_batch_size=4096, s3_endpoint="http://localhost:9000";, s3_region="us-east-1", - s3_auth_username="minioadmin", - s3_auth_password="minioadmin", + s3_auth_username="rustfsadmin", + s3_auth_password="rustfsadmin", s3_large_binaries_base_uri="s3://texera-large-binaries/objects/0/", ) diff --git a/amber/src/test/python/pytexera/storage/test_large_binary_manager.py b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py index e1da5eee18..ae24df206a 100644 --- a/amber/src/test/python/pytexera/storage/test_large_binary_manager.py +++ b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py @@ -46,8 +46,8 @@ def _init_storage_config(): commit_batch_size=1000, s3_endpoint="http://localhost:9000";, s3_region="us-east-1", - s3_auth_username="minioadmin", - s3_auth_password="minioadmin", + s3_auth_username="rustfsadmin", + s3_auth_password="rustfsadmin", s3_large_binaries_base_uri=TEST_BASE_URI, ) diff --git a/bin/bootstrap-lakekeeper.sh b/bin/bootstrap-lakekeeper.sh index 17452e9f7d..3fd9f1d6a5 100755 --- a/bin/bootstrap-lakekeeper.sh +++ b/bin/bootstrap-lakekeeper.sh @@ -42,7 +42,7 @@ STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME="${STORAGE_ICEBERG_CATALOG_REST_WARE STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET="${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET:-texera-iceberg}" STORAGE_S3_REGION="${STORAGE_S3_REGION:-us-west-2}" STORAGE_S3_ENDPOINT="${STORAGE_S3_ENDPOINT:-http://localhost:9000}"; -STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_minio}" +STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_rustfs}" STORAGE_S3_AUTH_PASSWORD="${STORAGE_S3_AUTH_PASSWORD:-password}" # Lakekeeper binary — defaults to `lakekeeper` on $PATH (e.g. after diff --git a/bin/k8s/Chart.yaml b/bin/k8s/Chart.yaml index 548d9e531e..5bd5f1fa98 100644 --- a/bin/k8s/Chart.yaml +++ b/bin/k8s/Chart.yaml @@ -46,9 +46,9 @@ dependencies: version: 16.5.6 repository: https://charts.bitnami.com/bitnami - - name: minio - version: 15.0.7 - repository: https://charts.bitnami.com/bitnami + - name: rustfs + version: 1.0.0-rc.6 + repository: https://charts.rustfs.com - name: lakefs version: 1.8.1 diff --git a/bin/k8s/templates/external-names.yaml b/bin/k8s/templates/external-names.yaml index c79ce40f33..0e180bd37b 100644 --- a/bin/k8s/templates/external-names.yaml +++ b/bin/k8s/templates/external-names.yaml @@ -74,11 +74,11 @@ to access services in the main namespace using the same service names. ) | nindent 0 }} --- -{{/* MinIO ExternalName */}} +{{/* RustFS ExternalName */}} {{- include "external-name-service" (dict - "name" (printf "%s-minio" .Release.Name) + "name" (printf "%s-rustfs-svc" .Release.Name) "namespace" $workflowComputingUnitPoolNamespace - "externalName" (printf "%s-minio.%s.svc.cluster.local" .Release.Name $namespace) + "externalName" (printf "%s-rustfs-svc.%s.svc.cluster.local" .Release.Name $namespace) ) | nindent 0 }} --- diff --git a/bin/k8s/templates/file-service-deployment.yaml b/bin/k8s/templates/file-service-deployment.yaml index 6c9a4041f9..f56085737a 100644 --- a/bin/k8s/templates/file-service-deployment.yaml +++ b/bin/k8s/templates/file-service-deployment.yaml @@ -41,17 +41,17 @@ spec: env: # LakeFS & S3 Access - name: STORAGE_S3_ENDPOINT - value: http://{{ .Release.Name }}-minio:9000 + value: http://{{ .Release.Name }}-rustfs-svc:9000 - name: STORAGE_S3_AUTH_USERNAME valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-user + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_ACCESS_KEY - name: STORAGE_S3_AUTH_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-password + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_SECRET_KEY - name: STORAGE_LAKEFS_ENDPOINT value: http://{{ .Release.Name }}-lakefs:8000/api/v1 - name: STORAGE_LAKEFS_AUTH_USERNAME diff --git a/bin/k8s/templates/gateway-routes.yaml b/bin/k8s/templates/gateway-routes.yaml index 935d38bb07..ace2fc5ee7 100644 --- a/bin/k8s/templates/gateway-routes.yaml +++ b/bin/k8s/templates/gateway-routes.yaml @@ -159,24 +159,24 @@ spec: port: 3001 {{- end }} --- -# MinIO Route -{{- if .Values.minio.gateway.enabled }} +# RustFS Route +{{- if .Values.rustfs.gateway.enabled }} apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: - name: texera-minio-route + name: texera-rustfs-route namespace: {{ .Release.Namespace }} spec: parentRefs: - name: {{ .Release.Name }}-gateway hostnames: - - {{ .Values.minio.gateway.hostname }} + - {{ .Values.rustfs.gateway.hostname }} rules: - matches: - path: type: PathPrefix value: / backendRefs: - - name: {{ .Release.Name }}-minio + - name: {{ .Release.Name }}-rustfs-svc port: 9000 {{- end }} diff --git a/bin/k8s/templates/gateway.yaml b/bin/k8s/templates/gateway.yaml index 868439a673..dc79e39d6d 100644 --- a/bin/k8s/templates/gateway.yaml +++ b/bin/k8s/templates/gateway.yaml @@ -59,23 +59,23 @@ spec: mode: Terminate certificateRefs: - name: {{ (index .Values "gatewayConfig" "tlsSecretName") | default (printf "%s-cert" .Release.Name) }} - {{- if .Values.minio.gateway.enabled }} - - name: minio-http + {{- if .Values.rustfs.gateway.enabled }} + - name: rustfs-http protocol: HTTP port: 80 - hostname: {{ .Values.minio.gateway.hostname }} + hostname: {{ .Values.rustfs.gateway.hostname }} allowedRoutes: namespaces: from: Same - - name: minio-https + - name: rustfs-https protocol: HTTPS port: 443 - hostname: {{ .Values.minio.gateway.hostname }} + hostname: {{ .Values.rustfs.gateway.hostname }} allowedRoutes: namespaces: from: Same tls: mode: Terminate certificateRefs: - - name: {{ .Values.minio.gateway.tlsSecretName | default (printf "%s-minio-tls" .Release.Name) }} + - name: {{ .Values.rustfs.gateway.tlsSecretName | default (printf "%s-rustfs-tls" .Release.Name) }} {{- end }} diff --git a/bin/k8s/templates/lakekeeper-init-job.yaml b/bin/k8s/templates/lakekeeper-init-job.yaml index 0a3540b605..161638c896 100644 --- a/bin/k8s/templates/lakekeeper-init-job.yaml +++ b/bin/k8s/templates/lakekeeper-init-job.yaml @@ -28,22 +28,62 @@ spec: name: {{ .Release.Name }}-lakekeeper-init spec: restartPolicy: Never + # Creates the Iceberg warehouse bucket before the warehouse is registered. + # `rc` is RustFS's S3 client; using its image keeps the client out of the + # main container, which would otherwise have to fetch a binary at runtime. + initContainers: + - name: create-bucket + image: {{ .Values.lakekeeperInit.bucketClientImage }} + env: + - name: STORAGE_S3_ENDPOINT + value: http://{{ .Release.Name }}-rustfs-svc:9000 + - name: STORAGE_S3_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_ACCESS_KEY + - name: STORAGE_S3_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_SECRET_KEY + - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET + value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }} + - name: STORAGE_ICEBERG_CATALOG_REST_REGION + value: {{ .Values.lakekeeperInit.warehouse.region | quote }} + command: + - /bin/sh + - -c + - | + set -e + + echo "Waiting for the object store at ${STORAGE_S3_ENDPOINT}..." + until rc alias set local "${STORAGE_S3_ENDPOINT}" \ + "${STORAGE_S3_AUTH_USERNAME}" "${STORAGE_S3_AUTH_PASSWORD}" \ + --region "${STORAGE_ICEBERG_CATALOG_REST_REGION}" \ + --bucket-lookup path > /dev/null 2>&1; do + sleep 3 + done + + echo "Initializing object-store bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..." + rc bucket create --ignore-existing "local/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}" + echo "Bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' is ready." containers: - name: lakekeeper-init image: alpine:3.19 env: - name: STORAGE_S3_ENDPOINT - value: http://{{ .Release.Name }}-minio:9000 + value: http://{{ .Release.Name }}-rustfs-svc:9000 - name: STORAGE_S3_AUTH_USERNAME valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-user + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_ACCESS_KEY - name: STORAGE_S3_AUTH_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-password + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_SECRET_KEY - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }} - name: STORAGE_ICEBERG_CATALOG_REST_REGION @@ -62,9 +102,7 @@ spec: - | set -e - apk add --no-cache curl ca-certificates wget - wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc - chmod +x /usr/local/bin/mc + apk add --no-cache curl ca-certificates check_status() { if [ "$1" -ge 200 ] && [ "$1" -lt 300 ]; then @@ -85,14 +123,7 @@ spec: sleep 3 done - echo "Step 1: Initializing MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..." - mc alias set minio "${STORAGE_S3_ENDPOINT}" "${STORAGE_S3_AUTH_USERNAME}" "${STORAGE_S3_AUTH_PASSWORD}" || true - if mc ls minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} > /dev/null 2>&1; then - echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' already exists." - else - mc mb minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} - echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' created successfully." - fi + echo "Step 1: object-store bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' was created by the create-bucket init container." echo "Step 2: Initializing default project..." PROJECT_PAYLOAD="{\"project-id\":\"${LAKEKEEPER_PROJECT_ID}\",\"project-name\":\"${LAKEKEEPER_PROJECT_NAME}\"}" diff --git a/bin/k8s/templates/minio-persistence.yaml b/bin/k8s/templates/rustfs-persistence.yaml similarity index 92% rename from bin/k8s/templates/minio-persistence.yaml rename to bin/k8s/templates/rustfs-persistence.yaml index 94d22e5e3a..173f93481d 100644 --- a/bin/k8s/templates/minio-persistence.yaml +++ b/bin/k8s/templates/rustfs-persistence.yaml @@ -17,11 +17,11 @@ {{/* Define storage path configuration, please change it to your own path and make sure the path exists with the right permission*/}} {{/* This path only works for local-path storage class */}} -{{- $hostBasePath := .Values.persistence.minioHostLocalPath }} +{{- $hostBasePath := .Values.persistence.rustfsHostLocalPath }} -{{- if .Values.minio.persistence.enabled }} -{{- $name := "minio" }} -{{- $persistence := .Values.minio.persistence }} +{{- if .Values.rustfs.persistence.enabled }} +{{- $name := "rustfs" }} +{{- $persistence := .Values.rustfs.persistence }} {{- $volumeName := printf "%s-data-pv" $name }} {{- $claimName := printf "%s-data-pvc" $name }} {{- $storageClass := $persistence.storageClass | default "local-path" }} diff --git a/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml index 7a0185cd46..299297f133 100644 --- a/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml @@ -81,17 +81,17 @@ spec: value: http://{{ .Values.fileService.name }}-svc:9092/api/dataset/did/upload # S3 Access (for R UDF large binary support) - name: STORAGE_S3_ENDPOINT - value: http://{{ .Release.Name }}-minio:9000 + value: http://{{ .Release.Name }}-rustfs-svc:9000 - name: STORAGE_S3_AUTH_USERNAME valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-user + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_ACCESS_KEY - name: STORAGE_S3_AUTH_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-minio - key: root-password + name: {{ .Release.Name }}-rustfs-secret + key: RUSTFS_SECRET_KEY # LakeFS Access (should be removed in production environment) - name: STORAGE_LAKEFS_ENDPOINT value: http://{{ .Release.Name }}-lakefs.{{ .Release.Namespace }}:8000/api/v1 diff --git a/bin/k8s/values-development.yaml b/bin/k8s/values-development.yaml index aa64d6ecbc..db82a3b417 100644 --- a/bin/k8s/values-development.yaml +++ b/bin/k8s/values-development.yaml @@ -22,7 +22,7 @@ texera: imageTag: latest global: - # Required by Bitnami sub-charts (postgresql, minio) to allow custom images + # Required by the Bitnami postgresql sub-chart to allow custom images security: allowInsecureImages: true @@ -34,10 +34,10 @@ global: # - false: PVCs will remain after uninstall to preserve the data persistence: removeAfterUninstall: true - minioHostLocalPath: "" + rustfsHostLocalPath: "" postgresqlHostLocalPath: "" -# Part 1: the configuration of Postgres, Minio and LakeFS +# Part 1: the configuration of Postgres, RustFS and LakeFS postgresql: image: repository: groonga/pgroonga @@ -68,34 +68,50 @@ postgresql: initdb: scriptsConfigMap: "postgresql-init-script" -minio: - mode: standalone +rustfs: + replicaCount: 1 + drivesPerNode: 1 + mode: + standalone: + enabled: true + existingClaim: + dataClaim: "rustfs-data-pvc" + distributed: + enabled: false image: - repository: bitnamilegacy/minio - tag: 2025.3.12-debian-12-r0 + rustfs: + repository: rustfs/rustfs + tag: 1.0.0-rc.6 resources: requests: memory: "256Mi" limits: memory: "256Mi" + ingress: + enabled: false gateway: enabled: false - hostname: "" # the url for the minio, e.g. "minio.example.com" - tlsSecretName: "" # e.g. "minio-tls-secret" - auth: - rootUser: texera_minio - rootPassword: password + hostname: "" # the url for the object store, e.g. "rustfs.example.com" + tlsSecretName: "" # e.g. "rustfs-tls-secret" + secret: + rustfs: + access_key: texera_rustfs + secret_key: password + config: + rustfs: + region: us-west-2 + obs_log_directory: "" + log_level: warn service: - # In production, use ClusterIP to avoid exposing the minio to the internet + # In production, use ClusterIP to avoid exposing the object store to the internet # type: ClusterIP type: NodePort - nodePorts: - api: 31000 + endpoint: + nodePort: 31000 persistence: enabled: true size: 20Gi storageClass: local-path - existingClaim: "minio-data-pvc" lakefs: secrets: @@ -113,12 +129,12 @@ lakefs: blockstore: type: s3 s3: - endpoint: http://texera-minio:9000 + endpoint: http://texera-rustfs-svc:9000 pre_signed_expiry: 15m pre_signed_endpoint: http://localhost:31000 force_path_style: true credentials: - access_key_id: texera_minio + access_key_id: texera_rustfs secret_access_key: password # Part2: configurations of Texera-related micro services diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 313930d10a..d7faf7b946 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -22,7 +22,7 @@ texera: imageTag: 1.2.0-incubating global: - # Required by Bitnami sub-charts (postgresql, minio) to allow custom images + # Required by the Bitnami postgresql sub-chart to allow custom images security: allowInsecureImages: true @@ -34,10 +34,10 @@ global: # - false: PVCs will remain after uninstall to preserve the data persistence: removeAfterUninstall: true - minioHostLocalPath: "" + rustfsHostLocalPath: "" postgresqlHostLocalPath: "" -# Part 1: the configuration of Postgres, Minio and LakeFS +# Part 1: the configuration of Postgres, RustFS and LakeFS postgresql: image: repository: groonga/pgroonga @@ -65,29 +65,56 @@ postgresql: initdb: scriptsConfigMap: "postgresql-init-script" -minio: - mode: standalone +rustfs: + # Single-node object store, matching the previous standalone MinIO. The + # upstream chart defaults to a 4-replica distributed cluster; Texera's + # in-cluster store is a convenience for single-node installs, so keep it to + # one pod and one drive. + replicaCount: 1 + drivesPerNode: 1 + mode: + standalone: + enabled: true + existingClaim: + dataClaim: "rustfs-data-pvc" + distributed: + enabled: false image: - repository: bitnamilegacy/minio - tag: 2025.3.12-debian-12-r0 + rustfs: + repository: rustfs/rustfs + tag: 1.0.0-rc.6 + # Texera fronts the object store with Envoy Gateway (the `gateway` block + # below), not with an Ingress -- the upstream chart turns one on by default. + ingress: + enabled: false gateway: enabled: false - hostname: "" # the url for the minio, e.g. "minio.example.com" - tlsSecretName: "" # e.g. "minio-tls-secret" - auth: - rootUser: texera_minio - rootPassword: password + hostname: "" # the url for the object store, e.g. "rustfs.example.com" + tlsSecretName: "" # e.g. "rustfs-tls-secret" + secret: + rustfs: + access_key: texera_rustfs + secret_key: password + config: + rustfs: + # Must match the lakefs/lakekeeperInit region: it is part of the SigV4 scope. + region: us-west-2 + # Empty => log to stdout (`kubectl logs`) instead of a log PVC. + obs_log_directory: "" + log_level: warn service: - # In production, use ClusterIP to avoid exposing the minio to the internet + # In production, use ClusterIP to avoid exposing the object store to the internet # type: ClusterIP type: NodePort - nodePorts: - api: 31000 + endpoint: + nodePort: 31000 + # The chart's own PVC template is unused here: Texera provisions the claim + # itself (templates/rustfs-persistence.yaml) so that the local-path + # host directory stays under persistence.rustfsHostLocalPath. persistence: enabled: true size: 20Gi storageClass: local-path - existingClaim: "minio-data-pvc" lakefs: secrets: @@ -105,12 +132,12 @@ lakefs: blockstore: type: s3 s3: - endpoint: http://texera-minio:9000 + endpoint: http://texera-rustfs-svc:9000 pre_signed_expiry: 15m pre_signed_endpoint: http://localhost:31000 force_path_style: true credentials: - access_key_id: texera_minio + access_key_id: texera_rustfs secret_access_key: password lakekeeper: @@ -137,6 +164,10 @@ lakekeeper: lakekeeperInit: enabled: true + # S3 client image used by the create-bucket init container. `rc` is RustFS's + # client; any image whose entrypoint is an `mc`-style client with + # `alias set` / `bucket create` would do. + bucketClientImage: rustfs/rc:v0.1.35 defaultProject: id: "00000000-0000-0000-0000-000000000000" name: default diff --git a/bin/local-dev/README.md b/bin/local-dev/README.md index a560f9098b..56df4154fa 100644 --- a/bin/local-dev/README.md +++ b/bin/local-dev/README.md @@ -42,7 +42,7 @@ bin/local-dev/ ├── main.sh shell engine — sbt builds, service lifecycle, port checks ├── tui.py Textual dashboard surfaced by `bin/local-dev.sh -i` ├── docker-compose.override.yml overlay on top of bin/single-node/docker-compose.yml -│ (host-LAN-IP MinIO endpoint, Lakekeeper warehouse, etc.) +│ (host-LAN-IP RustFS endpoint, Lakekeeper warehouse, etc.) └── tests/ ├── test_local_dev_sh.sh bash smoke: license header, syntax, version, --help, │ error-on-bad-input, regression guards diff --git a/bin/single-node/.env b/bin/single-node/.env index 555e14db7d..4a8ba397bf 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -18,7 +18,7 @@ # Public host and ports exposed by the deployment TEXERA_HOST=http://localhost TEXERA_PORT=8080 -MINIO_PORT=9000 +RUSTFS_PORT=9000 # Log level for all Texera services (valid values: ERROR, WARN, INFO, DEBUG) TEXERA_SERVICE_LOG_LEVEL=INFO @@ -36,8 +36,8 @@ USER_SYS_ADMIN_PASSWORD=texera POSTGRES_USER=texera POSTGRES_PASSWORD=password -# S3 (MinIO) credentials -STORAGE_S3_AUTH_USERNAME=texera_minio +# S3 (RustFS) credentials +STORAGE_S3_AUTH_USERNAME=texera_rustfs STORAGE_S3_AUTH_PASSWORD=password # LakeFS server configuration @@ -46,7 +46,7 @@ LAKEFS_INSTALLATION_ACCESS_KEY_ID=AKIAIOSFOLKFSSAMPLES LAKEFS_INSTALLATION_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY LAKEFS_BLOCKSTORE_TYPE=s3 LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true -LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://texera-minio:9000 +LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://texera-rustfs:9000 LAKEFS_AUTH_ENCRYPT_SECRET_KEY=random_string_for_lakefs LAKEFS_LOGGING_LEVEL=INFO LAKEFS_STATS_ENABLED=1 @@ -60,7 +60,7 @@ LAKEKEEPER__PG_ENCRYPTION_KEY=texera_key LAKEKEEPER_BASE_URI=http://texera-lakekeeper:8181 # Texera storage endpoints -STORAGE_S3_ENDPOINT=http://texera-minio:9000 +STORAGE_S3_ENDPOINT=http://texera-rustfs:9000 STORAGE_S3_REGION=us-west-2 STORAGE_LAKEFS_ENDPOINT=http://texera-lakefs:8000/api/v1 STORAGE_JDBC_URL=jdbc:postgresql://texera-postgres:5432/texera_db?currentSchema=texera_db,public diff --git a/bin/single-node/README.md b/bin/single-node/README.md index 62d4834297..77e0f63407 100644 --- a/bin/single-node/README.md +++ b/bin/single-node/README.md @@ -153,11 +153,11 @@ All changes below are to the `.env` file in the installation folder, unless othe ### Run Texera on other ports By default, Texera uses: - Port 8080 for its web service -- Port 9000 for its MinIO storage service +- Port 9000 for its RustFS storage service To change these ports, open the `.env` file and update the corresponding variables: - For the web service port (8080): change `TEXERA_PORT=8080` to your desired port, e.g., `TEXERA_PORT=8081`. -- For the MinIO port (9000): change `MINIO_PORT=9000` to your desired port, e.g., `MINIO_PORT=9001`. +- For the RustFS port (9000): change `RUSTFS_PORT=9000` to your desired port, e.g., `RUSTFS_PORT=9001`. ### Change the locations of Texera data By default, Docker manages Texera's data locations. To change them to your own locations: diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 64247d9d66..52f3a89a0e 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -18,33 +18,43 @@ name: texera-single-node services: # Part1: Specification of the storage services used by Texera - # MinIO is an S3-compatible object storage used to store datasets and files. - minio: - image: minio/minio:RELEASE.2025-02-28T09-55-16Z - container_name: texera-minio + # RustFS is an S3-compatible object storage used to store datasets and files. + rustfs: + image: rustfs/rustfs:1.0.0-rc.6 + container_name: texera-rustfs ports: - - "${MINIO_PORT:-9000}:9000" + - "${RUSTFS_PORT:-9000}:9000" env_file: - .env environment: - - MINIO_ROOT_USER=${STORAGE_S3_AUTH_USERNAME} - - MINIO_ROOT_PASSWORD=${STORAGE_S3_AUTH_PASSWORD} + - RUSTFS_ACCESS_KEY=${STORAGE_S3_AUTH_USERNAME} + - RUSTFS_SECRET_KEY=${STORAGE_S3_AUTH_PASSWORD} + # Must match STORAGE_S3_REGION: the region is part of the SigV4 scope, and + # LakeFS sends its blockstore region on every request. + - RUSTFS_REGION=${STORAGE_S3_REGION} + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + # Empty log directory => log to stdout, so `docker compose logs rustfs` + # works and no second volume is needed. `warn` keeps that readable; the + # server is very chatty at info. + - RUSTFS_OBS_LOG_DIRECTORY= + - RUSTFS_OBS_LOGGER_LEVEL=warn volumes: - - minio_data:/data - command: server --console-address ":9001" /data + - rustfs_data:/data healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/live";] + test: ["CMD", "curl", "-sf", "http://localhost:9000/health";] interval: 5s timeout: 3s retries: 10 # This job creates the S3 bucket used by the Iceberg warehouse that the - # Lakekeeper service manages. - minio-init: - image: minio/mc:RELEASE.2025-05-21T01-59-54Z - container_name: texera-minio-init + # Lakekeeper service manages. `rc` is RustFS's S3 client, the counterpart of + # the `mc` client this used to run. + rustfs-init: + image: rustfs/rc:v0.1.35 + container_name: texera-rustfs-init depends_on: - minio: + rustfs: condition: service_healthy env_file: - .env @@ -53,9 +63,10 @@ services: command: - | set -e - mc alias set local "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" - mc mb --ignore-existing "local/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET" - echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' is ready." + rc alias set local "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" \ + --region "$$STORAGE_S3_REGION" --bucket-lookup path + rc bucket create --ignore-existing "local/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET" + echo "RustFS bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' is ready." # PostgreSQL with PGroonga extension for full-text search. # Used by lakeFS and Texera's metadata storage. @@ -83,13 +94,13 @@ services: depends_on: postgres: condition: service_healthy - minio: + rustfs: condition: service_started env_file: - .env environment: - # This port also need to be changed if the port of MinIO service is changed - - LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=${TEXERA_HOST}:${MINIO_PORT:-9000} + # This port also need to be changed if the port of RustFS service is changed + - LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=${TEXERA_HOST}:${RUSTFS_PORT:-9000} - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=${STORAGE_S3_AUTH_USERNAME} - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=${STORAGE_S3_AUTH_PASSWORD} entrypoint: ["/bin/sh", "-c"] @@ -126,7 +137,7 @@ services: depends_on: postgres: condition: service_healthy - minio: + rustfs: condition: service_started lakekeeper-migrate: condition: service_completed_successfully @@ -142,14 +153,14 @@ services: start_period: 10s # One-shot init container that creates the Lakekeeper default project and - # the Iceberg warehouse pointing at the MinIO bucket prepared by minio-init. + # the Iceberg warehouse pointing at the RustFS bucket prepared by rustfs-init. lakekeeper-init: image: alpine:3.19 container_name: texera-lakekeeper-init depends_on: lakekeeper: condition: service_healthy - minio-init: + rustfs-init: condition: service_completed_successfully env_file: - .env @@ -222,7 +233,7 @@ services: # The warehouse persists its S3 endpoint in Lakekeeper's own DB. The # local-dev launcher sets STORAGE_S3_ENDPOINT to the host LAN IP so # both this container and the host JVMs (Iceberg remote-signing) reach - # the same MinIO — but that IP changes across networks / DHCP leases. + # the same RustFS — but that IP changes across networks / DHCP leases. # This init is idempotent and would otherwise just skip, leaving a # stale endpoint that breaks workflow execution with an opaque Iceberg # RESTException. Refresh the stored endpoint on every run (#6195). @@ -314,7 +325,7 @@ services: container_name: file-service restart: unless-stopped depends_on: - minio: + rustfs: condition: service_started lakefs: condition: service_healthy @@ -536,6 +547,6 @@ networks: # persistent volumes volumes: - minio_data: + rustfs_data: postgres_data: workflow_result_data: \ No newline at end of file diff --git a/build.sbt b/build.sbt index 67b838a2a0..11f2e9447e 100644 --- a/build.sbt +++ b/build.sbt @@ -167,6 +167,7 @@ lazy val FileService = (project in file("file-service")) .dependsOn(WorkflowCore, Auth, Config) .configs(Test) .dependsOn(DAO % "test->test") // test scope dependency + .dependsOn(WorkflowCore % "test->test") // reuse RustFSContainer in MockLakeFS .settings( dependencyOverrides ++= Seq( // override it as io.dropwizard 4 require 2.16.1 or higher diff --git a/common/config/src/main/resources/storage.conf b/common/config/src/main/resources/storage.conf index 12a9919e04..244d2a171e 100644 --- a/common/config/src/main/resources/storage.conf +++ b/common/config/src/main/resources/storage.conf @@ -125,7 +125,7 @@ storage { } auth { - username = "texera_minio" + username = "texera_rustfs" username = ${?STORAGE_S3_AUTH_USERNAME} password = "password" diff --git a/common/workflow-core/build.sbt b/common/workflow-core/build.sbt index bdaa923827..391113f089 100644 --- a/common/workflow-core/build.sbt +++ b/common/workflow-core/build.sbt @@ -87,8 +87,9 @@ libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "3.2.15" % Test, // ScalaTest "junit" % "junit" % "4.13.2" % Test, // JUnit "com.novocode" % "junit-interface" % "0.11" % Test, // SBT interface for JUnit - "com.dimafeng" %% "testcontainers-scala-scalatest" % testcontainersVersion % Test, // Testcontainers ScalaTest integration - "com.dimafeng" %% "testcontainers-scala-minio" % testcontainersVersion % Test // MinIO Testcontainer Scala integration + // RustFS has no dedicated testcontainers-scala module; RustFSContainer builds it on the + // GenericContainer that testcontainers-scala-core (pulled in by -scalatest) provides. + "com.dimafeng" %% "testcontainers-scala-scalatest" % testcontainersVersion % Test // Testcontainers ScalaTest integration ) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala b/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala index 956de75520..11a9056055 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala @@ -30,7 +30,7 @@ import java.io.InputStream import scala.jdk.CollectionConverters._ /** - * S3Storage provides an abstraction for S3-compatible storage (e.g., MinIO). + * S3Storage provides an abstraction for S3-compatible storage (e.g., RustFS). * - Uses credentials and endpoint from StorageConfig. * - Supports object upload, download, listing, and deletion. */ @@ -44,14 +44,14 @@ object S3StorageClient { // Cap how many failed keys are listed in the error message. private[util] val MAX_LISTED_DELETE_ERRORS = 10 - // Initialize MinIO-compatible S3 Client + // Initialize the S3 client for the configured S3-compatible endpoint private lazy val s3Client: S3Client = { val credentials = AwsBasicCredentials.create(StorageConfig.s3Username, StorageConfig.s3Password) S3Client .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials)) .region(Region.of(StorageConfig.s3Region)) - .endpointOverride(java.net.URI.create(StorageConfig.s3Endpoint)) // MinIO URL + .endpointOverride(java.net.URI.create(StorageConfig.s3Endpoint)) // object-store URL .serviceConfiguration( S3Configuration.builder().pathStyleAccessEnabled(true).build() ) @@ -106,7 +106,7 @@ object S3StorageClient { * A trailing `/` is added when missing so the prefix matches on a path boundary (`a/b` deletes * `a/b/file` but not `a/bc/file`). An empty prefix would match the whole bucket and is rejected. * - * @param bucketName Target S3/MinIO bucket. + * @param bucketName Target S3 bucket. * @param directoryPrefix Non-empty key prefix to delete. */ def deleteDirectory(bucketName: String, directoryPrefix: String): Unit = { diff --git a/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala b/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala new file mode 100644 index 0000000000..fd07258f73 --- /dev/null +++ b/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.util + +import com.dimafeng.testcontainers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait + +import java.time.Duration + +/** + * Factory for the RustFS container used by the S3-backed test suites. + * + * testcontainers-scala ships a `MinIOContainer` module but no RustFS one, so this builds the + * equivalent on top of `GenericContainer`. The differences from `MinIOContainer` that matter: + * + * - credentials come from `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` rather than + * `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD`; + * - readiness is `GET /health` on the S3 port, not `GET /minio/health/live`; + * - `RUSTFS_REGION` must match the region the AWS SDK signs with, because the region is part + * of the SigV4 credential scope. + * + * The console (9001) is left disabled: no test drives it, and starting it only widens the + * surface the readiness check has to wait for. + */ +object RustFSContainer { + + /** Image pinned in lockstep with `bin/single-node/docker-compose.yml` and `bin/k8s/values.yaml`. */ + val ImageName: String = "rustfs/rustfs:1.0.0-rc.6" + + /** S3 API port inside the container. */ + val Port: Int = 9000 + + val DefaultUser: String = "texera_rustfs" + val DefaultPassword: String = "password" + val DefaultRegion: String = "us-west-2" + + def apply( + userName: String = DefaultUser, + password: String = DefaultPassword, + region: String = DefaultRegion + ): GenericContainer = { + val container = GenericContainer( + dockerImage = ImageName, + exposedPorts = Seq(Port), + env = Map( + "RUSTFS_ACCESS_KEY" -> userName, + "RUSTFS_SECRET_KEY" -> password, + "RUSTFS_REGION" -> region, + "RUSTFS_VOLUMES" -> "/data", + // Log to stdout so a failed start shows up in the testcontainers log consumer + // instead of a file inside the container. `warn` keeps that readable. + "RUSTFS_OBS_LOG_DIRECTORY" -> "", + "RUSTFS_OBS_LOGGER_LEVEL" -> "warn" + ), + waitStrategy = Wait + .forHttp("/health") + .forPort(Port) + .forStatusCode(200) + .withStartupTimeout(Duration.ofMinutes(2)) + ) + container + } +} diff --git a/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala b/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala index 1d3b77dbb1..4a25535493 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala @@ -19,14 +19,13 @@ package org.apache.texera.service.util -import com.dimafeng.testcontainers.MinIOContainer +import com.dimafeng.testcontainers.GenericContainer import org.apache.texera.amber.config.StorageConfig import org.scalatest.{BeforeAndAfterAll, Suite} -import org.testcontainers.utility.DockerImageName /** - * Base trait for tests requiring S3 storage (MinIO). - * Provides access to a single shared MinIO container across all test suites. + * Base trait for tests requiring S3 storage (RustFS). + * Provides access to a single shared RustFS container across all test suites. * * Usage: Mix this trait into any test suite that needs S3 storage. */ @@ -40,21 +39,17 @@ trait S3StorageTestBase extends BeforeAndAfterAll { this: Suite => } object S3StorageTestBase { - private lazy val container: MinIOContainer = { - val c = MinIOContainer( - dockerImageName = DockerImageName.parse("minio/minio:RELEASE.2025-02-28T09-55-16Z"), - userName = "texera_minio", - password = "password" - ) + private lazy val container: GenericContainer = { + val c = RustFSContainer(region = StorageConfig.s3Region) c.start() - val endpoint = s"http://${c.host}:${c.mappedPort(9000)}" + val endpoint = s"http://${c.host}:${c.mappedPort(RustFSContainer.Port)}" StorageConfig.s3Endpoint = endpoint - println(s"[S3Storage] Started shared MinIO at $endpoint") + println(s"[S3Storage] Started shared RustFS at $endpoint") sys.addShutdownHook { - println("[S3Storage] Stopping shared MinIO...") + println("[S3Storage] Stopping shared RustFS...") c.stop() } diff --git a/docs/contribution-guidelines/guide-for-developers.md b/docs/contribution-guidelines/guide-for-developers.md index 6871b2c636..c6e7a4f5b7 100644 --- a/docs/contribution-guidelines/guide-for-developers.md +++ b/docs/contribution-guidelines/guide-for-developers.md @@ -147,9 +147,9 @@ Execute `sql/iceberg_postgres_catalog.sql` to create the database for storing I psql -U postgres -f "sql/iceberg_postgres_catalog.sql" ``` -### Setup the LakeFS+Minio locally +### Setup the LakeFS+RustFS locally -Texera requires [LakeFS](https://lakefs.io/) and S3([Minio](https://min.io/docs/minio/kubernetes/upstream/index.html) is one of the implementations) as the dataset storage. Setting up these two storage services locally are required to make Texera's dataset feature functioning. +Texera requires [LakeFS](https://lakefs.io/) and S3([RustFS](https://docs.rustfs.com/) is one of the implementations) as the dataset storage. Setting up these two storage services locally are required to make Texera's dataset feature functioning. Install [Docker Desktop](https://docs.docker.com/desktop/setup/install/mac-install/) which contains both docker engine and docker compose. Make sure you launch the Docker after installing it. @@ -160,7 +160,7 @@ cd file-service/src/main/resources Edit `docker-compose.yml` by: search for `volumes` in the file and follow the instructions in the comment. This step is required otherwise your data will be lost if containers are deleted -Execute the following command to start LakeFS and Minio: +Execute the following command to start LakeFS and RustFS: ``` docker compose up ``` diff --git a/docs/getting-started/installing-using-docker.md b/docs/getting-started/installing-using-docker.md index 52968886d9..ce8b28df23 100644 --- a/docs/getting-started/installing-using-docker.md +++ b/docs/getting-started/installing-using-docker.md @@ -146,11 +146,11 @@ All changes below are to the `.env` file in the installation folder, unless othe ### Run Texera on other ports By default, Texera uses: - Port 8080 for its web service -- Port 9000 for its MinIO storage service +- Port 9000 for its RustFS storage service To change these ports, open the `.env` file and update the corresponding variables: - For the web service port (8080): change `TEXERA_PORT=8080` to your desired port, e.g., `TEXERA_PORT=8081`. -- For the MinIO port (9000): change `MINIO_PORT=9000` to your desired port, e.g., `MINIO_PORT=9001`. +- For the RustFS port (9000): change `RUSTFS_PORT=9000` to your desired port, e.g., `RUSTFS_PORT=9001`. ### Change the locations of Texera data By default, Docker manages Texera's data locations. To change them to your own locations: diff --git a/docs/getting-started/run-on-kubernetes.md b/docs/getting-started/run-on-kubernetes.md index 6503b8dc91..d51c3e3ce0 100644 --- a/docs/getting-started/run-on-kubernetes.md +++ b/docs/getting-started/run-on-kubernetes.md @@ -110,12 +110,12 @@ Once the deployments are running, you can access the Texera web interface. ### File Upload Error -If you see an error when trying to upload a file to a dataset, you may need to forward the port for MinIO (our file storage service). +If you see an error when trying to upload a file to a dataset, you may need to forward the port for RustFS (our file storage service). Run the following command in a new terminal: ```bash -kubectl port-forward -n texera-dev service/texera-minio 31000:9000 +kubectl port-forward -n texera-dev service/texera-rustfs-svc 31000:9000 ``` This maps the service's port `9000` to your local port `31000`. diff --git a/docs/overview.md b/docs/overview.md index 2b232db723..08e1adde76 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -37,7 +37,7 @@ At its core, Texera acts as a bridge between a highly accessible frontend and a 1. **Web-Based Interface (Frontend):** A rich GUI running directly in your browser. It allows users to construct data processing pipelines by dragging and dropping blocks on a canvas. No installation is required on client machines. 2. **Distributed Engine (Backend):** When a workflow is submitted, the Texera engine compiles the graphical representation into an optimized, distributed execution plan. It then spins up computing units to process massive datasets in parallel. -3. **Storage Integration:** Texera integrates smoothly with modern data lake and storage technologies (like LakeFS and MinIO) to persistently log runs and save datasets securely. +3. **Storage Integration:** Texera integrates smoothly with modern data lake and storage technologies (like LakeFS and RustFS) to persistently log runs and save datasets securely. --- diff --git a/docs/tutorials/guide-to-launch-lakekeeper.md b/docs/tutorials/guide-to-launch-lakekeeper.md index a2eb5d9a81..fc4eedfb14 100644 --- a/docs/tutorials/guide-to-launch-lakekeeper.md +++ b/docs/tutorials/guide-to-launch-lakekeeper.md @@ -71,7 +71,7 @@ For more information of why using RESTCatalog, see [Issue #4126](https://github. +LAKEKEEPER__PG_DATABASE_URL_WRITE="postgres://<user>:<urlencoded_password>@<host>:5432/texera_lakekeeper" ``` - If you have customized storage-related values in `common/config/src/main/resources/storage.conf` (for example, the bucket name, S3 endpoint, or MinIO credentials), check the below environment variables in the script and modify their values accordingly: + If you have customized storage-related values in `common/config/src/main/resources/storage.conf` (for example, the bucket name, S3 endpoint, or RustFS credentials), check the below environment variables in the script and modify their values accordingly: ```shell # Storage settings — must stay in sync with storage.conf @@ -81,7 +81,7 @@ STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME="${STORAGE_ICEBERG_CATALOG_REST_WARE STORAGE_ICEBERG_CATALOG_REST_REGION="${STORAGE_ICEBERG_CATALOG_REST_REGION:-us-west-2}" STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET="${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET:-texera-iceberg}" STORAGE_S3_ENDPOINT="${STORAGE_S3_ENDPOINT:-http://localhost:9000}"; -STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_minio}" +STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_rustfs}" STORAGE_S3_AUTH_PASSWORD="${STORAGE_S3_AUTH_PASSWORD:-password}" ``` @@ -96,7 +96,7 @@ STORAGE_S3_AUTH_PASSWORD="${STORAGE_S3_AUTH_PASSWORD:-password}" 1. Start Lakekeeper if it's not already running (on http://localhost:8181) 2. Bootstrap the Lakekeeper server (creates the default project) - 3. Create the texera-iceberg bucket in MinIO if it doesn't exist + 3. Create the texera-iceberg bucket in RustFS if it doesn't exist 4. Register the texera warehouse with Lakekeeper, pointing at that bucket diff --git a/file-service/build.sbt b/file-service/build.sbt index c93f5e3230..f816443d21 100644 --- a/file-service/build.sbt +++ b/file-service/build.sbt @@ -74,9 +74,10 @@ libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % Test, // Mockito for mocking "org.assertj" % "assertj-core" % assertjVersion % Test, // AssertJ for assertions "com.novocode" % "junit-interface" % "0.11" % Test, // SBT interface for JUnit + // RustFS has no dedicated testcontainers-scala module; MockLakeFS uses the shared + // RustFSContainer from WorkflowCore's test sources (see the test->test dep in build.sbt). "com.dimafeng" %% "testcontainers-scala-scalatest" % testcontainersVersion % Test, // Testcontainers ScalaTest integration "com.dimafeng" %% "testcontainers-scala-postgresql" % testcontainersVersion % Test, // PostgreSQL Testcontainer Scala integration - "com.dimafeng" %% "testcontainers-scala-minio" % testcontainersVersion % Test, // MinIO Testcontainer Scala integration ) ///////////////////////////////////////////////////////////////////////////// diff --git a/file-service/src/main/resources/docker-compose.yml b/file-service/src/main/resources/docker-compose.yml index f4d2a07256..95088150b1 100644 --- a/file-service/src/main/resources/docker-compose.yml +++ b/file-service/src/main/resources/docker-compose.yml @@ -17,20 +17,23 @@ name: texera-lakefs services: - minio: - image: minio/minio:RELEASE.2025-02-28T09-55-16Z - container_name: texera-lakefs-minio + rustfs: + image: rustfs/rustfs:1.0.0-rc.6 + container_name: texera-lakefs-rustfs restart: unless-stopped ports: - "9000:9000" - "9001:9001" environment: - - MINIO_ROOT_USER=texera_minio - - MINIO_ROOT_PASSWORD=password - command: server --console-address ":9001" /data + - RUSTFS_ACCESS_KEY=texera_rustfs + - RUSTFS_SECRET_KEY=password + # Must match the region the AWS SDK signs with: it is part of the SigV4 scope. + - RUSTFS_REGION=us-west-2 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 # The lines below are recommended to mount a host path in order to persist your data even if the container is removed. volumes: - - minio_data:/data + - rustfs_data:/data postgres: image: postgres:15 @@ -56,16 +59,16 @@ services: depends_on: postgres: condition: service_healthy - minio: + rustfs: condition: service_started ports: - "8000:8000" environment: - LAKEFS_BLOCKSTORE_TYPE=s3 - LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true - - LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://minio:9000 + - LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://rustfs:9000 - LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=http://localhost:9000 - - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=texera_minio + - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=texera_rustfs - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=password - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=random_string_for_lakefs - LAKEFS_LOGGING_LEVEL=INFO @@ -94,5 +97,5 @@ networks: # Named Docker volumes — uncomment the following lines if you mounted host paths above. volumes: - minio_data: + rustfs_data: postgres_data: diff --git a/file-service/src/main/resources/minio-config.yml b/file-service/src/main/resources/rustfs-config.yml similarity index 69% rename from file-service/src/main/resources/minio-config.yml rename to file-service/src/main/resources/rustfs-config.yml index 0622b92bdc..f29fed9e27 100644 --- a/file-service/src/main/resources/minio-config.yml +++ b/file-service/src/main/resources/rustfs-config.yml @@ -18,15 +18,20 @@ version: '3.8' services: - minio: - image: minio/minio:latest - container_name: minio + rustfs: + image: rustfs/rustfs:1.0.0-rc.6 + container_name: rustfs ports: - - "9500:9000" # MinIO API - - "9501:9001" # MinIO Console UI + - "9500:9000" # S3 API + - "9501:9001" # Console UI environment: - - MINIO_ROOT_USER=texera_minio - - MINIO_ROOT_PASSWORD=password + - RUSTFS_ACCESS_KEY=texera_rustfs + - RUSTFS_SECRET_KEY=password + - RUSTFS_REGION=us-west-2 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 volumes: - - /Users/baijiadong/Desktop/chenlab/texera/core/file-service/src/main/user-resources/minio:/data - command: server --console-address ":9001" /data \ No newline at end of file + - rustfs_data:/data + +volumes: + rustfs_data: \ No newline at end of file diff --git a/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala b/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala index 62e60bd267..35ddbb4405 100644 --- a/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala +++ b/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala @@ -22,7 +22,7 @@ package org.apache.texera.service import com.dimafeng.testcontainers._ import io.lakefs.clients.sdk.{ApiClient, RepositoriesApi} import org.apache.texera.amber.config.StorageConfig -import org.apache.texera.service.util.S3StorageClient +import org.apache.texera.service.util.{RustFSContainer, S3StorageClient} import org.scalatest.{BeforeAndAfterAll, Suite} import org.testcontainers.containers.Network import org.testcontainers.utility.DockerImageName @@ -34,7 +34,7 @@ import software.amazon.awssdk.services.s3.S3Configuration import java.net.URI /** - * Trait to spin up a LakeFS + MinIO + Postgres stack using Testcontainers, + * Trait to spin up a LakeFS + RustFS + Postgres stack using Testcontainers, * similar to how MockTexeraDB uses EmbeddedPostgres. */ trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self: Suite => @@ -53,29 +53,27 @@ trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self: Suit postgres.container.withNetwork(network) // LakeFS bakes the pre-signed endpoint into its env before containers start, - // so MinIO cannot use a dynamically mapped host port: presigned URLs must be - // reachable from the host at an address known ahead of time. Reserve a free - // host port and pin MinIO's 9000 to it. - val minioHostPort: Int = { + // so the object store cannot use a dynamically mapped host port: presigned URLs + // must be reachable from the host at an address known ahead of time. Reserve a + // free host port and pin the S3 port to it. + val objectStoreHostPort: Int = { val socket = new java.net.ServerSocket(0) try socket.getLocalPort finally socket.close() } - // MinIO for object storage - val minio = MinIOContainer( - dockerImageName = DockerImageName.parse("minio/minio:RELEASE.2025-02-28T09-55-16Z"), - userName = "texera_minio", - password = "password" - ) - minio.container.withNetwork(network) - minio.container.withCreateContainerCmdModifier { cmd => + // RustFS for object storage + val objectStore: GenericContainer = RustFSContainer() + objectStore.container.withNetwork(network) + objectStore.container.withCreateContainerCmdModifier { cmd => import com.github.dockerjava.api.model.{ExposedPort, PortBinding, Ports} - // setting explicit bindings replaces them all, so 9001 (console) must keep - // a dynamic binding or the container readiness check never passes + // RustFSContainer exposes only the S3 port, so pinning it is the whole + // binding set; the console is not started. cmd.getHostConfig.withPortBindings( - new PortBinding(Ports.Binding.bindPort(minioHostPort), ExposedPort.tcp(9000)), - new PortBinding(Ports.Binding.empty(), ExposedPort.tcp(9001)) + new PortBinding( + Ports.Binding.bindPort(objectStoreHostPort), + ExposedPort.tcp(RustFSContainer.Port) + ) ) } @@ -98,10 +96,11 @@ trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self: Suit env = Map( "LAKEFS_BLOCKSTORE_TYPE" -> "s3", "LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE" -> "true", - "LAKEFS_BLOCKSTORE_S3_ENDPOINT" -> s"http://${minio.container.getNetworkAliases.get(0)}:9000", - "LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT" -> s"http://localhost:$minioHostPort";, - "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID" -> "texera_minio", - "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY" -> "password", + "LAKEFS_BLOCKSTORE_S3_ENDPOINT" -> s"http://${objectStore.container.getNetworkAliases + .get(0)}:${RustFSContainer.Port}", + "LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT" -> s"http://localhost:$objectStoreHostPort";, + "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID" -> RustFSContainer.DefaultUser, + "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY" -> RustFSContainer.DefaultPassword, "LAKEFS_AUTH_ENCRYPT_SECRET_KEY" -> "random_string_for_lakefs", "LAKEFS_LOGGING_LEVEL" -> "INFO", "LAKEFS_STATS_ENABLED" -> "1", @@ -114,10 +113,11 @@ trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self: Suit ) lakefs.container.withNetwork(network) - override val container = MultipleContainers(postgres, minio, lakefs) + override val container = MultipleContainers(postgres, objectStore, lakefs) def lakefsBaseUrl: String = s"http://${lakefs.host}:${lakefs.mappedPort(8000)}" - def minioEndpoint: String = s"http://${minio.host}:${minio.mappedPort(9000)}" + def objectStoreEndpoint: String = + s"http://${objectStore.host}:${objectStore.mappedPort(RustFSContainer.Port)}" def lakefsApiBasePath: String = s"$lakefsBaseUrl/api/v1" // ---- Clients (lazy so they initialize after containers are started) ---- @@ -134,20 +134,20 @@ trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self: Suit lazy val repositoriesApi: RepositoriesApi = new RepositoriesApi(lakefsApiClient) /** - * S3 client instance for testing pointed at MinIO. + * S3 client instance for testing pointed at RustFS. * * Notes: - * - Region can be any value for MinIO, but MUST match what your signing expects. - * so we use that. + * - The region MUST match RUSTFS_REGION: it is part of the SigV4 credential scope. * - Path-style is important: http://host:port/bucket/key */ lazy val s3Client: S3Client = { //Temporal credentials for testing purposes only - val creds = AwsBasicCredentials.create("texera_minio", "password") + val creds = + AwsBasicCredentials.create(RustFSContainer.DefaultUser, RustFSContainer.DefaultPassword) S3Client .builder() .endpointOverride(URI.create(StorageConfig.s3Endpoint)) // set in afterStart() - .region(Region.US_WEST_2) // Required for `.build()`; not important in this test config. + .region(Region.US_WEST_2) // must match RustFSContainer's RUSTFS_REGION .credentialsProvider(StaticCredentialsProvider.create(creds)) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) .build() @@ -172,7 +172,7 @@ trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self: Suit } // replace storage endpoints in StorageConfig - StorageConfig.s3Endpoint = minioEndpoint + StorageConfig.s3Endpoint = objectStoreEndpoint StorageConfig.lakefsEndpoint = lakefsApiBasePath // create S3 bucket used by lakeFS in tests
