This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 4176c29 chore(ci): parallel run integration tests (#340)
4176c29 is described below
commit 4176c2966ad3042aa4ca49dc5f231cf58a613d8f
Author: Shiyan Xu <[email protected]>
AuthorDate: Mon Jun 2 02:39:34 2025 -0500
chore(ci): parallel run integration tests (#340)
Split and parallel run integration tests (demo).
---
.github/workflows/ci.yml | 8 ++-
demo/README.md | 24 ++++++-
.../{sql-datafusion => apps/datafusion}/Cargo.toml | 2 +-
.../datafusion}/src/main.rs | 0
.../hudi-file-group-api}/cpp/CMakeLists.txt | 4 +-
.../hudi-file-group-api}/cpp/main.cpp | 0
.../hudi-table-api/python}/src/__init__.py | 0
.../hudi-table-api/python}/src/main.py | 0
.../hudi-table-api/rust}/Cargo.toml | 2 +-
.../hudi-table-api/rust}/src/main.rs | 0
demo/ci_run.sh | 77 ++++++++++++++++++++++
demo/compose.yaml | 3 +
demo/run_demo.sh | 56 ----------------
demo/sql-datafusion/run.sh | 21 ------
demo/table-api-python/run.sh | 21 ------
demo/table-api-rust/run.sh | 21 ------
16 files changed, 110 insertions(+), 129 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d97c0a6..e9880d8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -126,13 +126,17 @@ jobs:
if-no-files-found: 'error'
integration-tests:
+ strategy:
+ fail-fast: false
+ matrix:
+ app-path: [ 'datafusion', 'hudi-file-group-api/cpp',
'hudi-table-api/rust' , 'hudi-table-api/python' ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- - name: Integration tests
+ - name: Integration test - ${{ matrix.app-path }}
run: |
cd demo
- ./run_demo.sh
+ ./ci_run.sh ${{ matrix.app-path }}
publish-coverage:
name: Publish coverage reports to codecov.io
diff --git a/demo/README.md b/demo/README.md
index fa22e2f..8934b40 100644
--- a/demo/README.md
+++ b/demo/README.md
@@ -19,8 +19,26 @@
# Demo
-The demo runs on docker compose; the infrastructure is defined in
[`compose.yaml`](compose.yaml) and [`infra`](infra).
+This directory contains demo applications showcasing Hudi-rs APIs across
different programming languages. The demos run on Docker Compose and also serve
as integration tests in the CI pipeline (see
[`.github/workflows/ci.yml`](../.github/workflows/ci.yml)).
-There are multiple independent demo apps residing in directories like
[`table-api-python`](table-api-python).
+## Infrastructure
-The demo apps are also used for integration tests.
+The demo infrastructure is defined in [`compose.yaml`](compose.yaml).
+
+Test data copied from the [`crates/test`](../crates/test/data) are placed in
Minio, providing COW and MOR tables at:
+- `s3://hudi-demo/cow/`
+- `s3://hudi-demo/mor/`
+
+## Demo Apps
+
+### [`datafusion`](apps/datafusion)
+**Rust + SQL**: Demonstrates querying Hudi tables using Apache DataFusion with
SQL syntax.
+
+### [`hudi-table-api/rust`](apps/hudi-table-api/rust)
+**Rust**: Shows native Rust API for reading Hudi table snapshots.
+
+### [`hudi-table-api/python`](apps/hudi-table-api/python)
+**Python**: Python bindings for Hudi tables with PyArrow integration.
+
+### [`hudi-file-group-api/cpp`](apps/hudi-file-group-api/cpp)
+**C++**: File group reading using C++ bindings and Arrow C ABI.
diff --git a/demo/sql-datafusion/Cargo.toml b/demo/apps/datafusion/Cargo.toml
similarity index 93%
rename from demo/sql-datafusion/Cargo.toml
rename to demo/apps/datafusion/Cargo.toml
index c9babe1..4938de9 100644
--- a/demo/sql-datafusion/Cargo.toml
+++ b/demo/apps/datafusion/Cargo.toml
@@ -26,4 +26,4 @@ edition = "2021"
[dependencies]
tokio = "^1"
datafusion = "~45.0.0"
-hudi = { path = "../../crates/hudi", features = ["datafusion"] }
+hudi = { path = "../../../crates/hudi", features = ["datafusion"] }
diff --git a/demo/sql-datafusion/src/main.rs b/demo/apps/datafusion/src/main.rs
similarity index 100%
rename from demo/sql-datafusion/src/main.rs
rename to demo/apps/datafusion/src/main.rs
diff --git a/demo/file-group-api/cpp/CMakeLists.txt
b/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
similarity index 94%
rename from demo/file-group-api/cpp/CMakeLists.txt
rename to demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
index e1ac93f..b8ffff1 100644
--- a/demo/file-group-api/cpp/CMakeLists.txt
+++ b/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
@@ -21,10 +21,8 @@ project(hudi_cpp_example)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
-# Add the hudi-cpp subdirectory (adjust path as needed)
-add_subdirectory(../../../cpp hudi-cpp)
+add_subdirectory(../../../../cpp hudi-cpp)
-# Add the example executable
add_executable(file_group_api_cpp
main.cpp
)
diff --git a/demo/file-group-api/cpp/main.cpp
b/demo/apps/hudi-file-group-api/cpp/main.cpp
similarity index 100%
rename from demo/file-group-api/cpp/main.cpp
rename to demo/apps/hudi-file-group-api/cpp/main.cpp
diff --git a/demo/table-api-python/src/__init__.py
b/demo/apps/hudi-table-api/python/src/__init__.py
similarity index 100%
rename from demo/table-api-python/src/__init__.py
rename to demo/apps/hudi-table-api/python/src/__init__.py
diff --git a/demo/table-api-python/src/main.py
b/demo/apps/hudi-table-api/python/src/main.py
similarity index 100%
rename from demo/table-api-python/src/main.py
rename to demo/apps/hudi-table-api/python/src/main.py
diff --git a/demo/table-api-rust/Cargo.toml
b/demo/apps/hudi-table-api/rust/Cargo.toml
similarity index 95%
rename from demo/table-api-rust/Cargo.toml
rename to demo/apps/hudi-table-api/rust/Cargo.toml
index c807db6..39139a2 100644
--- a/demo/table-api-rust/Cargo.toml
+++ b/demo/apps/hudi-table-api/rust/Cargo.toml
@@ -27,4 +27,4 @@ edition = "2021"
tokio = "^1"
arrow = { version = "~54.1.0", features = ["pyarrow"] }
-hudi = { path = "../../crates/hudi" }
+hudi = { path = "../../../../crates/hudi" }
diff --git a/demo/table-api-rust/src/main.rs
b/demo/apps/hudi-table-api/rust/src/main.rs
similarity index 100%
rename from demo/table-api-rust/src/main.rs
rename to demo/apps/hudi-table-api/rust/src/main.rs
diff --git a/demo/ci_run.sh b/demo/ci_run.sh
new file mode 100755
index 0000000..4de17fe
--- /dev/null
+++ b/demo/ci_run.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+docker compose up --build -d
+
+max_attempts=30
+attempt=0
+
+until [ "$(docker inspect -f '{{.State.Status}}' runner)" = "running" ] || [
$attempt -eq $max_attempts ]; do
+ attempt=$(( $attempt + 1 ))
+ echo "Waiting for container... (attempt $attempt of $max_attempts)"
+ sleep 1
+done
+
+if [ $attempt -eq $max_attempts ]; then
+ echo "Container failed to become ready in time"
+ exit 1
+fi
+
+app_path=$1
+if [ -z "$app_path" ]; then
+ echo "Usage: $0 <path_to_app>"
+ exit 1
+fi
+
+app_path_in_container="/opt/hudi-rs/demo/apps/$app_path"
+if [ "$app_path" = "datafusion" ]; then
+ docker compose exec -T runner /bin/bash -c "
+ source /opt/venv/bin/activate && \
+ cd /opt/hudi-rs && make setup develop && \
+ cd $app_path_in_container && \
+ cargo run -- --no-build --no-tests
+ "
+elif [ "$app_path" = "hudi-table-api/rust" ]; then
+ docker compose exec -T runner /bin/bash -c "
+ source /opt/venv/bin/activate && \
+ cd /opt/hudi-rs && make setup develop && \
+ cd $app_path_in_container && \
+ cargo run -- --no-build --no-tests
+ "
+elif [ "$app_path" = "hudi-table-api/python" ]; then
+ docker compose exec -T runner /bin/bash -c "
+ source /opt/venv/bin/activate && \
+ cd /opt/hudi-rs && make setup develop && \
+ cd $app_path_in_container && \
+ python -m src.main
+ "
+elif [ "$app_path" = "hudi-file-group-api/cpp" ]; then
+ docker compose exec -T runner /bin/bash -c "
+ cd /opt/hudi-rs/cpp && cargo build --release && \
+ cd $app_path_in_container && \
+ mkdir build && cd build && \
+ cmake .. && \
+ make && \
+ ./file_group_api_cpp
+ "
+else
+ echo "Unknown app path: $app_path"
+ exit 1
+fi
diff --git a/demo/compose.yaml b/demo/compose.yaml
index ab299c9..955bf98 100644
--- a/demo/compose.yaml
+++ b/demo/compose.yaml
@@ -16,6 +16,7 @@
# under the License.
---
services:
+ # storage layer containing test data
minio:
image: quay.io/minio/minio:latest
container_name: minio
@@ -32,6 +33,7 @@ services:
timeout: 5s
retries: 3
+ # minio client to prepare test data
mc:
build:
context: ./infra/mc
@@ -50,6 +52,7 @@ services:
- -c
- /opt/prepare_data.sh
+ # execution environment for running tests
runner:
build:
context: ./infra/runner
diff --git a/demo/run_demo.sh b/demo/run_demo.sh
deleted file mode 100755
index 32d9330..0000000
--- a/demo/run_demo.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-docker compose up --build -d
-
-max_attempts=30
-attempt=0
-
-until [ "$(docker inspect -f '{{.State.Status}}' runner)" = "running" ] || [
$attempt -eq $max_attempts ]; do
- attempt=$(( $attempt + 1 ))
- echo "Waiting for container... (attempt $attempt of $max_attempts)"
- sleep 1
-done
-
-if [ $attempt -eq $max_attempts ]; then
- echo "Container failed to become ready in time"
- exit 1
-fi
-
-# Run the C++ demo app
-docker compose exec -T runner /bin/bash -c "
- cd /opt/hudi-rs/cpp && \
- cargo build --release && \
- cd /opt/hudi-rs/demo/file-group-api/cpp && \
- mkdir build && cd build && \
- cmake .. && \
- make && \
- ./file_group_api_cpp
- "
-
-# Run the Rust and Python demo apps
-# Note: no need to activate venv since this is already in a container
-docker compose exec -T runner /bin/bash -c "
- cd /opt/hudi-rs && \
- make setup develop && \
- cd /opt/hudi-rs/demo/sql-datafusion && ./run.sh &&\
- cd /opt/hudi-rs/demo/table-api-python && ./run.sh && \
- cd /opt/hudi-rs/demo/table-api-rust && ./run.sh
- "
diff --git a/demo/sql-datafusion/run.sh b/demo/sql-datafusion/run.sh
deleted file mode 100755
index 2a6071f..0000000
--- a/demo/sql-datafusion/run.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-cargo run -- --no-build --no-tests
diff --git a/demo/table-api-python/run.sh b/demo/table-api-python/run.sh
deleted file mode 100755
index e80a057..0000000
--- a/demo/table-api-python/run.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-python -m src.main
diff --git a/demo/table-api-rust/run.sh b/demo/table-api-rust/run.sh
deleted file mode 100755
index 2a6071f..0000000
--- a/demo/table-api-rust/run.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-cargo run -- --no-build --no-tests
