This is an automated email from the ASF dual-hosted git repository.
yuchanns pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 23f01dd54 ci(bindings/go): simplify and improve Go bindings test
infrastructure (#6293)
23f01dd54 is described below
commit 23f01dd545ebd2a7445f08cc5e7de8a19e70fb25
Author: Hanchin Hsieh <m...@yuchanns.xyz>
AuthorDate: Fri Jun 20 17:58:05 2025 +0800
ci(bindings/go): simplify and improve Go bindings test infrastructure
(#6293)
* ci(bindings/go): simplify and improve Go bindings test infrastructure
Refactor the Go bindings CI workflow to use a more streamlined approach:
- Replace complex matrix build configuration with simplified workflow
- Enhance Makefile with better organized targets and workspace management
- Update GitHub actions to use unified build process
- Simplify documentation by removing Windows-specific complex setup
* ci(bindings/go): reduce diff
---
.../actions/test_behavior_binding_go/action.yaml | 9 +-
.github/workflows/test_behavior_binding_go.yml | 120 ++-------------------
bindings/go/Makefile | 76 ++++++++++---
bindings/go/README.md | 49 +--------
4 files changed, 80 insertions(+), 174 deletions(-)
diff --git a/.github/actions/test_behavior_binding_go/action.yaml
b/.github/actions/test_behavior_binding_go/action.yaml
index bf14c322e..21c9ef7e5 100644
--- a/.github/actions/test_behavior_binding_go/action.yaml
+++ b/.github/actions/test_behavior_binding_go/action.yaml
@@ -40,10 +40,13 @@ runs:
uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
- name: Run Test Binding Go
shell: bash
- working-directory: bindings/go/tests/behavior_tests
run: |
- export
DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/opt/libffi/lib
- go test -v -run TestBehavior
+ if [ "${{ runner.os }}" == "Windows" ]; then
+ # https://github.com/ilammy/msvc-dev-cmd/issues/25
+ rm /usr/bin/link.exe
+ fi
+ cp bindings/go/Makefile .
+ make tests
env:
OPENDAL_TEST: ${{ inputs.service }}
CGO_ENABLE: 0
diff --git a/.github/workflows/test_behavior_binding_go.yml
b/.github/workflows/test_behavior_binding_go.yml
index c36ce7adf..dc58707d6 100644
--- a/.github/workflows/test_behavior_binding_go.yml
+++ b/.github/workflows/test_behavior_binding_go.yml
@@ -28,35 +28,22 @@ on:
type: string
jobs:
- set-build:
- runs-on: ubuntu-latest
- outputs:
- build: ${{ steps.set-matrix-build.outputs.build }}
- steps:
- - uses: actions/checkout@v4
- - id: set-matrix-build
- name: Setup Matrix Build
- run: |
- MATRIX=$(yq -o=json -I=0 '[.build[] | select(.os == "${{ inputs.os
}}")]' .github/scripts/test_go_binding/matrix.yaml | sed 's/ //g')
- echo "Matrix:"
- echo "$MATRIX" | jq .
- echo "build=$MATRIX" >> $GITHUB_OUTPUT
-
test:
- needs: [set-build]
name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
runs-on: ${{ inputs.os }}
strategy:
fail-fast: false
matrix:
cases: ${{ fromJson(inputs.cases) }}
- build: ${{ fromJson(needs.set-build.outputs.build) }}
steps:
- uses: actions/checkout@v4
+ - uses: actions/checkout@v4
+ with:
+ path: "opendal"
- uses: actions/checkout@v4
with:
repository: "apache/opendal-go-services"
- path: "tools"
+ path: "opendal-go-services"
- name: Setup Rust toolchain
uses: ./.github/actions/setup
with:
@@ -74,17 +61,6 @@ jobs:
with:
connect-host: ${{ secrets.OP_CONNECT_HOST }}
connect-token: ${{ secrets.OP_CONNECT_TOKEN }}
- - name: Setup Target (Linux/macOS)
- if: runner.os != 'Windows'
- env:
- TARGET: ${{ matrix.build.target }}
- run: rustup target add $TARGET
- - name: Setup Target (Windows)
- if: runner.os == 'Windows'
- env:
- TARGET: ${{ matrix.build.target }}
- run: |
- rustup target add $env:TARGET
- uses: actions/setup-go@v5
with:
go-version: stable
@@ -92,97 +68,21 @@ jobs:
bindings/go/go.sum
bindings/go/tests/behavior_tests/go.sum
- name: Install dependencies (Linux)
- if: ${{ matrix.build.os == 'ubuntu-latest' }}
+ if: runner.os == 'Linux'
run: sudo apt install zstd
- name: Install dependencies (macOS)
- if: ${{ matrix.build.os == 'macos-latest' }}
+ if: runner.os == 'macOS'
run: brew install zstd libffi
- name: Install dependencies (Windows)
- if: ${{ matrix.build.os == 'windows-latest' }}
- uses: ilammy/msvc-dev-cmd@v1
- - name: Build C Binding (Linux/macOS)
- working-directory: bindings/c
- if: runner.os != 'Windows'
- env:
- VERSION: "latest"
- SERVICE: ${{ matrix.cases.service }}
- TARGET: ${{ matrix.build.target }}
- CC: ${{ matrix.build.cc }}
- OS: ${{ matrix.build.os }}
- run: |
- SERVICE_DASH="${SERVICE//_/-}"
- cargo build --target $TARGET --release --features
"opendal/services-${SERVICE_DASH}"
- DIR=$GITHUB_WORKSPACE/libopendal_c_${VERSION}_${SERVICE}_$TARGET
- mkdir $DIR
- if [ ${OS} == 'ubuntu-latest' ]; then
- SO=so
- else
- SO=dylib
- fi
- zstd -19 ./target/${TARGET}/release/libopendal_c.${SO} -o
$DIR/libopendal_c.${TARGET}.$SO.zst
- - name: Build C Binding (Windows)
- working-directory: bindings/c
if: runner.os == 'Windows'
- env:
- VERSION: "latest"
- SERVICE: ${{ matrix.cases.service }}
- TARGET: ${{ matrix.build.target }}
- CC: ${{ matrix.build.cc }}
- run: |
- $service_dash = ${env:SERVICE} -replace '_', '-'
- cargo build --target $env:TARGET --release --features
"opendal/services-$service_dash"
-
$DIR="$env:GITHUB_WORKSPACE\libopendal_c_${env:VERSION}_${env:SERVICE}_${env:TARGET}"
- Rename-Item -Path "./target/$env:TARGET/release/opendal_c.dll"
-NewName "libopendal_c.dll"
- New-Item -ItemType Directory -Force -Path $DIR
- zstd -19 "./target/${env:TARGET}/release/libopendal_c.dll" -o
"$DIR/libopendal_c.${env:TARGET}.dll.zst"
- - name: Build Go Artifact
- working-directory: tools/internal/generate
- env:
- MATRIX: '{"build": [${{ toJson(matrix.build) }}], "service": ["${{
matrix.cases.service }}"]}'
- VERSION: "latest"
- run: |
- go run generate.go
- - name: Setup Go Workspace (Linux/macOS)
- env:
- SERVICE: ${{ matrix.cases.service }}
- working-directory: bindings/go/tests
- if: runner.os != 'Windows'
- run: |
- go work init
- go work use ..
- go work use ./behavior_tests
- go work use $GITHUB_WORKSPACE/$(echo $SERVICE | sed 's/-/_/g')
- - name: Setup Go Workspace (Windows)
+ uses: ilammy/msvc-dev-cmd@v1
+ - name: Setup Go Workspace
env:
SERVICE: ${{ matrix.cases.service }}
- working-directory: bindings/go/tests
- if: runner.os == 'Windows'
run: |
go work init
- go work use ..
- go work use ./behavior_tests
- go work use $env:GITHUB_WORKSPACE/$($env:SERVICE -replace '-','_')
- - name: Generate Test Scheme
- env:
- SERVICE: ${{ matrix.cases.service }}
- run: |
- python ./.github/scripts/test_go_binding/generate_test_scheme.py
- - name: Setup Go Module (Linux/macOS)
- env:
- SERVICE: ${{ matrix.cases.service }}
- working-directory: bindings/go/tests/behavior_tests
- if: runner.os != 'Windows'
- run: |
- go mod edit
-replace=github.com/apache/opendal-go-services/${SERVICE}=$GITHUB_WORKSPACE/$SERVICE
- go mod tidy
- - name: Setup Go Module (Windows)
- env:
- SERVICE: ${{ matrix.cases.service }}
- working-directory: bindings/go/tests/behavior_tests
- if: runner.os == 'Windows'
- run: |
- go mod edit
-replace=github.com/apache/opendal-go-services/$env:SERVICE=$env:GITHUB_WORKSPACE/$env:SERVICE
- go mod tidy
+ go work use ./opendal/bindings/go
+ go work use ./opendal/bindings/go/tests/behavior_tests
- name: Test Core
uses: ./.github/actions/test_behavior_binding_go
with:
diff --git a/bindings/go/Makefile b/bindings/go/Makefile
index 861b42661..a415bc058 100644
--- a/bindings/go/Makefile
+++ b/bindings/go/Makefile
@@ -32,17 +32,11 @@ endif
# Define file extensions for different OS
ifeq ($(OS),linux)
LIB_EXT := so
-endif
-ifeq ($(OS),darwin)
+else ifeq ($(OS),darwin)
LIB_EXT := dylib
-endif
-ifeq ($(findstring mingw,$(OS)),mingw)
- OS := windows
+else
LIB_EXT := dll
-endif
-ifeq ($(findstring msys,$(OS)),msys)
OS := windows
- LIB_EXT := dll
endif
# Check if OPENDAL_TEST environment variable is set
@@ -67,10 +61,13 @@ FINAL_LIB_FILE :=
$(GITHUB_WORKSPACE)/$(SERVICE_PKG)/libopendal_c.$(OS).$(ARCH).
# Build C library only when source files change
$(C_LIB_FILE): opendal/bindings/c/src/*.rs opendal/bindings/c/Cargo.toml
@echo "Building C library for service: $(SERVICE)"
- cd opendal/bindings/c && cargo build --features
"opendal/services-$(SERVICE)"
+ cd opendal/bindings/c && cargo build --features
"opendal/services-$(shell echo $(SERVICE) | tr '_' '-')"
+ifeq ($(OS),windows)
+ @echo "Renaming opendal_c.$(LIB_EXT) to libopendal_c.$(LIB_EXT) on
Windows"
+ mv opendal/bindings/c/target/debug/opendal_c.$(LIB_EXT)
opendal/bindings/c/target/debug/libopendal_c.$(LIB_EXT)
+endif
-# Create compressed library and generate services in one step
-# This target depends on the final library file location, not intermediate
+# Create compressed library and generate Go services
$(FINAL_LIB_FILE): $(C_LIB_FILE)
@echo "Compressing library and generating services for $(SERVICE) on
$(OS)..."
@mkdir -p $(GITHUB_WORKSPACE)/libopendal_c_$(VERSION)_$(SERVICE)_$(OS)
@@ -84,8 +81,58 @@ $(FINAL_LIB_FILE): $(C_LIB_FILE)
@echo "Cleaning up intermediate directory..."
@rm -rf $(GITHUB_WORKSPACE)/libopendal_c_$(VERSION)_$(SERVICE)_$(OS)
-# Generate services target now just depends on the final file
-generate-services: $(FINAL_LIB_FILE)
+# Setup Go workspace for the service
+setup-workspace-$(SERVICE):
+ @echo "Setting up Go workspace for service: $(SERVICE)"
+ go work use $(GITHUB_WORKSPACE)/$(SERVICE)
+ @echo "Setting up Go module replacement for service: $(SERVICE)"
+ cd opendal/bindings/go/tests/behavior_tests && \
+ go mod edit
-replace=github.com/apache/opendal-go-services/$(SERVICE)=$(GITHUB_WORKSPACE)/$(SERVICE)
&& \
+ go mod tidy
+
+define SCHEME_TEST_CONTENT
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// generated by github.com/apache/opendal/bindings/go
+
+package opendal_test
+
+import (
+ opendal "github.com/apache/opendal/bindings/go"
+ "github.com/apache/opendal-go-services/$(SERVICE)"
+)
+
+// Add more schemes for behavior tests here.
+var schemes = []opendal.Scheme{
+ $(SERVICE).Scheme,
+}
+endef
+
+export SCHEME_TEST_CONTENT
+# Generate scheme test file
+generate-scheme-$(SERVICE):
+ @echo "Generating scheme test file..."
+ @mkdir -p opendal/bindings/go/tests/behavior_tests
+ @echo "$$SCHEME_TEST_CONTENT" >
opendal/bindings/go/tests/behavior_tests/scheme_test.go
+
+# Generate services target with all steps
+generate-services: $(FINAL_LIB_FILE) setup-workspace-$(SERVICE)
generate-scheme-$(SERVICE)
# Run tests
run-tests: generate-services
@@ -101,7 +148,10 @@ clean:
@echo "Cleaning all generated files..."
rm -rf opendal/bindings/c/target
rm -rf opendal-go-services/*/libopendal_c.*
+ rm -rf opendal-go-services/*/opendal_c.*
rm -rf opendal-go-services/libopendal_c_*
+ rm -rf opendal-go-services/opendal_c_*
+ cd opendal/bindings/go/tests/behavior_tests && go clean -testcache
# Help target
help:
diff --git a/bindings/go/README.md b/bindings/go/README.md
index 265c9b170..fc4bad89d 100644
--- a/bindings/go/README.md
+++ b/bindings/go/README.md
@@ -222,7 +222,7 @@ We use `go workspace` to manage and build the dependencies.
To set up the worksp
<details>
<summary>
- For Linux and macOS
+ For Linux, macOS and Windows (MSVC)
</summary>
<br/>
@@ -236,9 +236,6 @@ git clone --depth 1
g...@github.com:apache/opendal-go-services.git
go work init
go work use ./opendal/bindings/go
go work use ./opendal/bindings/go/tests/behavior_tests
-# use the backend you want to test, e.g., fs or memory
-go work use ./opendal-go-services/fs
-go work use ./opendal-go-services/memory
cp opendal/bindings/go/Makefile .
@@ -261,50 +258,6 @@ cd -
</details>
-<details>
- <summary>
- For Windows
- </summary>
-
-<br/>
-
-```powershell
-New-Item -ItemType Directory -Path opendal_workspace
-Set-Location -Path opendal_workspace
-
-git clone --depth 1 g...@github.com:apache/opendal.git
-git clone --depth 1 g...@github.com:apache/opendal-go-services.git
-
-go work init
-go work use ./opendal/bindings/go
-go work use ./opendal/bindings/go/tests/behavior_tests
-# use the backend you want to test, e.g., fs or memory
-go work use ./opendal-go-services/fs
-go work use ./opendal-go-services/memory
-
-@'
-# Check if OPENDAL_TEST is set\;if (-not $env:OPENDAL_TEST) {\; Write-Error
"OPENDAL_TEST environment variable is not set"\; Write-Host "Please set
OPENDAL_TEST to specify which backend to test (e.g., fs or memory)"\; exit
1\;}\;# Specify the backend to test\;Set-Item -Path Env:SERVICE -Value
"$env:OPENDAL_TEST"\;# Get architecture\;$architecture = (Get-WmiObject
Win32_OperatingSystem).OSArchitecture\;\;if ($architecture -like "*64*") {\;
$ARCH = "x86_64"\;} else {\; $ARCH [...]
-'@ -replace "\\;","`n" | Out-File -FilePath "MakeTest.ps1" -Encoding UTF8
-
-Pop-Location
-```
-
-To build and run tests, run the following commands:
-
-```powershell
-Set-Location -Path opendal_workspace
-# specify the backend to test
-$env:OPENDAL_TEST = "fs"
-$env:OPENDAL_FS_ROOT = $env:TEMP
-
-# build the C binding and run the tests
-.\MakeTest.ps1
-
-Pop-Location
-```
-
-</details>
-
## License and Trademarks
Licensed under the Apache License, Version 2.0:
http://www.apache.org/licenses/LICENSE-2.0
