This is an automated email from the ASF dual-hosted git repository. jiayu pushed a commit to branch update/duckdb-150-benchmark-defaults in repository https://gitbox.apache.org/repos/asf/sedona-spatialbench.git
commit 1468bc48b7aa4a652df4783e5c5d1477867d3939 Author: Jia Yu <[email protected]> AuthorDate: Sun Mar 22 20:02:04 2026 -0700 Update DuckDB to 1.5.0, default to SF10 with 1800s timeout, and add crash resilience - Remove DuckDB <1.5.0 version cap and default to stable release instead of nightly (spatial extension is native in 1.5+) - Change default benchmark scale factor from 1 to 10 with 1800s query timeout - Save benchmark results incrementally so partial results survive crashes - Add continue-on-error to benchmark steps so artifacts are uploaded even on failure, and the summarize job can report whatever completed Closes #92 --- .github/workflows/benchmark.yml | 53 +++++++++++++++++++++++------------------ benchmark/run_benchmark.py | 10 +++++++- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9294d55..faf384d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,7 +23,7 @@ on: scale_factor: description: 'Scale factor for benchmark' required: false - default: '1' + default: '10' type: choice options: - '0.1' @@ -35,9 +35,9 @@ on: default: 'duckdb,geopandas,sedonadb,spatial_polars' type: string timeout: - description: 'Query timeout in seconds (default: 60, increase for full benchmark)' + description: 'Query timeout in seconds (default: 1800)' required: false - default: '60' + default: '1800' type: string sedonadb_version: description: 'SedonaDB version (e.g., 1.0.0, leave empty for latest)' @@ -76,7 +76,7 @@ on: duckdb_nightly: description: 'Use DuckDB pre-release/nightly build (ignores version if true)' required: false - default: true + default: false type: boolean concurrency: @@ -85,9 +85,9 @@ concurrency: env: CARGO_TERM_COLOR: always - SCALE_FACTOR: ${{ github.event.inputs.scale_factor || '1' }} + SCALE_FACTOR: ${{ github.event.inputs.scale_factor || '10' }} BENCHMARK_ENGINES: ${{ github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars' }} - QUERY_TIMEOUT: ${{ github.event.inputs.timeout || '60' }} + QUERY_TIMEOUT: ${{ github.event.inputs.timeout || '1800' }} BENCHMARK_RUNS: ${{ github.event.inputs.runs || '3' }} # Package versions (empty = latest, can be overridden via workflow_dispatch) SEDONADB_VERSION: ${{ github.event.inputs.sedonadb_version }} @@ -96,7 +96,7 @@ env: SPATIAL_POLARS_VERSION: ${{ github.event.inputs.spatial_polars_version }} # Nightly build options (default: true) SEDONADB_NIGHTLY: ${{ github.event.inputs.sedonadb_nightly || 'true' }} - DUCKDB_NIGHTLY: ${{ github.event.inputs.duckdb_nightly || 'true' }} + DUCKDB_NIGHTLY: ${{ github.event.inputs.duckdb_nightly || 'false' }} # Hugging Face dataset for benchmark data HF_DATASET: apache-sedona/spatialbench HF_DATA_VERSION: v0.1.0 @@ -104,7 +104,7 @@ env: jobs: # Download benchmark data from Hugging Face download-data: - name: Download Data (SF${{ github.event.inputs.scale_factor || '1' }}) + name: Download Data (SF${{ github.event.inputs.scale_factor || '10' }}) runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -186,7 +186,7 @@ jobs: du -sh benchmark-data-sf${{ env.SCALE_FACTOR }}/ benchmark-duckdb: - name: Benchmark DuckDB (SF${{ github.event.inputs.scale_factor || '1' }}) + name: Benchmark DuckDB (SF${{ github.event.inputs.scale_factor || '10' }}) needs: download-data runs-on: ubuntu-latest if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'duckdb') @@ -212,9 +212,8 @@ jobs: echo "DUCKDB_VERSION: ${{ env.DUCKDB_VERSION }}" echo "======================================" if [ "${{ env.DUCKDB_NIGHTLY }}" = "true" ]; then - # Use --pre to install pre-release dev builds (e.g., 1.4.4.dev48) - # Constraint <1.5.0 ensures we get 1.4.x branch dev builds - pip install "duckdb<1.5.0" --pre pyarrow pandas + # Use --pre to install pre-release dev builds + pip install duckdb --pre pyarrow pandas elif [ -n "${{ env.DUCKDB_VERSION }}" ]; then pip install "duckdb==${{ env.DUCKDB_VERSION }}" pyarrow pandas else @@ -224,10 +223,12 @@ jobs: - name: Pre-install DuckDB spatial extension run: | - # Dev builds don't have spatial extension in core_nightly, so always use default repo - python -c "import duckdb; con = duckdb.connect(); con.execute('INSTALL spatial'); print('DuckDB spatial extension installed')" + # In DuckDB 1.5+, spatial is a native extension. INSTALL+LOAD still works for all versions. + python -c "import duckdb; con = duckdb.connect(); con.execute('INSTALL spatial'); con.execute('LOAD spatial'); print('DuckDB spatial extension installed and loaded')" - name: Run DuckDB benchmark + id: run-benchmark + continue-on-error: true run: | python benchmark/run_benchmark.py \ --data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \ @@ -238,6 +239,7 @@ jobs: --output duckdb_results.json - name: Upload results + if: always() && hashFiles('duckdb_results.json') != '' uses: actions/upload-artifact@v6 with: name: duckdb-results-sf${{ env.SCALE_FACTOR }} @@ -245,7 +247,7 @@ jobs: retention-days: 30 benchmark-geopandas: - name: Benchmark GeoPandas (SF${{ github.event.inputs.scale_factor || '1' }}) + name: Benchmark GeoPandas (SF${{ github.event.inputs.scale_factor || '10' }}) needs: download-data runs-on: ubuntu-latest if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'geopandas') @@ -274,6 +276,8 @@ jobs: echo "Installed GeoPandas version: $(python -c 'from importlib.metadata import version; print(version("geopandas"))')" - name: Run GeoPandas benchmark + id: run-benchmark + continue-on-error: true run: | python benchmark/run_benchmark.py \ --data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \ @@ -284,6 +288,7 @@ jobs: --output geopandas_results.json - name: Upload results + if: always() && hashFiles('geopandas_results.json') != '' uses: actions/upload-artifact@v6 with: name: geopandas-results-sf${{ env.SCALE_FACTOR }} @@ -291,7 +296,7 @@ jobs: retention-days: 30 benchmark-sedonadb: - name: Benchmark SedonaDB (SF${{ github.event.inputs.scale_factor || '1' }}) + name: Benchmark SedonaDB (SF${{ github.event.inputs.scale_factor || '10' }}) needs: download-data runs-on: ubuntu-latest if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'sedonadb') @@ -330,6 +335,8 @@ jobs: echo "Installed SedonaDB version: $(python -c 'from importlib.metadata import version; print(version("sedonadb"))')" - name: Run SedonaDB benchmark + id: run-benchmark + continue-on-error: true run: | python benchmark/run_benchmark.py \ --data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \ @@ -340,6 +347,7 @@ jobs: --output sedonadb_results.json - name: Upload results + if: always() && hashFiles('sedonadb_results.json') != '' uses: actions/upload-artifact@v6 with: name: sedonadb-results-sf${{ env.SCALE_FACTOR }} @@ -347,7 +355,7 @@ jobs: retention-days: 30 benchmark-spatial-polars: - name: Benchmark Spatial Polars (SF${{ github.event.inputs.scale_factor || '1' }}) + name: Benchmark Spatial Polars (SF${{ github.event.inputs.scale_factor || '10' }}) needs: download-data runs-on: ubuntu-latest if: contains(github.event.inputs.engines || 'duckdb,geopandas,sedonadb,spatial_polars', 'spatial_polars') @@ -376,6 +384,8 @@ jobs: echo "Installed Spatial Polars version: $(python -c 'from importlib.metadata import version; print(version("spatial-polars"))')" - name: Run Spatial Polars benchmark + id: run-benchmark + continue-on-error: true run: | python benchmark/run_benchmark.py \ --data-dir benchmark-data-sf${{ env.SCALE_FACTOR }} \ @@ -386,6 +396,7 @@ jobs: --output spatial_polars_results.json - name: Upload results + if: always() && hashFiles('spatial_polars_results.json') != '' uses: actions/upload-artifact@v6 with: name: spatial_polars-results-sf${{ env.SCALE_FACTOR }} @@ -393,15 +404,14 @@ jobs: retention-days: 30 summarize-results: - name: Summarize Results (SF${{ github.event.inputs.scale_factor || '1' }}) + name: Summarize Results (SF${{ github.event.inputs.scale_factor || '10' }}) needs: [benchmark-duckdb, benchmark-geopandas, benchmark-sedonadb, benchmark-spatial-polars] - if: always() && (needs.benchmark-duckdb.result == 'success' || needs.benchmark-geopandas.result == 'success' || needs.benchmark-sedonadb.result == 'success' || needs.benchmark-spatial-polars.result == 'success') + if: always() && (needs.benchmark-duckdb.result != 'cancelled' || needs.benchmark-geopandas.result != 'cancelled' || needs.benchmark-sedonadb.result != 'cancelled' || needs.benchmark-spatial-polars.result != 'cancelled') runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Download DuckDB results - if: needs.benchmark-duckdb.result == 'success' uses: actions/download-artifact@v7 with: name: duckdb-results-sf${{ env.SCALE_FACTOR }} @@ -409,7 +419,6 @@ jobs: continue-on-error: true - name: Download GeoPandas results - if: needs.benchmark-geopandas.result == 'success' uses: actions/download-artifact@v7 with: name: geopandas-results-sf${{ env.SCALE_FACTOR }} @@ -417,7 +426,6 @@ jobs: continue-on-error: true - name: Download SedonaDB results - if: needs.benchmark-sedonadb.result == 'success' uses: actions/download-artifact@v7 with: name: sedonadb-results-sf${{ env.SCALE_FACTOR }} @@ -425,7 +433,6 @@ jobs: continue-on-error: true - name: Download Spatial Polars results - if: needs.benchmark-spatial-polars.result == 'success' uses: actions/download-artifact@v7 with: name: spatial_polars-results-sf${{ env.SCALE_FACTOR }} diff --git a/benchmark/run_benchmark.py b/benchmark/run_benchmark.py index e05e237..4a8d8d5 100755 --- a/benchmark/run_benchmark.py +++ b/benchmark/run_benchmark.py @@ -428,6 +428,7 @@ def run_benchmark( timeout: int, scale_factor: float, runs: int = 3, + output_file: str | None = None, ) -> BenchmarkSuite: """Generic benchmark runner for any engine. @@ -438,6 +439,9 @@ def run_benchmark( If runs > 1 and the first run succeeds, additional runs are performed and the average time is reported for fair comparison. + + If output_file is provided, results are saved incrementally after each + query so that partial results survive if the runner crashes mid-way. """ from importlib.metadata import version as pkg_version @@ -538,6 +542,10 @@ def run_benchmark( if result.status == "success": suite.total_time += result.time_seconds + # Save partial results after each query so they survive crashes + if output_file: + save_results([suite], output_file) + return suite @@ -629,7 +637,7 @@ def main(): print(f" {table}: {path}") results = [ - run_benchmark(engine, data_paths, queries, args.timeout, args.scale_factor, args.runs) + run_benchmark(engine, data_paths, queries, args.timeout, args.scale_factor, args.runs, args.output) for engine in engines ]
