This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new d9615dc9cb GH-50046: [CI][C++] Improve caching with
apache/infrastructure-actions/stash and more general cache keys (#50047)
d9615dc9cb is described below
commit d9615dc9cbd5ceac679ed1788b59629c9ef541a6
Author: Rok Mihevc <[email protected]>
AuthorDate: Fri Jun 5 02:59:23 2026 +0200
GH-50046: [CI][C++] Improve caching with
apache/infrastructure-actions/stash and more general cache keys (#50047)
### Rationale for this change
C++ CI jobs currently key `ccache`/Docker volume caches with
`hashFiles('cpp/**')`. This creates new immutable GitHub Actions cache entries
whenever any C++ file changes, even though `ccache` already handles per-file
invalidation internally.
Recent CI logs show that when caches are restored, C++ jobs get high ccache
hit rates, but GitHub cache restore misses are common and cause large CI
runtime regressions.
### What changes are included in this PR?
* Replace `actions/cache` with
`apache/infrastructure-actions/stash/restore` and
`apache/infrastructure-actions/stash/save`.
* Remove `hashFiles('cpp/**')` from cache keys.
* Use stable restore prefixes per job/config.
### Are these changes tested?
By CI?
### Are there any user-facing changes?
CI users should see faster builds..
* GitHub Issue: #50046
Lead-authored-by: Rok Mihevc <[email protected]>
Co-authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.github/workflows/cpp.yml | 44 +++++++++++++------
.github/workflows/cpp_extra.yml | 91 ++++++++++++++++++++++++++++-----------
.github/workflows/cpp_windows.yml | 15 +++++--
3 files changed, 110 insertions(+), 40 deletions(-)
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 90c06c7be0..155133f6a9 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -63,6 +63,7 @@ concurrency:
cancel-in-progress: true
permissions:
+ actions: read
contents: read
env:
@@ -119,12 +120,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
submodules: recursive
- - name: Cache Docker Volumes
- uses: actions/cache@v5
+ - name: Restore Docker Volumes
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
- key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
- restore-keys: ${{ matrix.image }}-
+ key: cpp-${{ matrix.image }}
- name: Setup Python on hosted runner
if: |
matrix.runs-on == 'ubuntu-latest'
@@ -149,6 +149,14 @@ jobs:
sudo sysctl -w vm.mmap_rnd_bits=28
source ci/scripts/util_enable_core_dumps.sh
archery docker run ${{ matrix.image }}
+ - name: Save Docker Volumes
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: .docker
+ key: cpp-${{ matrix.image }}
+ include-hidden-files: true
- name: Docker Push
if: >-
success() &&
@@ -254,12 +262,11 @@ jobs:
id: ccache-info
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- - name: Cache ccache
- uses: actions/cache@v5
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
- key: cpp-ccache-macos-${{ matrix.macos-version }}-${{
hashFiles('cpp/**') }}
- restore-keys: cpp-ccache-macos-${{ matrix.macos-version }}-
+ key: cpp-ccache-macos-${{ matrix.macos-version }}
- name: Build
run: |
if [ "${{ matrix.macos-version }}" = "15-intel" ]; then
@@ -276,6 +283,13 @@ jobs:
export BUILD_WARNING_LEVEL=PRODUCTION
fi
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
+ - name: Save ccache
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ${{ steps.ccache-info.outputs.cache-dir }}
+ key: cpp-ccache-macos-${{ matrix.macos-version }}
- name: Test
shell: bash
run: |
@@ -361,17 +375,23 @@ jobs:
- name: Setup MSYS2
shell: msys2 {0}
run: ci/scripts/msys2_setup.sh cpp
- - name: Cache ccache
- uses: actions/cache@v5
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ccache
- key: cpp-ccache-${{ matrix.msystem_lower}}-${{ hashFiles('cpp/**') }}
- restore-keys: cpp-ccache-${{ matrix.msystem_lower}}-
+ key: cpp-ccache-${{ matrix.msystem_lower}}
- name: Build
shell: msys2 {0}
run: |
export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+ - name: Save ccache
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ccache
+ key: cpp-ccache-${{ matrix.msystem_lower}}
- name: Download Timezone Database
if: matrix.msystem_upper == 'CLANG64'
shell: bash
diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml
index 73b06f9dee..265a6f66b8 100644
--- a/.github/workflows/cpp_extra.yml
+++ b/.github/workflows/cpp_extra.yml
@@ -141,12 +141,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
submodules: recursive
- - name: Cache Docker Volumes
- uses: actions/cache@v5
+ - name: Restore Docker Volumes
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
- key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }}
- restore-keys: extra-${{ matrix.image }}-
+ key: cpp-extra-${{ matrix.image }}
- name: Setup Python
uses: actions/setup-python@v6
with:
@@ -169,6 +168,14 @@ jobs:
done
fi
archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image
}}
+ - name: Save Docker Volumes
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: .docker
+ key: cpp-extra-${{ matrix.image }}
+ include-hidden-files: true
- name: Docker Push
if: >-
success() &&
@@ -204,6 +211,8 @@ jobs:
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'),
'CI: Extra: C++')
timeout-minutes: 240
permissions:
+ actions: read
+ contents: read
# This is for using GitHub Packages for vcpkg cache
packages: write
strategy:
@@ -227,12 +236,11 @@ jobs:
- name: Free up disk space
run: |
ci/scripts/util_free_space.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v5
+ - name: Restore Docker Volumes
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
- key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }}
- restore-keys: jni-${{ matrix.platform.runs-on }}-
+ key: jni-${{ matrix.platform.runs-on }}
- name: Setup Python
uses: actions/setup-python@v6
with:
@@ -248,6 +256,14 @@ jobs:
run: |
source ci/scripts/util_enable_core_dumps.sh
archery docker run cpp-jni
+ - name: Save Docker Volumes
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: .docker
+ key: jni-${{ matrix.platform.runs-on }}
+ include-hidden-files: true
- name: Docker Push
if: >-
success() &&
@@ -300,12 +316,11 @@ jobs:
- name: Prepare ccache
run: |
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
- - name: Cache ccache
- uses: actions/cache@v5
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ccache
- key: jni-macos-${{ hashFiles('cpp/**') }}
- restore-keys: jni-macos-
+ key: jni-macos
- name: CMake
run: |
cmake \
@@ -347,6 +362,13 @@ jobs:
cmake --build cpp/examples/minimal_build.build
cd cpp/examples/minimal_build
../minimal_build.build/arrow-example
+ - name: Save ccache
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ccache
+ key: jni-macos
odbc-linux:
needs: check-labels
@@ -372,12 +394,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
submodules: recursive
- - name: Cache Docker Volumes
- uses: actions/cache@v5
+ - name: Restore Docker Volumes
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: .docker
- key: ubuntu-cpp-odbc-${{ hashFiles('cpp/**') }}
- restore-keys: ubuntu-cpp-odbc-
+ key: ubuntu-cpp-odbc
- name: Setup Python on hosted runner
uses: actions/setup-python@v6
with:
@@ -393,6 +414,14 @@ jobs:
sudo sysctl -w vm.mmap_rnd_bits=28
source ci/scripts/util_enable_core_dumps.sh
archery docker run ubuntu-cpp-odbc
+ - name: Save Docker Volumes
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: .docker
+ key: ubuntu-cpp-odbc
+ include-hidden-files: true
- name: Docker Push
if: >-
success() &&
@@ -464,12 +493,11 @@ jobs:
id: ccache-info
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- - name: Cache ccache
- uses: actions/[email protected]
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
- key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{
matrix.build-type }}-${{ hashFiles('cpp/**') }}
- restore-keys: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{
matrix.build-type }}-
+ key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{
matrix.build-type }}
- name: Build
run: |
# Homebrew uses /usr/local as prefix. So packages
@@ -486,6 +514,13 @@ jobs:
export ARROW_CMAKE_ARGS="-DODBC_INCLUDE_DIR=$ODBC_INCLUDE_DIR"
export CXXFLAGS="$CXXFLAGS -I$ODBC_INCLUDE_DIR"
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
+ - name: Save ccache
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ${{ steps.ccache-info.outputs.cache-dir }}
+ key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{
matrix.build-type }}
- name: Setup Python
uses: actions/setup-python@v6
with:
@@ -541,6 +576,8 @@ jobs:
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'),
'CI: Extra: C++')
timeout-minutes: 240
permissions:
+ actions: read
+ contents: read
packages: write
env:
ARROW_BUILD_SHARED: ON
@@ -589,12 +626,11 @@ jobs:
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- - name: Cache ccache
- uses: actions/cache@v5
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
- key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }}
- restore-keys: cpp-odbc-ccache-windows-x64-
+ key: cpp-odbc-ccache-windows-x64
- name: Checkout vcpkg
uses: actions/checkout@v6
with:
@@ -630,6 +666,13 @@ jobs:
call "C:\Program Files\Microsoft Visual
Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
set VCPKG_ROOT=%VCPKG_ROOT_KEEP%
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
+ - name: Save ccache
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ${{ steps.ccache-info.outputs.cache-dir }}
+ key: cpp-odbc-ccache-windows-x64
- name: Register Flight SQL ODBC Driver
shell: cmd
run: |
diff --git a/.github/workflows/cpp_windows.yml
b/.github/workflows/cpp_windows.yml
index 18afb74ace..0fa816b3a2 100644
--- a/.github/workflows/cpp_windows.yml
+++ b/.github/workflows/cpp_windows.yml
@@ -34,6 +34,7 @@ on:
type: string
permissions:
+ actions: read
contents: read
jobs:
@@ -107,12 +108,11 @@ jobs:
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- - name: Cache ccache
- uses: actions/cache@v5
+ - name: Restore ccache
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
- key: cpp-ccache-windows-${{ inputs.arch }}-${{ hashFiles('cpp/**') }}
- restore-keys: cpp-ccache-windows-${{ inputs.arch }}-
+ key: cpp-ccache-windows-${{ inputs.arch }}
- name: Build
shell: cmd
env:
@@ -120,6 +120,13 @@ jobs:
run: |
call "C:\Program Files\Microsoft Visual
Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" %VCVARS_ARCH%
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
+ - name: Save ccache
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: ${{ steps.ccache-info.outputs.cache-dir }}
+ key: cpp-ccache-windows-${{ inputs.arch }}
- name: Test
shell: cmd
env:
