This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 91450d849 build: support to cache archives of fetched dependencies
(#3257)
91450d849 is described below
commit 91450d84949cd0f6426b87941be8b2d5b95f5ab1
Author: Twice <[email protected]>
AuthorDate: Wed Nov 12 00:17:13 2025 +0800
build: support to cache archives of fetched dependencies (#3257)
This patch adds a new cmake variable `DEPS_FETCH_DIR`:
- fetched dependence archives will be put to this dir;
- if the archive file in this dir exists, downloading phase will be
skipped.
Hence this patch enables "offline building" of kvrocks.
Steps for offline building:
1. in an online environment, fetch kvrocks source repo and run cmake
with `-DDEPS_FETCH_DIR=<dep-dir>`.
2. copy kvrocks source code along with `<dep-dir>` to an offline
environment.
3. in the offline environment, build kvrocks with
`-DDEPS_FETCH_DIR=<dep-dir>`.
An example of files in the dep dir:
```sh
$ ll build/deps
total 40M
-rw-r--r-- 1 twice twice 559K Nov 11 18:44 cpptrace-v1.0.3.zip
-rw-r--r-- 1 twice twice 747K Nov 11 18:43 fmt-12.1.0.zip
-rw-r--r-- 1 twice twice 1.1M Nov 11 18:42 gtest-v1.17.0.zip
-rw-r--r-- 1 twice twice 958K Nov 11 18:42 jemalloc-5.3.0.zip
-rw-r--r-- 1 twice twice 2.2M Nov 11 18:43 jsoncons-v1.4.3.zip
-rw-r--r-- 1 twice twice 904K Nov 11 18:43
libevent-release-2.1.12-stable.zip
-rw-r--r-- 1 twice twice 1.2M Nov 11 18:44
luajit-c0a8e68325ec261a77bde1c8eabad398168ffe74.zip
-rw-r--r-- 1 twice twice 479K Nov 11 18:42 lz4-v1.10.0.zip
-rw-r--r-- 1 twice twice 460K Nov 11 18:43 pegtl-3.2.8.zip
-rw-r--r-- 1 twice twice 1015K Nov 11 18:44 rangev3-0.12.0.zip
-rw-r--r-- 1 twice twice 15M Nov 11 18:43 rocksdb-v10.6.2.zip
-rw-r--r-- 1 twice twice 1.2M Nov 11 18:42 snappy-1.2.2-rtti.zip
-rw-r--r-- 1 twice twice 70K Nov 11 18:43 span-v0.11.0.zip
-rw-r--r-- 1 twice twice 355K Nov 11 18:43 spdlog-v1.16.0.zip
-rw-r--r-- 1 twice twice 7.4M Nov 11 18:43 tbb-v2022.3.0.zip
-rw-r--r-- 1 twice twice 98K Nov 11 18:43
trie-906e6abd1e7063f1dacd3a6b270aa654b525eb0a.zip
-rw-r--r-- 1 twice twice 1.2M Nov 11 18:43 xxhash-v0.8.3.zip
-rw-r--r-- 1 twice twice 2.5M Nov 11 18:42 zlib-2.2.4.zip
-rw-r--r-- 1 twice twice 2.7M Nov 11 18:43 zstd-v1.5.7.zip
```
---
CMakeLists.txt | 6 +++++-
cmake/pegtl.cmake | 4 ++--
cmake/utils.cmake | 40 +++++++++++++++++++++++-----------------
3 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7d7a175c1..e69a65ae1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,8 +73,12 @@ set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(DEPS_FETCH_PROXY "" CACHE STRING
- "a template URL to proxy the traffic for fetching dependencies, e.g. with
DEPS_FETCH_PROXY = https://some-proxy/,
+ "a template URL to proxy the traffic for fetching dependencies,
+ e.g. with DEPS_FETCH_PROXY = https://some-proxy/,
https://example/some-dep.zip ->
https://some-proxy/https://example/some-dep.zip";)
+set(DEPS_FETCH_DIR "" CACHE PATH
+ "if specified, dependencies will be downloaded into this directory first,
+ and reused in subsequent builds")
if(ENABLE_ASAN AND ENABLE_TSAN)
message(FATAL_ERROR "ASan and TSan cannot be used at the same time")
diff --git a/cmake/pegtl.cmake b/cmake/pegtl.cmake
index 6e4b581f2..f71f60b78 100644
--- a/cmake/pegtl.cmake
+++ b/cmake/pegtl.cmake
@@ -19,9 +19,9 @@ include_guard()
include(cmake/utils.cmake)
-FetchContent_DeclareGitHubTarWithMirror(pegtl
+FetchContent_DeclareGitHubWithMirror(pegtl
taocpp/PEGTL 3.2.8
- MD5=50339029d1bb037909b28c382214033e
+ MD5=5c919edd001ef157b0d25fc9dcc8b3e1
)
FetchContent_MakeAvailableWithArgs(pegtl)
diff --git a/cmake/utils.cmake b/cmake/utils.cmake
index cce10cbf9..4645e45e6 100644
--- a/cmake/utils.cmake
+++ b/cmake/utils.cmake
@@ -46,23 +46,29 @@ function(FetchContent_MakeAvailableWithArgs dep)
endif()
endfunction()
-function(FetchContent_DeclareWithMirror dep url hash)
- FetchContent_Declare(${dep}
- URL ${DEPS_FETCH_PROXY}${url}
- URL_HASH ${hash}
- )
-endfunction()
-
function(FetchContent_DeclareGitHubWithMirror dep repo tag hash)
- FetchContent_DeclareWithMirror(${dep}
- https://github.com/${repo}/archive/${tag}.zip
- ${hash}
- )
-endfunction()
+ set(_dep_fetch_url
"${DEPS_FETCH_PROXY}https://github.com/${repo}/archive/${tag}.zip";)
+ if(DEPS_FETCH_DIR)
+ get_filename_component(_deps_fetch_dir ${DEPS_FETCH_DIR} ABSOLUTE)
+ set(_dep_fetch_dst ${_deps_fetch_dir}/${dep}-${tag}.zip)
-function(FetchContent_DeclareGitHubTarWithMirror dep repo tag hash)
- FetchContent_DeclareWithMirror(${dep}
- https://github.com/${repo}/archive/${tag}.tar.gz
- ${hash}
- )
+ if(NOT EXISTS ${_dep_fetch_dst})
+ message("Downloading ${_dep_fetch_url} to ${_dep_fetch_dst}...")
+ file(DOWNLOAD ${_dep_fetch_url} ${_dep_fetch_dst} STATUS
_dep_download_status)
+ list(GET _dep_download_status 0 _dep_download_status_code)
+ if(NOT _dep_download_status_code EQUAL 0)
+ message(FATAL_ERROR "Failed to download ${_dep_fetch_url} to
${_dep_fetch_dst}")
+ endif()
+ endif()
+
+ FetchContent_Declare(${dep}
+ URL ${_dep_fetch_dst}
+ URL_HASH ${hash}
+ )
+ else()
+ FetchContent_Declare(${dep}
+ URL ${_dep_fetch_url}
+ URL_HASH ${hash}
+ )
+ endif()
endfunction()
