This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 382ffd94f98 GH-51329: [C++][CI] Test static linking with S3 and fix
Azure/GCS dependencies on static builds (#51280)
382ffd94f98 is described below
commit 382ffd94f98b8df0b4283ab5c53e59c08cac8323
Author: tadeja <[email protected]>
AuthorDate: Wed Sep 30 08:43:01 2026 +0200
GH-51329: [C++][CI] Test static linking with S3 and fix Azure/GCS
dependencies on static builds (#51280)
### Rationale for this change
Fix #51329
Additional downstream static-linking test, after #50195 moved S3 into the
separate ArrowS3 library.
### What changes are included in this PR?
Test downstream static linking of `ArrowS3::arrow_s3_static` via
minimal_build example and `cpp_test.sh`,
Fix dependencies for static Azure and GCS consumers: append `libcurl` for
bundled GCS, append `libcurl` and `LibXml2` for bundled Azure(NOT WIN32),
append links for system Azure SDK
and pkg-config --static arrow to include `libcurl` and `libxml-2.0` for
bundled GCS/Azure
### Are these changes tested?
Verified by CI
### Are there any user-facing changes?
Downstream consumers now have required static dependencies for Azure/GCS
* GitHub Issue: #51329
Authored-by: Tadeja Kadunc <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
ci/scripts/cpp_test.sh | 7 ++++
cpp/cmake_modules/ThirdpartyToolchain.cmake | 40 ++++++++++++++++++++++
cpp/examples/minimal_build/CMakeLists.txt | 18 ++++++++++
cpp/examples/minimal_build/example.cc | 9 +++++
cpp/src/arrow/CMakeLists.txt | 12 +++++++
.../apache-arrow/apt/debian-forky/Dockerfile | 1 +
.../apache-arrow/apt/debian-trixie/Dockerfile | 1 +
.../apache-arrow/apt/ubuntu-jammy/Dockerfile | 1 +
.../apache-arrow/apt/ubuntu-noble/Dockerfile | 1 +
.../apache-arrow/apt/ubuntu-resolute/Dockerfile | 1 +
.../linux-packages/apache-arrow/debian/control.in | 2 ++
.../linux-packages/apache-arrow/yum/arrow.spec.in | 1 +
12 files changed, 94 insertions(+)
diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh
index 18c301c5fda..0863afc13a2 100755
--- a/ci/scripts/cpp_test.sh
+++ b/ci/scripts/cpp_test.sh
@@ -146,12 +146,19 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \
cmake \
-S "${source_dir}/examples/minimal_build" \
-B "${build_dir}/examples/minimal_build" \
+ -DARROW_EXAMPLE_S3_STATIC_CHECK=ON \
-DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
cmake --build "${build_dir}/examples/minimal_build"
pushd "${source_dir}/examples/minimal_build"
# PATH= is for Windows.
PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
"${build_dir}/examples/minimal_build/arrow-example"
+ # Test static linking with S3
+ s3_example="${build_dir}/examples/minimal_build/arrow-example-s3-static"
+ if [ -x "${s3_example}" ]; then
+ PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
+ "${s3_example}"
+ fi
popd
fi
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 807387fa830..04f93629d07 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1419,6 +1419,27 @@ macro(find_curl ARROW_CURL_PACKAGE_PREFIX)
endif()
endmacro()
+# ----------------------------------------------------------------------
+# pkg-config
+
+# SDK libraries (on macOS) may be available without .pc files
+macro(arrow_append_pc_system_library PC_PACKAGE PC_PREFIX FALLBACK)
+ find_package(PkgConfig QUIET)
+ if(PkgConfig_FOUND)
+ pkg_check_modules(${PC_PREFIX}
+ ${PC_PACKAGE}
+ NO_CMAKE_PATH
+ NO_CMAKE_ENVIRONMENT_PATH
+ QUIET)
+ endif()
+ if(PkgConfig_FOUND AND ${PC_PREFIX}_FOUND)
+ string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${PC_PACKAGE}")
+ else()
+ message(STATUS "No .pc for ${PC_PACKAGE}. Using ${FALLBACK} in arrow.pc")
+ string(APPEND ARROW_PC_LIBS_PRIVATE " ${FALLBACK}")
+ endif()
+endmacro()
+
# ----------------------------------------------------------------------
# Snappy
@@ -4365,6 +4386,25 @@ if(ARROW_WITH_AZURE_SDK)
resolve_dependency(Azure REQUIRED_VERSION 1.10.2)
set(AZURE_SDK_LINK_LIBRARIES Azure::azure-storage-files-datalake
Azure::azure-storage-blobs
Azure::azure-identity)
+ if(AZURE_SDK_VENDORED AND NOT WIN32)
+ find_curl(ARROW)
+ find_package(LibXml2 REQUIRED)
+ list(APPEND ARROW_SYSTEM_DEPENDENCIES LibXml2)
+ endif()
+endif()
+
+if(ARROW_BUILD_STATIC)
+ if((ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
+ OR (ARROW_AZURE
+ AND AZURE_SDK_VENDORED
+ AND NOT WIN32))
+ arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl")
+ endif()
+ if(ARROW_AZURE
+ AND AZURE_SDK_VENDORED
+ AND NOT WIN32)
+ arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2")
+ endif()
endif()
# ----------------------------------------------------------------------
diff --git a/cpp/examples/minimal_build/CMakeLists.txt
b/cpp/examples/minimal_build/CMakeLists.txt
index d0a0a1e0a22..e1a9217401f 100644
--- a/cpp/examples/minimal_build/CMakeLists.txt
+++ b/cpp/examples/minimal_build/CMakeLists.txt
@@ -49,3 +49,21 @@ if(ARROW_LINK_SHARED)
else()
target_link_libraries(arrow-example PRIVATE Arrow::arrow_static)
endif()
+
+option(ARROW_EXAMPLE_S3_STATIC_CHECK "Verify S3 and static linking" OFF)
+# Check only when Arrow got built with S3 by ArrowOptions.cmake via
find_package
+if(ARROW_EXAMPLE_S3_STATIC_CHECK
+ AND ARROW_S3
+ AND TARGET Arrow::arrow_static)
+ find_package(ArrowS3 REQUIRED)
+ if(NOT TARGET ArrowS3::arrow_s3_static)
+ message(FATAL_ERROR "ArrowS3::arrow_s3_static not found")
+ endif()
+ add_executable(arrow-example-s3-static example.cc)
+ target_compile_definitions(arrow-example-s3-static
+ PRIVATE ARROW_EXAMPLE_S3_STATIC_CHECK)
+ target_link_libraries(arrow-example-s3-static PRIVATE
ArrowS3::arrow_s3_static)
+elseif(ARROW_EXAMPLE_S3_STATIC_CHECK)
+ message(STATUS "Skipping arrow-example-s3-static: "
+ "requires ARROW_S3 and Arrow::arrow_static")
+endif()
diff --git a/cpp/examples/minimal_build/example.cc
b/cpp/examples/minimal_build/example.cc
index 9bfb9953edc..f600f9429d6 100644
--- a/cpp/examples/minimal_build/example.cc
+++ b/cpp/examples/minimal_build/example.cc
@@ -16,6 +16,9 @@
// under the License.
#include <arrow/csv/api.h>
+#ifdef ARROW_EXAMPLE_S3_STATIC_CHECK
+# include <arrow/filesystem/s3fs.h>
+#endif
#include <arrow/io/api.h>
#include <arrow/ipc/api.h>
#include <arrow/pretty_print.h>
@@ -54,6 +57,12 @@ Status RunMain(int argc, char** argv) {
ARROW_RETURN_NOT_OK(batch_writer->WriteTable(*table));
ARROW_RETURN_NOT_OK(batch_writer->Close());
+#ifdef ARROW_EXAMPLE_S3_STATIC_CHECK
+ std::cerr << "* Verify S3 initializes and finalizes" << std::endl;
+ ARROW_RETURN_NOT_OK(arrow::fs::EnsureS3Initialized());
+ ARROW_RETURN_NOT_OK(arrow::fs::EnsureS3Finalized());
+#endif
+
return Status::OK();
}
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 9f9ef7f2625..36edf50ba94 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -36,6 +36,18 @@ set(ARROW_STATIC_INSTALL_INTERFACE_LIBS)
if(ARROW_GCS)
if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
+ elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
+ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
+ endif()
+endif()
+
+if(ARROW_AZURE)
+ if(Azure_SOURCE STREQUAL "SYSTEM")
+ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS
${AZURE_SDK_LINK_LIBRARIES})
+ elseif(Azure_SOURCE STREQUAL "BUNDLED")
+ if(NOT WIN32)
+ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl
LibXml2::LibXml2)
+ endif()
endif()
endif()
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
index 674e16d705a..d9246d37c91 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile
@@ -68,6 +68,7 @@ RUN \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
+ libxml2-dev \
libxxhash-dev \
libzstd-dev \
llvm-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
index 2c992768418..66548e8c2f0 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile
@@ -68,6 +68,7 @@ RUN \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
+ libxml2-dev \
libxxhash-dev \
libzstd-dev \
llvm-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
index 644bc695bbd..a5b732689ba 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
@@ -60,6 +60,7 @@ RUN \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
+ libxml2-dev \
libxxhash-dev \
libzstd-dev \
llvm-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
index 496609a8925..7e9119d313e 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile
@@ -61,6 +61,7 @@ RUN \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
+ libxml2-dev \
libxxhash-dev \
libzstd-dev \
llvm-dev \
diff --git
a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
index 19f1434b7f6..b1bc2f4dd17 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile
@@ -61,6 +61,7 @@ RUN \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
+ libxml2-dev \
libxxhash-dev \
libzstd-dev \
llvm-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in
b/dev/tasks/linux-packages/apache-arrow/debian/control.in
index b66fe53dbaa..2aad235b936 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/control.in
+++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -26,6 +26,7 @@ Build-Depends:
libssl-dev,
libthrift-dev,
libutf8proc-dev,
+ libxml2-dev,
libxxhash-dev,
libzstd-dev,
meson,
@@ -185,6 +186,7 @@ Depends:
libsnappy-dev,
libssl-dev,
libutf8proc-dev,
+ libxml2-dev,
libxxhash-dev,
libzstd-dev,
nlohmann-json-dev | nlohmann-json3-dev,
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 5e6646f3864..91255b8c274 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -277,6 +277,7 @@ Requires: curl-devel
%if %{use_gcs} && !%{use_bundled_nlohmann_json}
Requires: json-devel
%endif
+Requires: libxml2-devel
Requires: libzstd-devel
Requires: lz4-devel %{lz4_requirement}
Requires: openssl-devel
