This is an automated email from the ASF dual-hosted git repository. raulcd pushed a commit to branch maint-26.0.x in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 840f69b0e86cd336882c7ef2eeed96141984f1bf Author: Curt Hagenlocher <[email protected]> AuthorDate: Wed Oct 28 19:02:33 2026 -0700 GH-55165: [C++][Packaging][R] Use Brotli 1.2.1 for bundled, vcpkg and R Windows builds (#55167) There is a security problem in bundled Brotli (1.0.9), pinned vcpkg Brotli (1.2.0) and rtools' Brotli (1.0.9). We should update them. * Bump the bundled Brotli to 1.2.1 from 1.0.9. * Bump vcpkg to 2026.10.24 (`61d7eaeb09`) from 2026.05.27 (`9b965a1168`). * Use the bundled Brotli instead of the rtools one for R Windows binaries. Yes. Yes. **PR code and description written by:** - [x] Human - [x] AI **Reviewed before submission by:** - [x] Human - [ ] AI - [ ] Not reviewed * GitHub Issue: #55165 Lead-authored-by: Raúl Cumplido <[email protected]> Co-authored-by: Curt Hagenlocher <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]> --- .env | 2 +- c_glib/vcpkg.json | 2 +- ci/scripts/PKGBUILD | 7 ++- ci/scripts/r_windows_build.sh | 6 +-- ci/vcpkg/ports.patch | 82 ++++++++++++++++++++++++----- cpp/cmake_modules/ThirdpartyToolchain.cmake | 47 +++-------------- cpp/thirdparty/versions.txt | 4 +- cpp/vcpkg.json | 2 +- r/configure.win | 3 +- 9 files changed, 91 insertions(+), 64 deletions(-) diff --git a/.env b/.env index cd30f7aa7c..bff4edd664 100644 --- a/.env +++ b/.env @@ -92,7 +92,7 @@ TZ=UTC # Used through compose.yaml and serves as the default version for the # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the # docker tags more readable. -VCPKG="9b965a116838c6cdcd36bca60d1b81b030c8ab8d" # 2026.05.27 (not release, upstream commit) +VCPKG="61d7eaeb094a6acbc23d9c2e54de9208175fda47" # 2026.10.24 (not release, upstream commit) # This must be updated when we update # ci/docker/python-*-windows-*.dockerfile or the vcpkg config. diff --git a/c_glib/vcpkg.json b/c_glib/vcpkg.json index 77a423f77a..ebec649da2 100644 --- a/c_glib/vcpkg.json +++ b/c_glib/vcpkg.json @@ -7,5 +7,5 @@ "pkgconf" ], "$comment": "We can update builtin-baseline by 'vcpkg x-update-baseline'", - "builtin-baseline": "9b965a116838c6cdcd36bca60d1b81b030c8ab8d" + "builtin-baseline": "61d7eaeb094a6acbc23d9c2e54de9208175fda47" } diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index e9031a5736..a7685e735c 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -31,8 +31,7 @@ depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-snappy" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" - "${MINGW_PACKAGE_PREFIX}-zstd" - "${MINGW_PACKAGE_PREFIX}-brotli") + "${MINGW_PACKAGE_PREFIX}-zstd") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") @@ -96,6 +95,9 @@ build() { # But one on rtools packages doesn't have the fix. So we can't use # the MINGW one. + # We use the bundled Brotli instead of the MINGW one because the one + # on rtools packages is unmaintained and stuck at 1.0.9. + # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments @@ -137,6 +139,7 @@ build() { -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_CXXFLAGS="${CPPFLAGS}" \ -DAWSSDK_SOURCE=BUNDLED \ + -DBrotli_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE="release" \ -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \ -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index 7eb8c0c2b5..eec1fb5bd6 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -65,7 +65,7 @@ if [ -d mingw64/lib/ ]; then # Move the 64-bit versions of libarrow into the expected location mv mingw64/lib/*.a "${DST_DIR}/lib/x64" # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" + cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" fi # Same for the 32-bit versions @@ -73,7 +73,7 @@ if [ -d mingw32/lib/ ]; then ls "${MSYS_LIB_DIR}/mingw32/lib/" mkdir -p "${DST_DIR}/lib/i386" mv mingw32/lib/*.a "${DST_DIR}/lib/i386" - cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" + cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" fi # Do the same also for ucrt64 @@ -81,7 +81,7 @@ if [ -d ucrt64/lib/ ]; then ls "${MSYS_LIB_DIR}/ucrt64/lib/" mkdir -p "${DST_DIR}/lib/x64-ucrt" mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" - cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt + cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt fi # Create build artifact diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index db38c25de3..dd62862d19 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -19,19 +19,6 @@ index 73f5aece46..68cdb3966f 100644 ) vcpkg_cmake_install() vcpkg_copy_pdbs() -diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake -index 7764357a6d..da7374ecec 100644 ---- a/ports/llvm/portfile.cmake -+++ b/ports/llvm/portfile.cmake -@@ -302,6 +302,8 @@ vcpkg_cmake_configure( - ${FEATURE_OPTIONS} - MAYBE_UNUSED_VARIABLES - COMPILER_RT_ENABLE_IOS -+ BOLT_TOOLS_INSTALL_DIR -+ LIBOMP_INSTALL_ALIASES - ) - - vcpkg_cmake_install(ADD_BIN_TO_PATH) diff --git a/ports/orc/orc-fix-exception-propagation.diff b/ports/orc/orc-fix-exception-propagation.diff new file mode 100644 index 0000000000..25568e70cd @@ -114,3 +101,72 @@ index 5df5e94..225e5fb 100644 ) if("tool" IN_LIST FEATURES) +diff --git a/ports/nanoarrow/portfile.cmake b/ports/nanoarrow/portfile.cmake +index 34f113f..10ca500 100644 +--- a/ports/nanoarrow/portfile.cmake ++++ b/ports/nanoarrow/portfile.cmake +@@ -24,6 +24,10 @@ vcpkg_cmake_configure( + OPTIONS + -DNANOARROW_INSTALL_SHARED=${NANOARROW_INSTALL_SHARED} + -DNANOARROW_DEBUG_EXTRA_WARNINGS=OFF ++ # Prefix the C symbols so that consumers linking nanoarrow ++ # statically (e.g. azure-storage-blobs-cpp inside libarrow) do ++ # not export or clash on plain Arrow* names. ++ -DNANOARROW_NAMESPACE=VcpkgNanoarrow + ${FEATURE_OPTIONS} + ${FLATCCRT_OPTIONS} + ) +@@ -36,12 +40,12 @@ vcpkg_cmake_config_fixup( + DO_NOT_DELETE_PARENT_CONFIG_PATH + ) + +-# Fix bare "flatccrt" in INTERFACE_LINK_LIBRARIES - Windows needs the full path ++# Fix bare "flatccrt" in INTERFACE_LINK_LIBRARIES - consumers need the full path + # Tracked upstream at https://github.com/apache/arrow-nanoarrow/issues/922 +-if ("ipc" IN_LIST FEATURES AND VCPKG_TARGET_IS_WINDOWS) ++if ("ipc" IN_LIST FEATURES) + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/nanoarrow/nanoarrow-targets.cmake" + [[\$<LINK_ONLY:flatccrt>]] +- [[\$<LINK_ONLY:${_IMPORT_PREFIX}/lib/flatccrt.lib>]] ++ "\\\$<LINK_ONLY:\${_IMPORT_PREFIX}/lib/${VCPKG_TARGET_STATIC_LIBRARY_PREFIX}flatccrt${VCPKG_TARGET_STATIC_LIBRARY_SUFFIX}>" + ) + endif() + +diff --git a/ports/protobuf/fix-msvc-global-empty-string.patch b/ports/protobuf/fix-msvc-global-empty-string.patch +new file mode 100644 +index 0000000..da1d995 +--- /dev/null ++++ b/ports/protobuf/fix-msvc-global-empty-string.patch +@@ -0,0 +1,20 @@ ++diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h ++index 7347203..a587be8 100644 ++--- a/src/google/protobuf/port.h +++++ b/src/google/protobuf/port.h ++@@ -876,9 +876,14 @@ class alignas(8) GlobalEmptyStringConstexpr { ++ // evaluation, not that a constant evaluated instance can leak into runtime. ++ // Memory allocated during constant evaluation can't be used for runtime ++ // objects. ++-#if !defined(__XTENSA__) +++#if !defined(__XTENSA__) && !defined(_MSC_VER) ++ // Disable the optimization for Xtensa. ++ // Compilation fails on Xtensa: b/467129751 +++ // Disable the optimization for MSVC too, as protobuf 33.x did. +++ // MSVC mangles the type of fixed_address_empty_string into its symbol +++ // name, so libprotobuf and generated code built with different C++ +++ // standards (C++17 vs C++20) fail to link. +++ // See https://github.com/microsoft/vcpkg/issues/54654 ++ template < ++ typename Alloc = NonConstexprAllocator<char>, ++ int = std::basic_string<char, std::char_traits<char>, Alloc>().size()> +diff --git a/ports/protobuf/portfile.cmake b/ports/protobuf/portfile.cmake +index 74570d4..ae656fc 100644 +--- a/ports/protobuf/portfile.cmake ++++ b/ports/protobuf/portfile.cmake +@@ -11,6 +11,7 @@ vcpkg_from_github( + fix-install-dirs.patch + fix-constinit-with-clang-cl.patch + fix-upb.patch ++ fix-msvc-global-empty-string.patch + ) + + string(COMPARE EQUAL "${TARGET_TRIPLET}" "${HOST_TRIPLET}" protobuf_BUILD_PROTOC_BINARIES) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 368a26efcf..dd7228249b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1540,36 +1540,19 @@ macro(build_brotli) set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") set(BROTLI_LIB_DIR "${BROTLI_PREFIX}/lib") + # Brotli >= 1.1.0 names its static libraries without a "-static" + # suffix when BUILD_SHARED_LIBS=OFF. set(BROTLI_STATIC_LIBRARY_ENC - "${BROTLI_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(BROTLI_STATIC_LIBRARY_DEC - "${BROTLI_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(BROTLI_STATIC_LIBRARY_COMMON - "${BROTLI_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - set(BROTLI_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}") - - set(BROTLI_EP_OPTIONS) - if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - # "cmake install" is disabled for Brotli on Emscripten, so the - # default INSTALL_COMMAND fails. We need to disable the default - # INSTALL_COMMAND. - list(APPEND - BROTLI_EP_OPTIONS - INSTALL_COMMAND - ${CMAKE_COMMAND} - -E - true) - - set(BROTLI_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/brotli_ep-prefix/src/brotli_ep-build) - set(BROTLI_BUILD_LIBS - "${BROTLI_BUILD_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX}" - "${BROTLI_BUILD_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX}" - "${BROTLI_BUILD_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - endif() + set(BROTLI_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}" + -DBROTLI_BUILD_TOOLS=OFF -DBROTLI_DISABLE_TESTS=ON) externalproject_add(brotli_ep ${EP_COMMON_OPTIONS} @@ -1578,22 +1561,8 @@ macro(build_brotli) BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}" - ${BROTLI_BUILD_BYPRODUCTS} CMAKE_ARGS ${BROTLI_CMAKE_ARGS} - STEP_TARGETS headers_copy ${BROTLI_EP_OPTIONS}) - - if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - # Copy the libraries to our install directory manually. - set(BROTLI_BUILD_INCLUDE_DIR - ${CMAKE_CURRENT_BINARY_DIR}/brotli_ep-prefix/src/brotli_ep/c/include/brotli) - add_custom_command(TARGET brotli_ep - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${BROTLI_BUILD_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}*${CMAKE_STATIC_LIBRARY_SUFFIX} - ${BROTLI_LIB_DIR} - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${BROTLI_BUILD_INCLUDE_DIR} ${BROTLI_INCLUDE_DIR}/brotli) - endif() + STEP_TARGETS headers_copy) file(MAKE_DIRECTORY "${BROTLI_INCLUDE_DIR}") diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 1cdffde80c..b3b7ce50a7 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -58,8 +58,8 @@ ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 ARROW_BOOST_BUILD_VERSION=1.92.0 ARROW_BOOST_BUILD_SHA256_CHECKSUM=f51707c27359a0df0cac1beada86de31bb5eed5e8285592dadec384df99c2984 -ARROW_BROTLI_BUILD_VERSION=v1.0.9 -ARROW_BROTLI_BUILD_SHA256_CHECKSUM=f9e8d81d0405ba66d181529af42a3354f838c939095ff99930da6aa9cdf6fe46 +ARROW_BROTLI_BUILD_VERSION=v1.2.1 +ARROW_BROTLI_BUILD_SHA256_CHECKSUM=98d35d11a78dc453aebb7d10c3748f3f73ff78c8267c0dc697b8631f67bf7e8f ARROW_BZIP2_BUILD_VERSION=1.0.8 ARROW_BZIP2_BUILD_SHA256_CHECKSUM=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 ARROW_CARES_BUILD_VERSION=1.34.6 diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index f94602e9cc..b4f75517af 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -59,5 +59,5 @@ "zstd" ], "$comment": "We can update builtin-baseline by 'vcpkg x-update-baseline'", - "builtin-baseline": "9b965a116838c6cdcd36bca60d1b81b030c8ab8d" + "builtin-baseline": "61d7eaeb094a6acbc23d9c2e54de9208175fda47" } diff --git a/r/configure.win b/r/configure.win index f33d965809..be9d1044d1 100755 --- a/r/configure.win +++ b/r/configure.win @@ -60,7 +60,6 @@ function configure_binaries() { OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" - BROTLI_LIBS="-lbrotlienc -lbrotlidec -lbrotlicommon" # Common goes last since dec and enc depend on it # We build aws-sdk-cpp bundled now, so the AWS libs are included in arrow_bundled_dependencies # but we also need to include these Windows system libraries AWS_LIBS="-luserenv -lversion -lws2_32 -lbcrypt -lwininet -lwinhttp -lsecur32 -lshlwapi -lncrypt" @@ -90,7 +89,7 @@ function configure_binaries() { PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) ' PKG_LIBS="$PKG_LIBS -L${RWINLIB}/lib"'$(R_ARCH)$(CRT) ' PKG_LIBS="$PKG_LIBS -larrow_dataset -larrow_acero -lparquet -larrow_compute -larrow_s3 -larrow -larrow_bundled_dependencies \ - -lutf8proc -lsnappy -lz -lzstd -llz4 -lbz2 ${BROTLI_LIBS} -lole32 \ + -lutf8proc -lsnappy -lz -lzstd -llz4 -lbz2 -lole32 \ ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS} }
