This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 70c67086c0 ARROW-12590: [C++][R] Update copies of Homebrew files to
reflect recent updates (#13769)
70c67086c0 is described below
commit 70c67086c01bb1b64c19e3f46382574bd554dcf1
Author: Jacob Wujciak-Jens <[email protected]>
AuthorDate: Thu Aug 18 01:50:32 2022 +0200
ARROW-12590: [C++][R] Update copies of Homebrew files to reflect recent
updates (#13769)
Building on #12157
Lead-authored-by: Jacob Wujciak-Jens <[email protected]>
Co-authored-by: Jonathan Keane <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
LICENSE.txt | 8 +--
.../{apache-arrow.rb => apache-arrow-static.rb} | 36 +++++++------
.../homebrew-formulae/autobrew/apache-arrow.rb | 14 +++--
dev/tasks/macros.jinja | 33 ++++++++----
dev/tasks/r/github.macos.autobrew.yml | 62 ++++++++++++++--------
dev/tasks/r/github.packages.yml | 7 +--
r/tools/autobrew | 18 ++++++-
7 files changed, 112 insertions(+), 66 deletions(-)
diff --git a/LICENSE.txt b/LICENSE.txt
index 843cf4f6a5..a82c22aece 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1990,12 +1990,14 @@ for PyArrow. Ibis is released under the Apache License,
Version 2.0.
This project includes code from the autobrew project.
-* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
- are based on code from the autobrew project.
+The following files are based on code from the autobrew project:
+* r/tools/autobrew
+* dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+* dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb
Copyright (c) 2019, Jeroen Ooms
License: MIT
-Homepage: https://github.com/jeroen/autobrew
+Homepage: https://github.com/autobrew/
--------------------------------------------------------------------------------
diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
b/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb
similarity index 67%
copy from dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
copy to dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb
index a4491a4cc2..79d1da2c21 100644
--- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb
@@ -15,32 +15,41 @@
# specific language governing permissions and limitations
# under the License.
-# https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb
-class ApacheArrow < Formula
+# Cloned from
https://github.com/autobrew/homebrew-core/blob/high-sierra/Formula/apache-arrow.rb
+# Upstream any relevant changes (dependencies, build args ...)
+
+class ApacheArrowStatic < Formula
desc "Columnar in-memory analytics layer designed to accelerate big data"
homepage "https://arrow.apache.org/";
- url
"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-9.0.0.9000/apache-arrow-9.0.0.9000.tar.gz";
+ url
"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-8.0.0.9000/apache-arrow-8.0.0.9000.tar.gz";
+ # Uncomment and update to test on a release candidate
+ # mirror
"https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-8.0.0-rc1/apache-arrow-8.0.0.tar.gz";
sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28"
head "https://github.com/apache/arrow.git";
bottle do
- cellar :any
- sha256 "a55211ba6f464681b7ca1b48defdad9cfbe1cf6fad8ff9ec875dc5a3c8f3c5ed"
=> :el_capitan_or_later
- root_url "https://autobrew.github.io/bottles";
+ sha256 cellar: :any, arm64_big_sur:
"ef89d21a110b89840cc6148add685d407e75bd633bc8f79625eb33d00e3694b4"
+ sha256 cellar: :any, big_sur:
"6fcb9f55d44eb61d295a8020e039a0622bdc044b220cfffef855f3e3ab8057a1"
+ sha256 cellar: :any, catalina:
"bf71b17766688077fb9b4e679f309742c16524015287dd3633758c679752c977"
end
- # NOTE: if you add something here, be sure to add to PKG_LIBS in
r/tools/autobrew
depends_on "boost" => :build
- depends_on "brotli"
depends_on "cmake" => :build
- depends_on "aws-sdk-cpp"
+ depends_on "aws-sdk-cpp-static"
+ depends_on "brotli"
depends_on "lz4"
depends_on "snappy"
depends_on "thrift"
depends_on "zstd"
+ conflicts_with "apache-arrow", because: "both install Arrow"
+
def install
ENV.cxx11
+ # https://github.com/Homebrew/homebrew-core/issues/94724
+ # https://issues.apache.org/jira/browse/ARROW-15664
+ ENV["HOMEBREW_OPTIMIZATION_LEVEL"] = "O2"
+
args = %W[
-DARROW_BUILD_SHARED=OFF
-DARROW_BUILD_UTILITIES=ON
@@ -49,14 +58,11 @@ class ApacheArrow < Formula
-DARROW_DATASET=ON
-DARROW_FILESYSTEM=ON
-DARROW_GCS=ON
- -DARROW_HDFS=OFF
-DARROW_JEMALLOC=ON
-DARROW_JSON=ON
-DARROW_MIMALLOC=ON
-DARROW_PARQUET=ON
- -DARROW_PYTHON=OFF
-DARROW_S3=ON
- -DARROW_USE_GLOG=OFF
-DARROW_VERBOSE_THIRDPARTY_BUILD=ON
-DARROW_WITH_BROTLI=ON
-DARROW_WITH_BZ2=ON
@@ -64,9 +70,8 @@ class ApacheArrow < Formula
-DARROW_WITH_SNAPPY=ON
-DARROW_WITH_ZLIB=ON
-DARROW_WITH_ZSTD=ON
- -DCMAKE_UNITY_BUILD=OFF
- -DPARQUET_BUILD_EXECUTABLES=ON
-DLZ4_HOME=#{Formula["lz4"].prefix}
+ -DPARQUET_BUILD_EXECUTABLES=ON
-DTHRIFT_HOME=#{Formula["thrift"].prefix}
]
@@ -86,7 +91,8 @@ class ApacheArrow < Formula
return 0;
}
EOS
- system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}",
"-larrow", "-o", "test"
+ system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \
+ "-larrow", "-larrow_bundled_dependencies", "-o", "test"
system "./test"
end
end
diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
index a4491a4cc2..de0c65dae4 100644
--- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
@@ -16,6 +16,7 @@
# under the License.
# https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb
+
class ApacheArrow < Formula
desc "Columnar in-memory analytics layer designed to accelerate big data"
homepage "https://arrow.apache.org/";
@@ -25,15 +26,15 @@ class ApacheArrow < Formula
bottle do
cellar :any
- sha256 "a55211ba6f464681b7ca1b48defdad9cfbe1cf6fad8ff9ec875dc5a3c8f3c5ed"
=> :el_capitan_or_later
+ sha256 "9cd44700798638b5e3ee8774b3929f3fad815290d05572d1f39f01d6423eaad0"
=> :high_sierra
root_url "https://autobrew.github.io/bottles";
end
# NOTE: if you add something here, be sure to add to PKG_LIBS in
r/tools/autobrew
depends_on "boost" => :build
- depends_on "brotli"
depends_on "cmake" => :build
depends_on "aws-sdk-cpp"
+ depends_on "brotli"
depends_on "lz4"
depends_on "snappy"
depends_on "thrift"
@@ -49,14 +50,11 @@ class ApacheArrow < Formula
-DARROW_DATASET=ON
-DARROW_FILESYSTEM=ON
-DARROW_GCS=ON
- -DARROW_HDFS=OFF
-DARROW_JEMALLOC=ON
-DARROW_JSON=ON
-DARROW_MIMALLOC=ON
-DARROW_PARQUET=ON
- -DARROW_PYTHON=OFF
-DARROW_S3=ON
- -DARROW_USE_GLOG=OFF
-DARROW_VERBOSE_THIRDPARTY_BUILD=ON
-DARROW_WITH_BROTLI=ON
-DARROW_WITH_BZ2=ON
@@ -64,9 +62,8 @@ class ApacheArrow < Formula
-DARROW_WITH_SNAPPY=ON
-DARROW_WITH_ZLIB=ON
-DARROW_WITH_ZSTD=ON
- -DCMAKE_UNITY_BUILD=OFF
- -DPARQUET_BUILD_EXECUTABLES=ON
-DLZ4_HOME=#{Formula["lz4"].prefix}
+ -DPARQUET_BUILD_EXECUTABLES=ON
-DTHRIFT_HOME=#{Formula["thrift"].prefix}
]
@@ -86,7 +83,8 @@ class ApacheArrow < Formula
return 0;
}
EOS
- system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}",
"-larrow", "-o", "test"
+ system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \
+ "-larrow", "-larrow_bundled_dependencies", "-o", "test"
system "./test"
end
end
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index c7f7c647b7..bbabad69a7 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -226,6 +226,21 @@ on:
done
{% endmacro %}
+{%- macro pin_brew_formulae(is_fork = false) -%}
+{#- This macro expects the cwd to be arrow/r -#}
+ # Copy all *brew formulae
+ cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow*.rb tools/
+
+ # Pin the git commit in the formula to match
+ cd tools
+ if [ "{{ is_fork }}" == "true" ]; then
+ sed -i.bak -E -e 's/apache\/arrow.git"$/{{
arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{
arrow.head }}"'"/' apache-arrow*.rb
+ else
+ sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{
arrow.head }}"'"/' apache-arrow*.rb
+ fi
+ rm -f apache-arrow*.rb.bak
+{% endmacro %}
+
{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date
+%Y%m%d)\"\'' ) -%}
- name: Modify version
shell: bash
@@ -236,16 +251,8 @@ on:
DESCRIPTION
head DESCRIPTION
rm -f DESCRIPTION.bak
- cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
tools/apache-arrow.rb
- # Pin the git commit in the formula to match
- cd tools
- if [ "{{ is_fork }}" == "true" ]; then
- sed -i.bak -E -e 's/apache\/arrow.git"$/{{
arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{
arrow.head }}"'"/' apache-arrow.rb
- else
- sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{
arrow.head }}"'"/' apache-arrow.rb
- fi
- rm -f apache-arrow.rb.bak
+ {{ pin_brew_formulae(is_fork) }}
{% endmacro %}
{%- macro github_test_r_src_pkg() -%}
@@ -309,3 +316,11 @@ on:
# Set envvar for later steps by appending to $GITHUB_ENV
write(paste0("R_PROFILE_USER=", profile_path), file =
Sys.getenv("GITHUB_ENV"), append = TRUE)
{% endmacro %}
+
+{# Detect if we are using a fork or the upstream repo #}
+ {% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}
+ {# use filter to cast to string and convert to lowercase to match yaml
boolean #}
+ {% set is_fork = (not is_upstream_b)|lower %}
+
+{% set r_release = '4.2' %}
+{% set r_oldrel = '4.1' %}
diff --git a/dev/tasks/r/github.macos.autobrew.yml
b/dev/tasks/r/github.macos.autobrew.yml
index c6c56277de..1633682a43 100644
--- a/dev/tasks/r/github.macos.autobrew.yml
+++ b/dev/tasks/r/github.macos.autobrew.yml
@@ -16,44 +16,60 @@
# under the License.
{% import 'macros.jinja' as macros with context %}
+{% set is_fork = macros.is_fork %}
{{ macros.github_header() }}
jobs:
autobrew:
- name: "Autobrew"
- runs-on: macOS-10.15
+ name: "Autobrew {{ "${{ matrix.platform }}" }}"
+ runs-on: {{ "${{ matrix.platform }}" }}
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - macos-11
+ - macos-10.13 # self-hosted
+ r-version:
+ - {{ macros.r_release }}
+ - {{ macros.r_oldrel }}
steps:
{{ macros.github_checkout_arrow()|indent }}
-
- name: Configure autobrew script
run: |
+ # minio is pre-installed on the self-hosted 10.13 runer
+ if [ {{ '${{ matrix.platform }}' }} != macos-10.13 ]; then
+ # install minio for tests
+ brew install minio
+ fi
cd arrow/r
- # Put the formula inside r/ so that it's included in the package
build
- cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
tools/apache-arrow.rb
- # Pin the current commit in the formula to test so that we're not
always pulling from master
- sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{
arrow.remote }}.git", :revision => "{{ arrow.head }}"@' tools/apache-arrow.rb
&& rm -f tools/apache-arrow.rb.bak
- # Sometimes crossbow gives a remote URL with .git and sometimes not.
Make sure there's only one
- sed -i.bak -E -e '[email protected]@.git@' tools/apache-arrow.rb && rm
-f tools/apache-arrow.rb.bak
- # Get minio for S3 testing
- brew install minio
- - uses: r-lib/actions/setup-r@v1
- - name: Install dependencies
- run: |
- install.packages("remotes")
- remotes::install_deps("arrow/r", dependencies = TRUE)
- remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo"))
- shell: Rscript {0}
- - name: Session info
+ {{ macros.pin_brew_formulae(is_fork)|indent }}
+ - uses: r-lib/actions/setup-r@v2
+ if: matrix.platform != 'macos-10.13'
+ with:
+ r-version: {{ '${{ matrix.r-version }}' }}
+ use-public-rspm: true
+ - name: Setup R
+ if: matrix.platform == 'macos-10.13'
run: |
- options(width = 100)
- pkgs <- installed.packages()[, "Package"]
- sessioninfo::session_info(pkgs, include_base = TRUE)
- shell: Rscript {0}
+ # rig is a system utility that allows for switching
+ # between pre-installed R version on the self-hosted runners
+ rig default {{ '${{ matrix.r-version }}' }}
+ rig system setup-user-lib
+ rig system add-pak
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ working-directory: 'arrow/r'
+ extra-packages: |
+ any::rcmdcheck
+ any::sys
+ any::readr
- name: Check
env:
_R_CHECK_CRAN_INCOMING_: false
+ NOT_CRAN: true
ARROW_USE_PKG_CONFIG: false
+ ARROW_R_DEV: true
run: arrow/ci/scripts/r_test.sh arrow
- name: Dump install logs
run: cat arrow/r/check/arrow.Rcheck/00install.out
diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml
index 6e46b3ff43..7ca8bd7a5f 100644
--- a/dev/tasks/r/github.packages.yml
+++ b/dev/tasks/r/github.packages.yml
@@ -21,12 +21,7 @@
# crossbow submit --param custom_version=8.5.3 r-binary-packages
# if the param is unset defaults to the usual Ymd naming scheme
{% set package_version = custom_version|default("\\2.\'\"$(date +%Y%m%d)\"\'")
%}
-# We need this as boolean and string
-{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}
-# use filter to cast to string and convert to lowercase to match yaml boolean
-{% set is_fork = (not is_upstream_b)|lower %}
-{% set is_upstream = is_upstream_b|lower %}
-
+{% set is_fork = macros.is_fork %}
{{ macros.github_header() }}
diff --git a/r/tools/autobrew b/r/tools/autobrew
index ea46be2c0d..55fe957608 100644
--- a/r/tools/autobrew
+++ b/r/tools/autobrew
@@ -19,8 +19,15 @@
export HOMEBREW_NO_ANALYTICS=1
export HOMEBREW_NO_AUTO_UPDATE=1
-# Official Homebrew no longer supports El-Capitan
-UPSTREAM_ORG="autobrew"
+if [[ ${OSTYPE:6} -ge 20 ]]; then
+ # We are on a modern enough macOS, we can use the real brew
+ UPSTREAM_ORG="homebrew"
+ PKG_BREW_NAME="$PKG_BREW_NAME-static"
+else
+ # Official Homebrew no longer supports El-Capitan
+ # so we need to use the forked autobrew version of brew that supports old
macOSes
+ UPSTREAM_ORG="autobrew"
+fi
if [ "$DISABLE_AUTOBREW" ]; then return 0; fi
AUTOBREW=${TMPDIR-/tmp}
@@ -36,6 +43,13 @@ curl -fsSL
https://github.com/$UPSTREAM_ORG/brew/tarball/master | tar xz --strip
export HOMEBREW_CACHE="$AUTOBREW"
LOCAL_FORMULA="tools/${PKG_BREW_NAME}.rb"
if [ -f "$LOCAL_FORMULA" ]; then
+ if [[ ${OSTYPE:6} -ge 20 ]]; then
+ $BREW tap
+
+ # Tap https://github.com/autobrew/homebrew-cran so that we can get
dependencies from there
+ $BREW tap autobrew/cran
+ fi
+
# Use the local brew formula and install --HEAD
$BREW deps -n "$LOCAL_FORMULA" 2>/dev/null
BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null)
