marberts commented on code in PR #49553: URL: https://github.com/apache/arrow/pull/49553#discussion_r3456511372
########## dev/tasks/r/github.packages.yml: ########## @@ -1,435 +1,437 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{% set is_fork = macros.is_fork %} - -{{ macros.github_header() }} - -jobs: - source: - # This job will change the version to either the custom_version param or YMD format. - # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) - name: Source Package - runs-on: ubuntu-latest - outputs: - pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - - name: Save Version - id: save-version - shell: bash - run: | - echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT - - - uses: r-lib/actions/setup-r@v2 - with: - install-r: true - - - name: Build R source package - shell: bash - run: | - cd arrow/r - # Copy in the Arrow C++ source - make sync-cpp - R CMD build --no-build-vignettes . - - - name: Upload package artifact - uses: actions/upload-artifact@v4 - with: - name: r-pkg__src__contrib - path: arrow/r/arrow_*.tar.gz - - macos-cpp: - name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - needs: source - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: macos-15-intel, arch: "x86_64" } - - { runs_on: macos-14, arch: "arm64" } - env: - PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} - PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - {{ macros.github_checkout_arrow(action_v="3")|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - - name: Install Deps - run: | - brew install sccache ninja - brew install [email protected] - - name: Build libarrow - shell: bash - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - MACOSX_DEPLOYMENT_TARGET: "11.6" - ARROW_GCS: ON - ARROW_DEPENDENCY_SOURCE: BUNDLED - CMAKE_GENERATOR: Ninja - LIBARROW_MINIMAL: false - run: | - sccache --start-server - export EXTRA_CMAKE_FLAGS="-DOPENSSL_ROOT_DIR=$(brew --prefix [email protected])" - cd arrow - r/inst/build_arrow_static.sh - - name: Bundle libarrow - shell: bash - run: | - cd arrow/r/libarrow/dist - zip -r $PKG_FILE lib/ include/ - - name: Create Checksum - shell: bash - run: | - cd arrow/r/libarrow/dist - shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../../../../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - linux-cpp: - name: C++ Binary Linux {{ '${{ matrix.arch }}' }} - runs-on: {{ '${{ matrix.runs-on }}' }} - needs: source - strategy: - fail-fast: false - matrix: - include: - - arch: x86_64 - runs-on: ubuntu-latest - ubuntu: "22.04" - - arch: arm64 - runs-on: ubuntu-24.04-arm - ubuntu: "22.04" - env: - PKG_ID: r-libarrow-linux-{{ '${{ matrix.arch }}' }} - PKG_FILE: r-libarrow-linux-{{ '${{ matrix.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - {{ macros.github_install_archery()|indent }} - - name: Build libarrow - shell: bash - env: - ARCH: {{ "${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64/v8' }}" }} - UBUNTU: {{ '${{ matrix.ubuntu }}' }} - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: | - source arrow/ci/scripts/util_enable_core_dumps.sh - archery docker run ubuntu-cpp-static - - name: Bundle libarrow - shell: bash - run: | - # These files were created by the docker user so we have to chown them - sudo chown -R $USER:$USER arrow/r/libarrow - - cd arrow/r/libarrow/dist - zip -r $PKG_FILE lib/ include/ - - name: Create Checksum - shell: bash - run: | - cd arrow/r/libarrow/dist - shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../../../../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - windows-cpp: - name: C++ Binary Windows RTools (40 only) - needs: source - runs-on: windows-latest - env: - PKG_ID: r-libarrow-windows-x86_64 - PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - - run: git config --global core.autocrlf false - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - - uses: r-lib/actions/setup-r@v2 - with: - rtools-version: 40 - r-version: "4.0" - Ncpus: 2 - - name: Install sccache - shell: bash - run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache - - name: Build Arrow C++ with rtools40 - shell: bash - env: - ARROW_HOME: "arrow" - MINGW_ARCH: "ucrt64" - RTOOLS_VERSION: 40 - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: arrow/ci/scripts/r_windows_build.sh - - name: Create Checksum - shell: bash - run: | - cd build - sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - r-packages: - needs: [source, windows-cpp, macos-cpp] - name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: 'windows-latest', name: "Windows"} - - { runs_on: macos-15-intel, name: "macOS x86_64"} - - { runs_on: macos-14, name: "macOS arm64" } - r_version: [oldrel, release] - steps: - - uses: r-lib/actions/setup-r@v2 - with: - r-version: {{ '${{ matrix.r_version }}' }} - {{ macros.github_setup_local_r_repo(false, true, true)|indent }} - - name: Prepare Dependency Installation - shell: bash - run: | - tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION - - name: Install dependencies - uses: r-lib/actions/setup-r-dependencies@v2 - with: - working-directory: 'arrow' - extra-packages: cpp11 - - name: Set CRAN like openssl - if: contains(matrix.platform.name, 'macOS') - # The -E forwards the GITHUB_* envvars - shell: sudo -E Rscript {0} - run: | - # get the mac-recipes version of openssl from CRAN - source("https://mac.R-project.org/bin/install.R";) - install.libs("openssl") - - # override our cmakes default setting of the brew --prefix as root dir to avoid version conflicts. - if (Sys.info()[["machine"]] == "arm64"){ - cat("OPENSSL_ROOT_DIR=/opt/R/arm64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) - } else { - cat("OPENSSL_ROOT_DIR=/opt/R/x86_64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) - } - - name: Build Binary - id: build - shell: Rscript {0} - env: - NOT_CRAN: "false" # actions/setup-r sets this implicitly - ARROW_R_DEV: "true" - LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated - LIBARROW_BUILD: "false" - ARROW_R_ENFORCE_CHECKSUM: "true" - ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" - run: | - on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" - - # Build - Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) - INSTALL_opts <- "--build" - if (!on_windows) { - # Windows doesn't support the --strip arg - INSTALL_opts <- c(INSTALL_opts, "--strip") - } - - # always remove arrow (mainly for self-hosted runners) - try(remove.packages("arrow"), silent = TRUE) - - cat("Install arrow from dev repo.\n") - install.packages( - "arrow", - type = "source", - # The sub is necessary to prevent an error on windows. - repos = sub("file://", "file:", getOption("arrow.dev_repo")),, - INSTALL_opts = INSTALL_opts - ) - - - # Test - library(arrow) - arrow_info() - read_parquet(system.file("v0.7.1.parquet", package = "arrow")) - - # encode contrib.url for artifact name - cmd <- paste0( - "path=", - gsub( - "/", "__", - contrib.url("", type = "binary") - ), - "\n" - ) - cat(cmd, file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) - - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: r-pkg{{ '${{ steps.build.outputs.path }}' }} - path: arrow_* - test-linux-binary: - needs: [source, linux-cpp] - name: Test binary {{ '${{ matrix.config.image }}' }} {{ '${{ matrix.config.runner }}' }} - runs-on: {{ '${{ matrix.config.runner }}' }} - container: {{ '${{ matrix.config.image }}' }} - strategy: - fail-fast: false - matrix: - config: - # If libarrow_binary is unset, we're testing that we're automatically - # choosing a binary on this OS. If libarrow_binary is TRUE, we're on - # an OS that is not in the allowlist, so we have to opt-in to use the - # binary. Other env vars used in r_docker_configure.sh can be added - # here and wired up in the later steps. - # x86_64 tests - - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) - # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-24.04 - - {image: "posit/r-base:4.3-noble", runner: "ubuntu-latest"} - - {image: "posit/r-base:4.4-noble", runner: "ubuntu-latest"} - - {image: "posit/r-base:4.5-noble", runner: "ubuntu-latest"} - # ARM64 tests - - {image: "posit/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} - - {image: "posit/r-base:4.4-noble", runner: "ubuntu-24.04-arm"} - - {image: "posit/r-base:4.5-noble", runner: "ubuntu-24.04-arm"} - steps: - # Get the arrow checkout just for the docker config scripts - # Don't need submodules for this (hence false arg to macro): they fail on - # actions/checkout for some reason in this context - {{ macros.github_checkout_arrow(1, false, '3')|indent }} - - - name: Install system requirements - env: - ARROW_R_DEV: "TRUE" # To install curl/openssl in r_docker_configure.sh - shell: bash - run: | - # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) - echo "${RPREFIX}/bin" >> $GITHUB_PATH - export PATH="${RPREFIX}/bin:${PATH}" - cd arrow && ARROW_SOURCE_HOME=$(pwd) ./ci/scripts/r_docker_configure.sh - {{ macros.github_setup_local_r_repo(true, false)|indent }} - - name: Install arrow from our repo - env: - ARROW_R_DEV: "TRUE" - LIBARROW_BUILD: "FALSE" - LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} - ARROW_R_ENFORCE_CHECKSUM: "true" - ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" - shell: bash - run: | - Rscript -e ' - {{ macros.github_test_r_src_pkg()|indent(8) }} - ' - test-source: - needs: source - name: Test {{ '${{ matrix.platform.name }}' }} source build - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - {runs_on: "ubuntu-latest", name: "Linux"} - - {runs_on: "macos-15-intel" , name: "macOS"} - steps: - - name: Install R - uses: r-lib/actions/setup-r@v2 - {{ macros.github_setup_local_r_repo(false, false)|indent }} - {{ macros.github_checkout_arrow(action_v="3")|indent }} - - name: Install sccache - if: matrix.platform.name == 'Linux' - shell: bash - run: | - arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - - name: Install R package system dependencies (Linux) - if: matrix.platform.name == 'Linux' - run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev - - name: Install R package system dependencies (macOS) - if: matrix.platform.name == 'macOS' - run: brew install sccache openssl curl libuv - - name: Remove arrow/ - run: | - rm -rf arrow/ - - name: Enable parallel build - run: | - cores=`nproc || sysctl -n hw.logicalcpu` - echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV - - name: Install arrow source package - env: - # Test source build so be sure not to download a binary - LIBARROW_BINARY: "FALSE" - {{ macros.github_set_sccache_envvars()|indent(8) }} - shell: Rscript {0} - run: | - {{ macros.github_test_r_src_pkg()|indent(8) }} - - - name: Retry with verbosity if that failed - if: failure() - env: - LIBARROW_BINARY: "FALSE" - ARROW_R_DEV: "TRUE" - CMAKE_FIND_DEBUG_MODE: "ON" - {{ macros.github_set_sccache_envvars()|indent(8) }} - shell: Rscript {0} - run: | - {{ macros.github_test_r_src_pkg()|indent(8) }} - - upload-binaries: - # Only upload binaries if all tests pass. - needs: [r-packages, test-source, test-linux-binary] - name: Upload artifacts - runs-on: ubuntu-latest - steps: - {{ macros.github_checkout_arrow()|indent }} - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Install R - uses: r-lib/actions/setup-r@v2 - with: - install-r: true - - name: Move libarrow artifacts - run: | - mkdir -p binaries/ - mv artifacts/r-libarrow-*/* binaries/ - - name: Rename artifacts - shell: Rscript {0} - run: | - file_paths <- list.files("artifacts", include.dirs = FALSE, recursive = TRUE) - new_names <- file.path("binaries", sub("/", "__", file_paths)) - dir.create("binaries", showWarnings = FALSE) - file.copy(file.path("artifacts", file_paths), new_names) - - {{ macros.github_upload_releases("binaries/r-*")|indent }} +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{% set is_fork = macros.is_fork %} + +{{ macros.github_header() }} + +jobs: + source: + # This job will change the version to either the custom_version param or YMD format. + # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) + name: Source Package + runs-on: ubuntu-latest + outputs: + pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} + - name: Save Version + id: save-version + shell: bash + run: | + echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT + + - uses: r-lib/actions/setup-r@v2 + with: + install-r: true + + - name: Build R source package + shell: bash + run: | + cd arrow/r + # Copy in the Arrow C++ source + make sync-cpp + R CMD build --no-build-vignettes . + + - name: Upload package artifact + uses: actions/upload-artifact@v4 + with: + name: r-pkg__src__contrib + path: arrow/r/arrow_*.tar.gz + + macos-cpp: + name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + needs: source + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-15-intel, arch: "x86_64" } + - { runs_on: macos-14, arch: "arm64" } + env: + PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} + PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + {{ macros.github_checkout_arrow(action_v="3")|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + - name: Install Deps + run: | + brew install sccache ninja + brew install [email protected] + brew install libxml2 + - name: Build libarrow + shell: bash + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} + MACOSX_DEPLOYMENT_TARGET: "11.6" + ARROW_GCS: ON + ARROW_AZURE: ON Review Comment: Second change. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
