This is an automated email from the ASF dual-hosted git repository. jonkeane pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new 41c4143 ARROW-13199: [R] add ubuntu 21.04 to nightly builds 41c4143 is described below commit 41c4143992905cc85eb61a417cf9460c6db6b4df Author: Jonathan Keane <jke...@gmail.com> AuthorDate: Mon Jul 5 14:58:49 2021 -0500 ARROW-13199: [R] add ubuntu 21.04 to nightly builds Closes #10611 from jonkeane/ARROW-13199-r-ubuntu-21.04 Authored-by: Jonathan Keane <jke...@gmail.com> Signed-off-by: Jonathan Keane <jke...@gmail.com> --- .env | 1 + ci/docker/linux-apt-r.dockerfile | 10 ++ ci/docker/ubuntu-21.04-cpp.dockerfile | 160 ++++++++++++++++++++++++++++++++ dev/tasks/docker-tests/github.linux.yml | 8 +- dev/tasks/tasks.yml | 22 +++++ docker-compose.yml | 10 ++ r/configure | 7 +- r/tools/nixlibs.R | 5 + 8 files changed, 219 insertions(+), 4 deletions(-) diff --git a/.env b/.env index e06c6e5..579895e 100644 --- a/.env +++ b/.env @@ -61,6 +61,7 @@ SPARK=master DOTNET=3.1 R=4.1 ARROW_R_DEV=TRUE +GCC_VERSION="" # These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest R_ORG=rhub R_IMAGE=ubuntu-gcc-release diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 7b78da9..36f4fb2 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -61,6 +61,16 @@ RUN apt-get update -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +ARG gcc_version="" +RUN if [ "${gcc_version}" != "" ]; then \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 && \ + update-alternatives --set cc /usr/bin/gcc && \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 && \ + update-alternatives --set c++ /usr/bin/g++; \ + fi + # Ensure parallel R package installation, set CRAN repo mirror, # and use pre-built binaries where possible COPY ci/etc/rprofile /arrow/ci/etc/ diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile new file mode 100644 index 0000000..18c3778 --- /dev/null +++ b/ci/docker/ubuntu-21.04-cpp.dockerfile @@ -0,0 +1,160 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:20.04 +FROM ${base} +ARG arch + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/hirsute/ llvm-toolchain-hirsute-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/hirsute/ llvm-toolchain-hirsute-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + gdb \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + rapidjson-dev \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 + +ARG gcc_version="" +RUN if [ "${gcc_version}" = "" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + g++ \ + gcc; \ + else \ + if [ "${gcc_version}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends software-properties-common && \ + add-apt-repository ppa:ubuntu-toolchain-r/volatile; \ + fi; \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + g++-${gcc_version} \ + gcc-${gcc_version} && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \ + update-alternatives --set cc /usr/bin/gcc && \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \ + update-alternatives --set c++ /usr/bin/g++; \ + fi \ No newline at end of file diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index 06837a8..dd347b8 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -39,7 +39,13 @@ jobs: - name: Execute Docker Build shell: bash - run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }} + run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }} {{ command|default("") }} + + {% if '-r-' in image %} + - name: Dump R install logs + run: cat arrow/r/check/arrow.Rcheck/00install.out + continue-on-error: true + {% endif %} {% if arrow.branch == 'master' %} {{ macros.github_login_dockerhub()|indent }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index e908a63..7aea18f 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -934,6 +934,28 @@ tasks: r_tag: {{ r_tag }} {% endfor %} + test-r-ubuntu-21.04: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 21.04 + CLANG_TOOLS: 9 # can remove this when >=9 is the default + flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE' + image: ubuntu-r + + test-r-gcc-11: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 21.04 + CLANG_TOOLS: 9 # can remove this when >=9 is the default + GCC_VERSION: 11 + # S3 support is not buildable with gcc11 right now + flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE -e ARROW_S3=OFF' + image: ubuntu-r-only-r + test-r-rstudio-r-base-3.6-centos7-devtoolset-8: ci: azure template: r/azure.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index f65609a..6b435e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,6 +110,7 @@ x-hierarchy: - ubuntu-docs - ubuntu-python-sdist-test - ubuntu-r + - ubuntu-r-only-r - ubuntu-cuda-cpp: - ubuntu-cuda-python - ubuntu-csharp @@ -300,6 +301,7 @@ services: base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} + gcc_version: ${GCC_VERSION} shm_size: *shm-size ulimits: *ulimits environment: @@ -1008,12 +1010,14 @@ services: arch: ${ARCH} r: ${R} base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + gcc_version: ${GCC_VERSION} shm_size: *shm-size environment: <<: *ccache ARROW_R_CXXFLAGS: '-Werror' LIBARROW_BUILD: 'false' NOT_CRAN: 'true' + ARROW_R_DEV: ${ARROW_R_DEV} volumes: *ubuntu-volumes command: > /bin/bash -c " @@ -1021,6 +1025,12 @@ services: /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/r_test.sh /arrow" + ubuntu-r-only-r: + extends: ubuntu-r + command: > + /bin/bash -c " + /arrow/ci/scripts/r_test.sh /arrow" + r: # This lets you test building/installing the arrow R package # (including building the C++ library) on any Docker image that contains R diff --git a/r/configure b/r/configure index aa7e7a8..22a34bc 100755 --- a/r/configure +++ b/r/configure @@ -69,7 +69,7 @@ if [ "$FORCE_AUTOBREW" = "true" ] || [ "$FORCE_BUNDLED_BUILD" = "true" ]; then fi # Note that cflags may be empty in case of success -if [ "$ARROW_HOME" ]; then +if [ "$ARROW_HOME" ] && [ "$FORCE_BUNDLED_BUILD" != "true" ]; then echo "*** Using ARROW_HOME as the source of libarrow" PKG_CFLAGS="-I$ARROW_HOME/include $PKG_CFLAGS" PKG_DIRS="-L$ARROW_HOME/lib" @@ -88,7 +88,8 @@ else fi if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then - echo "*** Arrow C++ libraries found via pkg-config" + FOUND_LIB_DIR=`echo $PKG_DIRS | sed -e 's/^-L//'` + echo "*** Arrow C++ libraries found via pkg-config at $FOUND_LIB_DIR" PKG_CFLAGS="$PKGCONFIG_CFLAGS" PKG_LIBS=${PKGCONFIG_LIBS} PKG_DIRS=${PKGCONFIG_DIRS} @@ -244,7 +245,7 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then fi fi # prepend PKG_DIRS and append BUNDLED_LIBS to PKG_LIBS - PKG_LIBS="$PKG_DIRS $PKG_LIBS $BUNDLED_LIBS" + PKG_LIBS="$PKG_DIRS $PKG_LIBS $BUNDLED_LIBS -fno-lto" echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" else diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 97cc69f..f638b7b 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -326,6 +326,7 @@ build_libarrow <- function(src_dir, dst_dir) { CC = R_CMD_config("CC"), CXX = paste(R_CMD_config("CXX11"), R_CMD_config("CXX11STD")), # CXXFLAGS = R_CMD_config("CXX11FLAGS"), # We don't want the same debug symbols + ARROW_R_CXXFLAGS = paste(Sys.getenv("ARROW_R_CXXFLAGS", ""), "-fno-lto"), LDFLAGS = R_CMD_config("LDFLAGS") ) env_vars <- paste0(names(env_var_list), '="', env_var_list, '"', collapse = " ") @@ -415,6 +416,10 @@ cmake_version <- function(cmd = "cmake") { with_s3_support <- function(env_vars) { arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" || tolower(Sys.getenv("LIBARROW_MINIMAL")) == "false" + # but if ARROW_S3=OFF explicitly, we are definitely off, so override + if (toupper(Sys.getenv("ARROW_S3")) == "OFF" ) { + arrow_s3 <- FALSE + } if (arrow_s3) { # User wants S3 support. If they're using gcc, let's make sure the version is >= 4.9 # and make sure that we have curl and openssl system libs