This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new 6a5c43e ARROW-5054: [Release][Flight] Test Flight in Linux/macOS release verification scripts 6a5c43e is described below commit 6a5c43ef456cae2400ab85874f0b71f7463c5afa Author: Sutou Kouhei <k...@clear-code.com> AuthorDate: Mon Jun 24 19:49:33 2019 -0500 ARROW-5054: [Release][Flight] Test Flight in Linux/macOS release verification scripts Author: Sutou Kouhei <k...@clear-code.com> Author: Wes McKinney <wesm+...@apache.org> Closes #4613 from wesm/ARROW-5054 and squashes the following commits: a4917cafe <Sutou Kouhei> Add Emacs configuration for editing .sh 487cf64d7 <Sutou Kouhei> Add missing ninja argument 575cfe5cd <Sutou Kouhei> Fix path cb8ea34a4 <Sutou Kouhei> Don't create dummy top-level .git to avoid Java test failure 84bfd311c <Sutou Kouhei> Restore Go test a5a7b766e <Sutou Kouhei> Add support GLib test without configure e73e270cd <Sutou Kouhei> Update the required Node.js version b81ef910c <Sutou Kouhei> Add support for verifying local source 3eaa826a5 <Sutou Kouhei> Use $PWD 49c3bda84 <Sutou Kouhei> Unify verification script again f8e0f95ba <Wes McKinney> Fix C++ unit tests at least 21c87195e <Wes McKinney> Add test-release-verification.sh script to help 3cee0875f <Wes McKinney> Move release verification functions into a separate bash script to make reuse / testing easier --- .dir-locals.el | 20 +++ dev/release/02-source-test.rb | 2 + dev/release/02-source.sh | 6 +- dev/release/post-06-csharp.sh | 2 + dev/release/verify-release-candidate.sh | 230 +++++++++++++++++++------------- python/setup.py | 2 +- 6 files changed, 167 insertions(+), 95 deletions(-) diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 0000000..d0440a7 --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,20 @@ +;;; Licensed to the Apache Software Foundation (ASF) under one +;;; or more contributor license agreements. See the NOTICE file +;;; distributed with this work for additional information +;;; regarding copyright ownership. The ASF licenses this file +;;; to you under the Apache License, Version 2.0 (the +;;; "License"); you may not use this file except in compliance +;;; with the License. You may obtain a copy of the License at +;;; +;;; http://www.apache.org/licenses/LICENSE-2.0 +;;; +;;; Unless required by applicable law or agreed to in writing, +;;; software distributed under the License is distributed on an +;;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +;;; KIND, either express or implied. See the License for the +;;; specific language governing permissions and limitations +;;; under the License. + +((sh-mode . ((indent-tabs-mode . nil) + (sh-indentation . 2) + (sh-basic-offset . 2)))) diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb index 04daeea..e777c75 100644 --- a/dev/release/02-source-test.rb +++ b/dev/release/02-source-test.rb @@ -72,7 +72,9 @@ class SourceTest < Test::Unit::TestCase def test_csharp_git_commit_information source Dir.chdir("#{@tag_name}/csharp") do + FileUtils.mv("dummy.git", "../.git") sh("dotnet", "pack", "-c", "Release") + FileUtils.mv("../.git", "dummy.git") Dir.chdir("artifacts/Apache.Arrow/Release") do sh("unzip", "Apache.Arrow.#{@snapshot_version}.nupkg") FileUtils.chmod(0400, "Apache.Arrow.nuspec") diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh index 1a34055..edaeec9 100755 --- a/dev/release/02-source.sh +++ b/dev/release/02-source.sh @@ -90,11 +90,13 @@ cp -R -L ${tag}.tmp ${tag} rm -rf ${tag}.tmp # Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#. -mkdir ${tag}/.git && cd $_ +dummy_git=${tag}/csharp/dummy.git +mkdir ${dummy_git} +pushd ${dummy_git} echo ${release_hash} > HEAD echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config mkdir objects refs -cd - +popd # Create new tarball from modified source directory tar czf ${tarball} ${tag} diff --git a/dev/release/post-06-csharp.sh b/dev/release/post-06-csharp.sh index 59edc02..af61616 100755 --- a/dev/release/post-06-csharp.sh +++ b/dev/release/post-06-csharp.sh @@ -44,7 +44,9 @@ curl \ rm -rf ${archive_name} tar xf ${tar_gz} pushd ${archive_name}/csharp +mv dummy.git ../.git dotnet pack -c Release +mv ../.git dummy.git for package in artifacts/Apache.Arrow/Release/*.{nupkg,snupkg}; do dotnet nuget push \ ${package} \ diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index f14c99d..d016c87 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -23,7 +23,8 @@ # - Maven >= 3.3.9 # - JDK >=7 # - gcc >= 4.8 -# - nodejs >= 6.0.0 (best way is to use nvm) +# - Node.js >= 11.12 (best way is to use nvm) +# - Go >= 1.11 # # If using a non-system Boost, set BOOST_ROOT and add Boost libraries to # LD_LIBRARY_PATH. @@ -49,8 +50,6 @@ set -o pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" -ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' - detect_cuda() { if ! (which nvcc && which nvidia-smi) > /dev/null; then return 1 @@ -60,10 +59,15 @@ detect_cuda() { return $((${n_gpus} < 1)) } +# Build options for the C++ library + if [ -z "${ARROW_CUDA:-}" ] && detect_cuda; then ARROW_CUDA=ON fi : ${ARROW_CUDA:=OFF} +: ${ARROW_FLIGHT:=ON} + +ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' download_dist_file() { curl \ @@ -191,9 +195,14 @@ test_yum() { done } + setup_tempdir() { cleanup() { - rm -fr "$TMPDIR" + if [ "${TEST_SUCCESS}" = "yes" ]; then + rm -fr "${TMPDIR}" + else + echo "Failed to verify release candidate. See ${TMPDIR} for details." + fi } trap cleanup EXIT TMPDIR=$(mktemp -d -t "$1.XXXXX") @@ -208,7 +217,7 @@ setup_miniconda() { MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh fi - MINICONDA=`pwd`/test-miniconda + MINICONDA=$PWD/test-miniconda wget -O miniconda.sh $MINICONDA_URL bash miniconda.sh -b -p $MINICONDA @@ -226,6 +235,17 @@ setup_miniconda() { conda activate arrow-test } +# Build and test Java (Requires newer Maven -- I used 3.3.9) + +test_package_java() { + pushd java + + mvn test + mvn package + + popd +} + # Build and test C++ test_and_install_cpp() { @@ -236,6 +256,7 @@ test_and_install_cpp() { ${ARROW_CMAKE_OPTIONS} -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib +-DARROW_FLIGHT=${ARROW_FLIGHT} -DARROW_PLASMA=ON -DARROW_ORC=ON -DARROW_PYTHON=ON @@ -249,11 +270,11 @@ ${ARROW_CMAKE_OPTIONS} " cmake $ARROW_CMAKE_OPTIONS .. - make -j$NPROC - make install + make -j$NPROC install - # TODO: ARROW-5036 - ctest \ + # TODO: ARROW-5036: plasma-serialization_tests broken + # TODO: ARROW-5054: libgtest.so link failure in flight-server-test + LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \ --exclude-regex "plasma-serialization_tests" \ -j$NPROC \ --output-on-failure \ @@ -274,6 +295,9 @@ test_python() { if [ "${ARROW_CUDA}" = "ON" ]; then export PYARROW_WITH_CUDA=1 fi + if [ "${ARROW_FLIGHT}" = "ON" ]; then + export PYARROW_WITH_FLIGHT=1 + fi python setup.py build_ext --inplace py.test pyarrow -v --pdb @@ -281,19 +305,22 @@ test_python() { popd } - test_glib() { pushd c_glib if brew --prefix libffi > /dev/null 2>&1; then - ./configure --prefix=$ARROW_HOME \ - PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH - else - ./configure --prefix=$ARROW_HOME + PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH fi - make -j$NPROC - make install + if [ -f configure ]; then + ./configure --prefix=$ARROW_HOME + make -j$NPROC + make install + else + meson build --prefix=$ARROW_HOME --libdir=lib + ninja -C build + ninja -C build install + fi export GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0:$GI_TYPELIB_PATH @@ -343,10 +370,19 @@ test_ruby() { popd } +test_go() { + pushd go/arrow + + go get -v ./... + go test ./... + + popd +} + test_rust() { # install rust toolchain in a similar fashion like test-miniconda - export RUSTUP_HOME=`pwd`/test-rustup - export CARGO_HOME=`pwd`/test-rustup + export RUSTUP_HOME=$PWD/test-rustup + export CARGO_HOME=$PWD/test-rustup curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path @@ -370,85 +406,38 @@ test_rust() { popd } -test_go() { - pushd go - - go get -v ./... - go test ./... - - popd -} - -# Build and test Java (Requires newer Maven -- I used 3.3.9) - -test_package_java() { - pushd java - - mvn test - mvn package - - popd -} - # Run integration tests test_integration() { - JAVA_DIR=`pwd`/java - CPP_BUILD_DIR=`pwd`/cpp/build + JAVA_DIR=$PWD/java + CPP_BUILD_DIR=$PWD/cpp/build export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release pushd integration - python integration_test.py - - popd -} - -setup_tempdir "arrow-$VERSION" -echo "Working in sandbox $TMPDIR" -cd $TMPDIR + INTEGRATION_TEST_ARGS= -export ARROW_HOME=$TMPDIR/install -export PARQUET_HOME=$TMPDIR/install -export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH -export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH - -if [ "$(uname)" == "Darwin" ]; then - NPROC=$(sysctl -n hw.ncpu) -else - NPROC=$(nproc) -fi + if [ "${ARROW_FLIGHT}" = "ON" ]; then + INTEGRATION_TEST_ARGS=--run_flight + fi -import_gpg_keys + python integration_test.py $INTEGRATION_TEST_ARGS -# By default test all functionalities. -# To deactivate one test, deactivate the test and all of its dependents -# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1 -: ${TEST_DEFAULT:=1} -: ${TEST_JAVA:=${TEST_DEFAULT}} -: ${TEST_CPP:=${TEST_DEFAULT}} -: ${TEST_GLIB:=${TEST_DEFAULT}} -: ${TEST_RUBY:=${TEST_DEFAULT}} -: ${TEST_PYTHON:=${TEST_DEFAULT}} -: ${TEST_JS:=${TEST_DEFAULT}} -: ${TEST_INTEGRATION:=${TEST_DEFAULT}} -: ${TEST_GO:=${TEST_DEFAULT}} -: ${TEST_RUST:=${TEST_DEFAULT}} -: ${TEST_BINARY:=${TEST_DEFAULT}} -: ${TEST_APT:=${TEST_DEFAULT}} -: ${TEST_YUM:=${TEST_DEFAULT}} + popd +} -# Automatically test if its activated by a dependent -TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY})) -TEST_PYTHON=$((${TEST_PYTHON} + ${TEST_INTEGRATION})) -TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON})) -TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION})) -TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION})) +test_source_distribution() { + export ARROW_HOME=$TMPDIR/install + export PARQUET_HOME=$TMPDIR/install + export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH + export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH -if [ "$ARTIFACT" == "source" ]; then - TARBALL=apache-arrow-$1.tar.gz - DIST_NAME="apache-arrow-${VERSION}" + if [ "$(uname)" == "Darwin" ]; then + NPROC=$(sysctl -n hw.ncpu) + else + NPROC=$(nproc) + fi git clone https://github.com/apache/arrow-testing.git export ARROW_TEST_DATA=$PWD/arrow-testing/data @@ -456,10 +445,6 @@ if [ "$ARTIFACT" == "source" ]; then git clone https://github.com/apache/parquet-testing.git export PARQUET_TEST_DATA=$PWD/parquet-testing/data - fetch_archive $DIST_NAME - tar xvzf ${DIST_NAME}.tar.gz - cd ${DIST_NAME} - if [ ${TEST_JAVA} -gt 0 ]; then test_package_java fi @@ -479,16 +464,18 @@ if [ "$ARTIFACT" == "source" ]; then if [ ${TEST_JS} -gt 0 ]; then test_js fi - if [ ${TEST_INTEGRATION} -gt 0 ]; then - test_integration - fi if [ ${TEST_GO} -gt 0 ]; then test_go fi if [ ${TEST_RUST} -gt 0 ]; then test_rust fi -else + if [ ${TEST_INTEGRATION} -gt 0 ]; then + test_integration + fi +} + +test_binary_distribution() { : ${BINTRAY_REPOSITORY:=apache/arrow} if [ ${TEST_BINARY} -gt 0 ]; then @@ -500,7 +487,66 @@ else if [ ${TEST_YUM} -gt 0 ]; then test_yum fi +} + +# By default test all functionalities. +# To deactivate one test, deactivate the test and all of its dependents +# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1 +: ${TEST_DEFAULT:=1} +: ${TEST_SOURCE:=${TEST_DEFAULT}} +: ${TEST_JAVA:=${TEST_DEFAULT}} +: ${TEST_CPP:=${TEST_DEFAULT}} +: ${TEST_GLIB:=${TEST_DEFAULT}} +: ${TEST_RUBY:=${TEST_DEFAULT}} +: ${TEST_PYTHON:=${TEST_DEFAULT}} +: ${TEST_JS:=${TEST_DEFAULT}} +: ${TEST_GO:=${TEST_DEFAULT}} +: ${TEST_RUST:=${TEST_DEFAULT}} +: ${TEST_INTEGRATION:=${TEST_DEFAULT}} +: ${TEST_BINARY:=${TEST_DEFAULT}} +: ${TEST_APT:=${TEST_DEFAULT}} +: ${TEST_YUM:=${TEST_DEFAULT}} + +# Automatically test if its activated by a dependent +TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY})) +TEST_PYTHON=$((${TEST_PYTHON} + ${TEST_INTEGRATION})) +TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON})) +TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION})) +TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION})) + +: ${TEST_ARCHIVE:=apache-arrow-${VERSION}.tar.gz} +case "${TEST_ARCHIVE}" in + /*) + ;; + *) + TEST_ARCHIVE=${PWD}/${TEST_ARCHIVE} + ;; +esac + +TEST_SUCCESS=no + +setup_tempdir "arrow-${VERSION}" +echo "Working in sandbox ${TMPDIR}" +cd ${TMPDIR} + +if [ "${ARTIFACT}" == "source" ]; then + dist_name="apache-arrow-${VERSION}" + if [ ${TEST_SOURCE} -gt 0 ]; then + import_gpg_keys + fetch_archive ${dist_name} + tar xf ${dist_name}.tar.gz + else + mkdir -p ${dist_name} + tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1 + fi + pushd ${dist_name} + test_source_distribution + popd +else + import_gpg_keys + test_binary_distribution fi +TEST_SUCCESS=yes echo 'Release candidate looks good!' exit 0 diff --git a/python/setup.py b/python/setup.py index db07d85..319d092 100755 --- a/python/setup.py +++ b/python/setup.py @@ -511,7 +511,7 @@ def _move_shared_libs_unix(build_prefix, build_lib, lib_name): # If the event of not running from a git clone (e.g. from a git archive # or a Python sdist), see if we can set the version number ourselves default_version = '0.14.0-SNAPSHOT' -if (not os.path.exists('../.git/index') +if (not os.path.exists('../.git') and not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): if os.path.exists('PKG-INFO'): # We're probably in a Python sdist, setuptools_scm will handle fine