This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6e6f9b7 ARROW-2083: [CI] Detect changed components on Travis-CI
6e6f9b7 is described below
commit 6e6f9b7860dc756dd233bcac1ea3b91a154d44cc
Author: Antoine Pitrou <[email protected]>
AuthorDate: Thu Feb 8 11:01:40 2018 +0100
ARROW-2083: [CI] Detect changed components on Travis-CI
Hopefully this doesn't add too much maintenance burden.
Also allows to skip Travis builds by putting "[skip travis]" in the last
commit message.
Also skip non-Python non-C++ changes on AppVeyor.
Author: Antoine Pitrou <[email protected]>
Closes #1568 from pitrou/ARROW-2083-detect-changed-components and squashes
the following commits:
cdb371c [Antoine Pitrou] [skip appveyor] ARROW-2083: [CI] Detect changed
components on Travis-CI
---
.travis.yml | 85 +++++++----
appveyor.yml | 8 +
c_glib/meson_options.txt | 1 +
ci/travis_before_script_cpp.sh | 23 +--
ci/travis_build_parquet_cpp.sh | 3 +-
ci/travis_detect_changes.py | 170 +++++++++++++++++++++
ci/travis_install_conda.sh | 39 ++---
...nstall_conda.sh => travis_install_toolchain.sh} | 52 ++++---
ci/travis_lint.sh | 33 ++--
ci/travis_script_integration.sh | 3 +
ci/travis_script_java.sh | 1 -
...vis_script_java.sh => travis_script_javadoc.sh} | 3 +-
ci/travis_script_python.sh | 4 +-
13 files changed, 309 insertions(+), 116 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 58d6786..d591a99 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,11 +46,13 @@ matrix:
allow_failures:
- jdk: oraclejdk9
include:
+ # C++ & Python w/ clang 4.0
- compiler: gcc
language: cpp
os: linux
group: deprecated
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- export ARROW_TRAVIS_USE_TOOLCHAIN=1
- export ARROW_TRAVIS_VALGRIND=1
- export ARROW_TRAVIS_PLASMA=1
@@ -61,12 +63,13 @@ matrix:
- export CXX="clang++-4.0"
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+ # [OS X] C++ & Python w/ XCode 6.4
- compiler: clang
language: cpp
osx_image: xcode6.4
@@ -74,81 +77,96 @@ matrix:
cache:
addons:
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- export ARROW_TRAVIS_USE_TOOLCHAIN=1
- export ARROW_TRAVIS_PLASMA=1
- export ARROW_TRAVIS_ORC=1
- export ARROW_BUILD_WARNING_LEVEL=CHECKIN
- - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then travis_wait 50
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
+ - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7; fi
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6; fi
+ # [manylinux1] Python
- language: cpp
before_script:
- - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull
quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
+ - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
+ # Java w/ OpenJDK 7
- language: java
os: linux
jdk: openjdk7
+ before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+ - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
+ - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh; fi
+ # Java w/ Oracle JDK 9
- language: java
os: linux
- env: ARROW_TRAVIS_SKIP_SITE=yes
jdk: oraclejdk9
+ before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+ - if [ $ARROW_CI_JAVA_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_java.sh; fi
addons:
apt:
packages:
- oracle-java9-installer
+ # Integration w/ OpenJDK 8
- language: java
os: linux
env: ARROW_TEST_GROUP=integration
jdk: openjdk8
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- export CC="clang-4.0"
- export CXX="clang++-4.0"
- nvm install node
- - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+ - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
+ - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
+ - if [ $ARROW_CI_INTEGRATION_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_integration.sh; fi
+ # NodeJS
- language: node_js
os: linux
node_js: node
before_script:
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+ - if [ $ARROW_CI_JS_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_js.sh; fi
+ # C++ & glib w/ gcc 4.9 & autotools
- compiler: gcc
language: cpp
os: linux
env: BUILD_SYSTEM=autotools BUILD_TORCH_EXAMPLE=no
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- export CC="gcc-4.9"
- export CXX="g++-4.9"
- - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+ # C++ & glib w/ gcc 4.9 & meson
- compiler: gcc
language: cpp
os: linux
env: BUILD_SYSTEM=meson BUILD_TORCH_EXAMPLE=no
before_script:
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
- export CC="gcc-4.9"
- export CXX="g++-4.9"
- - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
+ # [OS X] C++ & glib w/ XCode 8.3 & autotools
- compiler: clang
osx_image: xcode8.3
os: osx
@@ -157,11 +175,12 @@ matrix:
rvm: 2.2
env: BUILD_SYSTEM=autotools
before_script:
- - brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ - eval `python $TRAVIS_BUILD_DIR/ci/travis_detect_changes.py`
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then brew update && brew bundle
--file=$TRAVIS_BUILD_DIR/c_glib/Brewfile; fi
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library; fi
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh; fi
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
+ - if [ $ARROW_CI_C_GLIB_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
before_install:
- ulimit -c unlimited -S
diff --git a/appveyor.yml b/appveyor.yml
index ea7922b..22d6158 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -18,6 +18,14 @@
# Operating system (build VM template)
os: Visual Studio 2015
+only_commits:
+ # Skip commits not related to Python or C++
+ files:
+ - appveyor.yml
+ - ci/
+ - cpp/
+ - python/
+
environment:
matrix:
- JOB: "Build"
diff --git a/c_glib/meson_options.txt b/c_glib/meson_options.txt
index 2988e1a..859a4e6 100644
--- a/c_glib/meson_options.txt
+++ b/c_glib/meson_options.txt
@@ -21,3 +21,4 @@ option('enable_gtk_doc',
type: 'boolean',
value: false,
description: 'Build document by GTK-Doc')
+
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 7725c56..4ffe97f 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -31,28 +31,7 @@ fi
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
# Set up C++ toolchain from conda-forge packages for faster builds
- conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
- jemalloc=4.4.0 \
- nomkl \
- boost-cpp \
- rapidjson \
- flatbuffers \
- gflags \
- gtest \
- lz4-c \
- snappy \
- ccache \
- zstd \
- brotli \
- zlib \
- cmake \
- curl \
- thrift-cpp=0.11.0 \
- ninja
-
- # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
- # updated sometime on October 2 or October 3
- conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
+ source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
fi
mkdir -p $ARROW_CPP_BUILD_DIR
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
index 4b6370e..fc4ae72 100755
--- a/ci/travis_build_parquet_cpp.sh
+++ b/ci/travis_build_parquet_cpp.sh
@@ -20,9 +20,10 @@ set -e
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
+
export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
-# $CPP_TOOLCHAIN set up in before_script_cpp
export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
diff --git a/ci/travis_detect_changes.py b/ci/travis_detect_changes.py
new file mode 100644
index 0000000..2842619
--- /dev/null
+++ b/ci/travis_detect_changes.py
@@ -0,0 +1,170 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+
+import functools
+import os
+import pprint
+import sys
+import subprocess
+
+
+perr = functools.partial(print, file=sys.stderr)
+
+LANGUAGE_TOPICS = ['c_glib', 'cpp', 'java', 'js', 'python']
+
+ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'site']
+
+
+def run_cmd(cmdline):
+ proc = subprocess.Popen(cmdline,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+ if proc.returncode != 0:
+ raise RuntimeError("Command {cmdline} failed with code {returncode}, "
+ "stderr was:\n{stderr}\n"
+ .format(cmdline=cmdline, returncode=proc.returncode,
+ stderr=err.decode()))
+ return out
+
+
+def get_commit_description(commit):
+ """
+ Return the textual description (title + body) of the given git commit.
+ """
+ out = run_cmd(["git", "show", "--no-patch", "--pretty=format:%B",
+ commit])
+ return out.decode('utf-8', 'ignore')
+
+
+def list_affected_files(commit_range):
+ """
+ Return a list of files changed by the given git commit range.
+ """
+ out = run_cmd(["git", "diff", "--name-only", commit_range])
+ return list(filter(None, (s.strip() for s in out.decode().splitlines())))
+
+
+def get_travis_head_commit():
+ return os.environ['TRAVIS_COMMIT']
+
+
+def get_travis_commit_range():
+ cr = os.environ['TRAVIS_COMMIT_RANGE']
+ # See
https://github.com/travis-ci/travis-ci/issues/4596#issuecomment-139811122
+ return cr.replace('...', '..')
+
+
+def list_travis_affected_files():
+ """
+ Return a list of files affected in the current Travis build.
+ """
+ commit_range = get_travis_commit_range()
+ try:
+ return list_affected_files(commit_range)
+ except RuntimeError:
+ # TRAVIS_COMMIT_RANGE can contain invalid revisions when
+ # building a branch (not a PR) after rebasing:
+ # https://github.com/travis-ci/travis-ci/issues/2668
+ if os.environ['TRAVIS_EVENT_TYPE'] == 'pull_request':
+ raise
+ # If it's a rebase, it's probably enough to use the last commit only
+ commit_range = '{0}^..'.format(get_travis_head_commit())
+ return list_affected_files(commit_range)
+
+
+def get_affected_topics(affected_files):
+ """
+ Return a dict of topics affected by the given files.
+ Each dict value is True if affected, False otherwise.
+ """
+ affected = dict.fromkeys(ALL_TOPICS, False)
+
+ for path in affected_files:
+ parts = []
+ head = path
+ while head:
+ head, tail = os.path.split(head)
+ parts.append(tail)
+ parts.reverse()
+ assert parts
+ p = parts[0]
+ fn = parts[-1]
+ if fn.startswith('README'):
+ continue
+ if p in ('ci', 'dev', '.travis.yml'):
+ # For these changes, test everything
+ for k in ALL_TOPICS:
+ affected[k] = True
+ break
+ elif p in ('cpp', 'format'):
+ # All languages are potentially affected
+ for k in LANGUAGE_TOPICS:
+ affected[k] = True
+ affected['integration'] = True
+ elif p in ('java', 'js'):
+ affected[p] = True
+ affected['integration'] = True
+ elif p in ('c_glib', 'integration', 'python', 'site'):
+ affected[p] = True
+
+ return affected
+
+
+def make_env_for_topics(affected):
+ return {'ARROW_CI_{0}_AFFECTED'.format(k.upper()): '1' if v else '0'
+ for k, v in affected.items()}
+
+
+def get_unix_shell_eval(env):
+ """
+ Return a shell-evalable string to setup some environment variables.
+ """
+ return "; ".join(("export {0}='{1}'".format(k, v)
+ for k, v in env.items()))
+
+
+def run_from_travis():
+ desc = get_commit_description(get_travis_head_commit())
+ if '[skip travis]' in desc:
+ # Skip everything
+ affected = dict.fromkeys(ALL_TOPICS, False)
+ elif '[force ci]' in desc or '[force travis]' in desc:
+ # Test everything
+ affected = dict.fromkeys(ALL_TOPICS, True)
+ else:
+ # Test affected topics
+ affected_files = list_travis_affected_files()
+ perr("Affected files:", affected_files)
+ affected = get_affected_topics(affected_files)
+ assert set(affected) <= set(ALL_TOPICS), affected
+
+ perr("Affected topics:")
+ perr(pprint.pformat(affected))
+ return get_unix_shell_eval(make_env_for_topics(affected))
+
+
+if __name__ == "__main__":
+ # This script should have its output evaluated by a shell,
+ # e.g. "eval `python ci/travis_detect_changes.py`"
+ try:
+ print(run_from_travis())
+ except:
+ # Make sure the enclosing eval will return an error
+ print("exit 1")
+ raise
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index caec9bb..7896b12 100755
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -19,28 +19,31 @@
set -e
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh";
-else
-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh";
-fi
+if (! which conda > /dev/null ); then
+ if [ $TRAVIS_OS_NAME == "linux" ]; then
+
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh";
+ else
+
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh";
+ fi
-wget --no-verbose -O miniconda.sh $MINICONDA_URL
+ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+ mkdir -p $CONDA_PKGS_DIRS
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-mkdir -p $CONDA_PKGS_DIRS
+ wget --no-verbose -O miniconda.sh $MINICONDA_URL
+ bash miniconda.sh -b -p $MINICONDA
+ export PATH="$MINICONDA/bin:$PATH"
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-conda update -y -q conda
-conda config --set auto_update_conda false
-conda info -a
+ conda update -y -q conda
+ conda config --set auto_update_conda false
+ conda info -a
-conda config --set show_channel_urls True
+ conda config --set show_channel_urls True
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
+ # Help with SSL timeouts to S3
+ conda config --set remote_connect_timeout_secs 12
+
+ conda config --add channels https://repo.continuum.io/pkgs/free
+ conda config --add channels conda-forge
+fi
-conda config --add channels https://repo.continuum.io/pkgs/free
-conda config --add channels conda-forge
conda info -a
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_toolchain.sh
similarity index 54%
copy from ci/travis_install_conda.sh
copy to ci/travis_install_toolchain.sh
index caec9bb..e01a084 100755
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_toolchain.sh
@@ -17,30 +17,32 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-if [ $TRAVIS_OS_NAME == "linux" ]; then
-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh";
-else
-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh";
-fi
-
-wget --no-verbose -O miniconda.sh $MINICONDA_URL
-
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-mkdir -p $CONDA_PKGS_DIRS
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-conda update -y -q conda
-conda config --set auto_update_conda false
-conda info -a
-
-conda config --set show_channel_urls True
-
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
-
-conda config --add channels https://repo.continuum.io/pkgs/free
-conda config --add channels conda-forge
-conda info -a
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
+if [ ! -e $CPP_TOOLCHAIN ]; then
+ # Set up C++ toolchain from conda-forge packages for faster builds
+ conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
+ jemalloc=4.4.0 \
+ nomkl \
+ boost-cpp \
+ rapidjson \
+ flatbuffers \
+ gflags \
+ gtest \
+ lz4-c \
+ snappy \
+ ccache \
+ zstd \
+ brotli \
+ zlib \
+ cmake \
+ curl \
+ thrift-cpp=0.11.0 \
+ ninja
+
+ # HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
+ # updated sometime on October 2 or October 3
+ conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
+fi
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index 096170a..0cbdf1a 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -20,25 +20,32 @@
set -ex
# Fail fast for code linting issues
-mkdir $TRAVIS_BUILD_DIR/cpp/lint
-pushd $TRAVIS_BUILD_DIR/cpp/lint
-cmake ..
-make lint
+if [ "$ARROW_CI_CPP_AFFECTED" != "0" ]; then
+ mkdir $TRAVIS_BUILD_DIR/cpp/lint
+ pushd $TRAVIS_BUILD_DIR/cpp/lint
-if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
- make check-format
+ cmake ..
+ make lint
+
+ if [ "$ARROW_TRAVIS_CLANG_FORMAT" == "1" ]; then
+ make check-format
+ fi
+
+ popd
fi
-popd
# Fail fast on style checks
-sudo pip install -q flake8
-PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+if [ "$ARROW_CI_PYTHON_AFFECTED" != "0" ]; then
+ sudo pip install -q flake8
-flake8 --count $PYTHON_DIR/pyarrow
+ PYTHON_DIR=$TRAVIS_BUILD_DIR/python
-# Check Cython files with some checks turned off
-flake8 --count --config=$PYTHON_DIR/.flake8.cython \
- $PYTHON_DIR/pyarrow
+ flake8 --count $PYTHON_DIR/pyarrow
+
+ # Check Cython files with some checks turned off
+ flake8 --count --config=$PYTHON_DIR/.flake8.cython \
+ $PYTHON_DIR/pyarrow
+fi
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 0c415dc..8c6da38 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -20,6 +20,9 @@
set -e
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
pushd $ARROW_JAVA_DIR
diff --git a/ci/travis_script_java.sh b/ci/travis_script_java.sh
index 58e5d42..a8ad94c 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_java.sh
@@ -25,6 +25,5 @@ pushd $JAVA_DIR
export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
mvn -B install
-[ "${ARROW_TRAVIS_SKIP_SITE}" = "yes" ] || mvn -B site
popd
diff --git a/ci/travis_script_java.sh b/ci/travis_script_javadoc.sh
similarity index 93%
copy from ci/travis_script_java.sh
copy to ci/travis_script_javadoc.sh
index 58e5d42..ccfb2da 100755
--- a/ci/travis_script_java.sh
+++ b/ci/travis_script_javadoc.sh
@@ -24,7 +24,6 @@ JAVA_DIR=${TRAVIS_BUILD_DIR}/java
pushd $JAVA_DIR
export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
-mvn -B install
-[ "${ARROW_TRAVIS_SKIP_SITE}" = "yes" ] || mvn -B site
+mvn -B site
popd
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 4187e75..dbaaab8 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -20,6 +20,8 @@ set -e
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
export ARROW_HOME=$ARROW_CPP_INSTALL
export PARQUET_HOME=$ARROW_PYTHON_PARQUET_HOME
export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
@@ -102,6 +104,6 @@ python -m pytest -vv -r sxX --durations=15 -s $PYARROW_PATH
--parquet
if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
# Build documentation once
pushd $ARROW_PYTHON_DIR/doc
- sphinx-build -b html -d _build/doctrees -W source _build/html
+ sphinx-build -q -b html -d _build/doctrees -W source _build/html
popd
fi
--
To stop receiving notification emails like this one, please contact
[email protected].
